move scripts in irace/expe/alpha/

eo/contrib/irace/expe/alpha/parse_baseline.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import sys
+
+print("algo,problem,seed,ECDF_AUC")
+
+algos_names = {
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":1, "replacement":0}) : "EA",
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":5, "replacement":0}) : "fEA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":2, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "xGA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":5, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "1ptGA",
+}
+
+for fname in sys.argv[1:]:
+
+    run = {}
+    for f in fname.strip(".dat").split("_"):
+        kv = f.split("=")
+        assert(len(kv)==2),str(kv)+" "+str(len(kv))
+        key,idx = kv[0], int(kv[1])
+        run[key] = idx
+
+    with open(fname) as fd:
+        auc = int(fd.readlines()[0]) * -1
+
+    algo = str({"crossover-rate":run["crossover-rate"], "cross-selector":run["cross-selector"], "crossover":run["crossover"], "mutation-rate":run["mutation-rate"], "mut-selector":run["mut-selector"], "mutation":run["mutation"], "replacement":run["replacement"]})
+
+    print(algos_names[algo], run["pb"], run["seed"], auc, sep=",")

eo/contrib/irace/expe/alpha/parse_baseline_average.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import sys
+
+print("problem,EA,fEA,xGA,1ptGA")
+
+algos_names = {
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":1, "replacement":0}) : "EA",
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":5, "replacement":0}) : "fEA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":2, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "xGA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":5, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "1ptGA",
+}
+
+data = {}
+
+# Parse
+for fname in sys.argv[1:]:
+
+    run = {}
+    for f in fname.strip(".dat").split("_"):
+        kv = f.split("=")
+        assert(len(kv)==2),str(kv)+" "+str(len(kv))
+        key,idx = kv[0], int(kv[1])
+        run[key] = idx
+
+    with open(fname) as fd:
+        auc = int(fd.readlines()[0]) * -1
+
+    algo = str({"crossover-rate":run["crossover-rate"], "cross-selector":run["cross-selector"], "crossover":run["crossover"], "mutation-rate":run["mutation-rate"], "mut-selector":run["mut-selector"], "mutation":run["mutation"], "replacement":run["replacement"]})
+
+    if run["pb"] in data:
+        data[run["pb"]][algos_names[algo]].append(auc)
+    else:
+        data[run["pb"]] = {
+                "EA"   : [],
+                "fEA"  : [],
+                "xGA"  : [],
+                "1ptGA": [],
+            }
+        data[run["pb"]][algos_names[algo]].append(auc)
+
+
+# Print CSV
+for pb in sorted(data.keys()):
+    print(pb, end="")
+    for algo in ["EA","fEA","xGA","1ptGA"]:
+        res = data[pb][algo]
+        print(",", sum(res)/len(res), end="", sep="")
+    print()

eo/contrib/irace/expe/alpha/parse_elites.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import sys
+
+print("algo,problem,seed,ECDF_AUC")
+
+for fname in sys.argv[1:]:
+
+    run = {}
+    for f in fname.strip(".dat").split("_"):
+        kv = f.split("=")
+        assert(len(kv)==2),str(kv)+" "+str(len(kv))
+        key,idx = kv[0], int(kv[1])
+        run[key] = idx
+
+    with open(fname) as fd:
+        auc = int(fd.readlines()[0]) * -1
+
+    algo = "pc={}_c={}_C={}_pm={}_m={}_M={}_R={}".format(run["crossover-rate"],run["cross-selector"],run["crossover"],run["mutation-rate"],run["mut-selector"],run["mutation"],run["replacement"])
+
+    print(algo, run["pb"], run["seed"], auc, sep=",")

eo/contrib/irace/expe/alpha/parse_irace_bests.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+
+print("pb,ecdf,id,crossover-rate,cross-selector,crossover,mutation-rate,mut-selector,mutation,replacement")
+for datadir in sys.argv[1:]:
+
+    for pb_dir in os.listdir(datadir):
+        if "results_problem" in pb_dir:
+            pb_id=pb_dir.replace("results_problem_","")
+            with open(os.path.join("./",datadir,pb_dir,"irace.log")) as fd:
+                data = fd.readlines()
+
+                # Find the last best configuration
+                bests = [line.strip() for line in data if "Best-so-far" in line]
+                best = bests[-1].split()
+                best_id, best_perf = best[2], best[5]
+                # print(best_id,best_perf)
+
+                # Filter the config detail
+                configs = [line.strip() for line in data if "--crossover-rate=" in line and best_id in line]
+                # print(configs)
+
+                # Format as CSV
+                for config in configs:
+                    algo = re.sub("\-\-\S*=", ",", config)
+                    csv_line = pb_id + "," + best_perf + "," + algo
+                    print(csv_line.replace(" ",""))

eo/contrib/irace/expe/alpha/plot_attain_mat.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+for p in range(18):
+
+    print("Pb",p,end=": ")
+    datadir="attain_mat_{pb}".format(pb=p)
+
+    try:
+        os.mkdir(datadir)
+    except FileExistsError:
+        pass
+
+    for i in range(50):
+        cmd="./release/fastga --seed={i} \
+            --crossover-rate=2 --cross-selector=2 --crossover=5 --mutation-rate=2 --mut-selector=2 --mutation=1 --replacement=0 \
+            --problem={pb} --buckets=20 --output-mat 2>/dev/null > {dir}/output_mat_{i}.csv"\
+            .format(dir=datadir, i=i, pb=p)
+        # print(cmd)
+        print(i,end=" ",flush=True)
+        os.system(cmd)
+
+
+    matrices=[]
+    for root, dirs, files in os.walk(datadir):
+        for filename in files:
+            matrices.append( np.genfromtxt(datadir+"/"+filename,delimiter=',') )
+
+    agg = matrices[0]
+    for mat in matrices[1:]:
+        agg += mat
+
+    # print(agg)
+
+    plt.rcParams["figure.figsize"] = (3,3)
+    plt.gca().pcolor(agg, edgecolors='grey', cmap="Blues")
+    plt.gca().set_xlabel("Time budget")
+    plt.gca().set_ylabel("Target")
+    plt.savefig("aittain_map_{pb}.png".format(pb=p), bbox_inches='tight')
+
+    print(".")

eo/contrib/irace/expe/alpha/run_algo.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# Array of problems.
+# You may set something like: (0 2 5 17)
+problems=($(seq 0 18))
+
+# Capture anything passed to the script
+outdir="$1"
+algo="${@:2}"
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+i=1 # Loop counter.
+for pb in "${problems[@]}" ; do # Iterate over the problems array.
+    for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+        # Forge a directory/log file name
+        # (remove double dashs and replace spaces with underscore).
+        name="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+
+        # This is the command to be ran.
+        cmd="${exe} --problem=${pb} --seed=${seed} ${algo}"
+        echo ${cmd} # Print the command.
+
+        # Progress print.
+        echo -n "problem ${pb}, run ${seed}"
+
+        # Actually start the command.
+        ${cmd} > "${outdir}/${name}.dat" 2> "${outdir}/${name}.log"
+
+        # Check for the most common problem in the log file.
+        cat "${outdir}/${name}.log" | grep "illogical performance"
+
+        perc=$(echo "scale=2;${i}/(${#problems[@]}*${runs})*100" | bc)
+        echo -e " -- ${perc}%"
+        i=$((i+1))
+    done
+done
+
+echo "Done $((${#problems[@]}*${runs})) runs, results in ${outdir}"

eo/contrib/irace/expe/alpha/run_baseline.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+outdir="$(date --iso-8601=minutes)_results_baselines"
+mkdir ${outdir}
+
+algos=(
+# (λ+λ)EA
+"--full-log=1 --crossover-rate=0 --cross-selector=0 --crossover=0 --mutation-rate=0 --mut-selector=0 --mutation=1 --replacement=0"
+# (λ+λ)fEA
+"--full-log=1 --crossover-rate=0 --cross-selector=0 --crossover=0 --mutation-rate=0 --mut-selector=0 --mutation=5 --replacement=0"
+# (λ+λ)xGA
+"--full-log=1 --crossover-rate=2 --cross-selector=2 --crossover=2 --mutation-rate=2 --mut-selector=2 --mutation=1 --replacement=0"
+# (λ+λ)1ptGA
+"--full-log=1 --crossover-rate=2 --cross-selector=2 --crossover=5 --mutation-rate=2 --mut-selector=2 --mutation=1 --replacement=0"
+)
+
+
+i=1 # Loop counter.
+for algo in "${algos[@]}" ; do
+    echo "${algo}"
+
+    name="$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+    ./run_algo.sh ${outdir} ${algo} &> "expe_${name}.log"
+
+    perc=$(echo "scale=2;${i}/${#algos[@]}*100" | bc)
+    echo -e "${perc}%\n"
+    i=$((i+1))
+done
+
+echo "Done"

eo/contrib/irace/expe/alpha/run_elite.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+outdir="$(date --iso-8601=minutes)_results_elites"
+mkdir ${outdir}
+
+# FIXME
+algos=(
+"--crossover-rate=1 --cross-selector=2 --crossover=1  --mutation-rate=2 --mut-selector=2 --mutation=8  --replacement=8"
+"--crossover-rate=4 --cross-selector=5 --crossover=2  --mutation-rate=3 --mut-selector=4 --mutation=9  --replacement=9"
+"--crossover-rate=1 --cross-selector=3 --crossover=8  --mutation-rate=2 --mut-selector=6 --mutation=3  --replacement=2"
+"--crossover-rate=2 --cross-selector=1 --crossover=1  --mutation-rate=2 --mut-selector=6 --mutation=9  --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=3 --mutation=7  --replacement=0"
+"--crossover-rate=0 --cross-selector=3 --crossover=2  --mutation-rate=4 --mut-selector=2 --mutation=6  --replacement=0"
+"--crossover-rate=3 --cross-selector=0 --crossover=3  --mutation-rate=4 --mut-selector=3 --mutation=10 --replacement=0"
+"--crossover-rate=0 --cross-selector=1 --crossover=0  --mutation-rate=3 --mut-selector=2 --mutation=10 --replacement=0"
+"--crossover-rate=2 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=5 --mutation=10 --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=5 --mutation=9  --replacement=0"
+"--crossover-rate=3 --cross-selector=2 --crossover=10 --mutation-rate=4 --mut-selector=2 --mutation=10 --replacement=0"
+"--crossover-rate=2 --cross-selector=2 --crossover=5  --mutation-rate=4 --mut-selector=3 --mutation=9  --replacement=0"
+"--crossover-rate=3 --cross-selector=6 --crossover=2  --mutation-rate=4 --mut-selector=1 --mutation=10 --replacement=0"
+"--crossover-rate=1 --cross-selector=5 --crossover=9  --mutation-rate=4 --mut-selector=2 --mutation=8  --replacement=0"
+"--crossover-rate=2 --cross-selector=5 --crossover=2  --mutation-rate=4 --mut-selector=6 --mutation=8  --replacement=0"
+"--crossover-rate=2 --cross-selector=2 --crossover=10 --mutation-rate=4 --mut-selector=6 --mutation=10 --replacement=0"
+"--crossover-rate=3 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=5 --mutation=10 --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=1 --mutation=8  --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=6 --mutation=9  --replacement=0"
+)
+
+pb=0 # Loop counter.
+for algo in "${algos[@]}" ; do
+    echo "Problem ${pb}"
+    echo -n "Runs: "
+
+    for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+        # This is the command to be ran.
+        cmd="${exe} --full-log=1 --problem=${pb} --seed=${seed} ${algo}"
+        # echo ${cmd} # Print the command.
+
+        # Forge a directory/log file name
+        # (remove double dashs and replace spaces with underscore).
+        name_run="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+        # echo $name_run
+
+        # Progress print.
+        echo -n "${seed} "
+
+        # Actually start the command.
+        ${cmd} > "${outdir}/${name_run}.dat" 2> "${outdir}/${name_run}.log"
+
+        # Check for the most common problem in the log file.
+        cat "${outdir}/${name_run}.log" | grep "illogical performance"
+    done
+
+    echo ""
+    perc=$(echo "scale=2;${pb}/${#algos[@]}*100" | bc)
+    echo -e "${perc}%\n"
+    pb=$((pb+1))
+
+done
+
+echo "Done"

eo/contrib/irace/expe/alpha/run_elites_all.sh

@@ -0,0 +1,61 @@
+
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+outdir="$(date --iso-8601=minutes)_results_elites_all"
+mkdir -p ${outdir}
+mkdir -p ${outdir}/raw
+mkdir -p ${outdir}/raw/data
+mkdir -p ${outdir}/raw/logs
+
+n=0
+algoid=0
+for line in $(cat results_irace_100k.csv|cut -s -d"," -f4-10); do
+    echo ""
+    date
+
+    a=($(echo $line | sed "s/,/ /g"))
+    algo="--crossover-rate=${a[0]} --cross-selector=${a[1]} --crossover=${a[2]}  --mutation-rate=${a[3]} --mut-selector=${a[4]} --mutation=${a[5]}  --replacement=${a[6]}"
+
+    for pb in $(seq 0 18) ; do
+        perc=$(echo "scale=3;${n}/(285*18)*100.0" | bc)
+        echo "${perc}% : algo ${algoid}/285, problem ${pb}/18"
+        # echo -n "Runs: "
+
+        for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+            # This is the command to be ran.
+            cmd="${exe} --full-log=1 --problem=${pb} --seed=${seed} ${algo}"
+            # echo ${cmd} # Print the command.
+
+            # Forge a directory/log file name
+            # (remove double dashs and replace spaces with underscore).
+            name_run="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+            # echo $name_run
+
+            # Progress print.
+            # echo -n "${seed} "
+
+            # Actually start the command.
+            ${cmd} > "${outdir}/raw/data/${name_run}.dat" 2> "${outdir}/raw/logs/${name_run}.log"
+
+            # Check for the most common problem in the log file.
+            cat "${outdir}/raw/logs/${name_run}.log" | grep "illogical performance"
+        done # seed
+        # echo ""
+
+        n=$(($n+1))
+    done # pb
+
+    algoid=$(($algoid+1))
+done
+
+# Move IOH logs in the results directory.
+mv ./FastGA_* ${outdir}
+
+echo "Done"
+date

eo/contrib/irace/expe/alpha/run_irace.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+outdir="$(date --iso-8601=minutes)_results_irace"
+mkdir ${outdir}
+cd ${outdir}
+
+for p in $(seq 0 18) ; do
+    echo -n "Problem ${p}... "
+    res="results_problem_${p}"
+    mkdir ${res}
+    cd ${res}
+
+    # Fore some reason, irace absolutely need those files...
+    cp ../../irace-config/example.scen .
+    cp ../../irace-config/default.instances .
+    cp ../../release/fastga .
+    cat ../../irace-config/target-runner | sed "s/{{PROBLEM}}/${p}/" > ./target-runner
+    chmod u+x target-runner
+
+    # Generate the parameter list file.
+    ./fastga -h > fastga.param 2>/dev/null
+    # /usr/lib/R/site-library/irace/bin/irace --scenario example.scen 2>&1 | tee irace.log
+    /usr/lib/R/site-library/irace/bin/irace --scenario example.scen &> irace.log
+
+    cd ..
+    echo " done"
+done

eo/contrib/irace/expe/alpha/run_irace_all.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+date -Iseconds
+echo "STARTS"
+
+for r in $(seq 15); do
+    echo "Run $r/15";
+    date -Iseconds
+    ./run_irace_parallel-batch.sh
+    date -Iseconds
+done
+
+echo "DONE"
+date -Iseconds

eo/contrib/irace/expe/alpha/run_irace_parallel-batch.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+outdir="$(date --iso-8601=ns)_results_irace"
+mkdir ${outdir}
+cd ${outdir}
+
+run(){
+    p="$1"
+
+    echo "Problem ${p}"
+    res="results_problem_${p}"
+    mkdir ${res}
+    cd ${res}
+
+    # Fore some reason, irace absolutely need those files...
+    cp ../../irace-config/example.scen .
+    cp ../../irace-config/default.instances .
+    cp ../../release/fastga .
+    cat ../../irace-config/target-runner | sed "s/{{PROBLEM}}/${p}/" > ./target-runner
+    chmod u+x target-runner
+
+    # Generate the parameter list file.
+    ./fastga -h > fastga.param 2>/dev/null
+    # /usr/lib/R/site-library/irace/bin/irace --scenario example.scen 2>&1 | tee irace.log
+    /usr/lib/R/site-library/irace/bin/irace --scenario example.scen &> irace.log
+
+    cd ..
+    echo "Done problem ${p}"
+}
+
+N=5 # Somehow 5 is the fastest on my 4-cores machine.
+(
+for pb in $(seq 0 18); do 
+    ((i=i%N)); ((i++==0)) && wait
+    run "$pb" &
+done
+wait
+)

eo/contrib/irace/expe/alpha/run_randoms.sh

@@ -0,0 +1,61 @@
+
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+outdir="$(date --iso-8601=minutes)_results_randoms"
+mkdir -p ${outdir}
+mkdir -p ${outdir}/raw
+mkdir -p ${outdir}/raw/data
+mkdir -p ${outdir}/raw/logs
+
+n=1
+algoid=0
+for algoid in $(seq 0 100); do
+    echo ""
+    date
+
+    a=( $((RANDOM%5)) $((RANDOM%7)) $((RANDOM%11)) $((RANDOM%5)) $((RANDOM%7)) $((RANDOM%11)) $((RANDOM%11)) )
+    algo="--crossover-rate=${a[0]} --cross-selector=${a[1]} --crossover=${a[2]}  --mutation-rate=${a[3]} --mut-selector=${a[4]} --mutation=${a[5]}  --replacement=${a[6]}"
+
+    for pb in $(seq 0 18) ; do
+        perc=$(echo "scale=3;${n}/(100*18)*100.0" | bc)
+        echo "${perc}% : algo ${algoid}/100, problem ${pb}/18"
+        # echo -n "Runs: "
+
+        for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+            # This is the command to be ran.
+            cmd="${exe} --full-log=1 --problem=${pb} --seed=${seed} ${algo}"
+            # echo ${cmd} # Print the command.
+
+            # Forge a directory/log file name
+            # (remove double dashs and replace spaces with underscore).
+            name_run="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+            # echo $name_run
+
+            # Progress print.
+            # echo -n "${seed} "
+
+            # Actually start the command.
+            ${cmd} > "${outdir}/raw/data/${name_run}.dat" 2> "${outdir}/raw/logs/${name_run}.log"
+
+            # Check for the most common problem in the log file.
+            cat "${outdir}/raw/logs/${name_run}.log" | grep "illogical performance"
+        done # seed
+        # echo ""
+
+        n=$(($n+1))
+    done # pb
+
+    algoid=$(($algoid+1))
+done
+
+# Move IOH logs in the results directory.
+mv ./FastGA_* ${outdir}
+
+echo "Done"
+date