Merge branch 'feat+num_foundry' of github.jdreo:jdreo/paradiseo

eo/contrib/irace/expe/alpha/parse_baseline.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import sys
+
+print("algo,problem,seed,ECDF_AUC")
+
+algos_names = {
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":1, "replacement":0}) : "EA",
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":5, "replacement":0}) : "fEA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":2, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "xGA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":5, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "1ptGA",
+}
+
+for fname in sys.argv[1:]:
+
+    run = {}
+    for f in fname.strip(".dat").split("_"):
+        kv = f.split("=")
+        assert(len(kv)==2),str(kv)+" "+str(len(kv))
+        key,idx = kv[0], int(kv[1])
+        run[key] = idx
+
+    with open(fname) as fd:
+        auc = int(fd.readlines()[0]) * -1
+
+    algo = str({"crossover-rate":run["crossover-rate"], "cross-selector":run["cross-selector"], "crossover":run["crossover"], "mutation-rate":run["mutation-rate"], "mut-selector":run["mut-selector"], "mutation":run["mutation"], "replacement":run["replacement"]})
+
+    print(algos_names[algo], run["pb"], run["seed"], auc, sep=",")

eo/contrib/irace/expe/alpha/parse_baseline_average.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import sys
+
+print("problem,EA,fEA,xGA,1ptGA")
+
+algos_names = {
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":1, "replacement":0}) : "EA",
+    str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":5, "replacement":0}) : "fEA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":2, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "xGA",
+    str({"crossover-rate":2, "cross-selector":2, "crossover":5, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "1ptGA",
+}
+
+data = {}
+
+# Parse
+for fname in sys.argv[1:]:
+
+    run = {}
+    for f in fname.strip(".dat").split("_"):
+        kv = f.split("=")
+        assert(len(kv)==2),str(kv)+" "+str(len(kv))
+        key,idx = kv[0], int(kv[1])
+        run[key] = idx
+
+    with open(fname) as fd:
+        auc = int(fd.readlines()[0]) * -1
+
+    algo = str({"crossover-rate":run["crossover-rate"], "cross-selector":run["cross-selector"], "crossover":run["crossover"], "mutation-rate":run["mutation-rate"], "mut-selector":run["mut-selector"], "mutation":run["mutation"], "replacement":run["replacement"]})
+
+    if run["pb"] in data:
+        data[run["pb"]][algos_names[algo]].append(auc)
+    else:
+        data[run["pb"]] = {
+                "EA"   : [],
+                "fEA"  : [],
+                "xGA"  : [],
+                "1ptGA": [],
+            }
+        data[run["pb"]][algos_names[algo]].append(auc)
+
+
+# Print CSV
+for pb in sorted(data.keys()):
+    print(pb, end="")
+    for algo in ["EA","fEA","xGA","1ptGA"]:
+        res = data[pb][algo]
+        print(",", sum(res)/len(res), end="", sep="")
+    print()

eo/contrib/irace/expe/alpha/parse_elites.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import sys
+
+print("algo,problem,seed,ECDF_AUC")
+
+for fname in sys.argv[1:]:
+
+    run = {}
+    for f in fname.strip(".dat").split("_"):
+        kv = f.split("=")
+        assert(len(kv)==2),str(kv)+" "+str(len(kv))
+        key,idx = kv[0], int(kv[1])
+        run[key] = idx
+
+    with open(fname) as fd:
+        auc = int(fd.readlines()[0]) * -1
+
+    algo = "pc={}_c={}_C={}_pm={}_m={}_M={}_R={}".format(run["crossover-rate"],run["cross-selector"],run["crossover"],run["mutation-rate"],run["mut-selector"],run["mutation"],run["replacement"])
+
+    print(algo, run["pb"], run["seed"], auc, sep=",")

eo/contrib/irace/expe/alpha/parse_irace_bests.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+
+print("pb,ecdf,id,crossover-rate,cross-selector,crossover,mutation-rate,mut-selector,mutation,replacement")
+for datadir in sys.argv[1:]:
+
+    for pb_dir in os.listdir(datadir):
+        if "results_problem" in pb_dir:
+            pb_id=pb_dir.replace("results_problem_","")
+            with open(os.path.join("./",datadir,pb_dir,"irace.log")) as fd:
+                data = fd.readlines()
+
+                # Find the last best configuration
+                bests = [line.strip() for line in data if "Best-so-far" in line]
+                best = bests[-1].split()
+                best_id, best_perf = best[2], best[5]
+                # print(best_id,best_perf)
+
+                # Filter the config detail
+                configs = [line.strip() for line in data if "--crossover-rate=" in line and best_id in line]
+                # print(configs)
+
+                # Format as CSV
+                for config in configs:
+                    algo = re.sub("\-\-\S*=", ",", config)
+                    csv_line = pb_id + "," + best_perf + "," + algo
+                    print(csv_line.replace(" ",""))

eo/contrib/irace/expe/alpha/plot_attain_mat.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+for p in range(18):
+
+    print("Pb",p,end=": ")
+    datadir="attain_mat_{pb}".format(pb=p)
+
+    try:
+        os.mkdir(datadir)
+    except FileExistsError:
+        pass
+
+    for i in range(50):
+        cmd="./release/fastga --seed={i} \
+            --crossover-rate=2 --cross-selector=2 --crossover=5 --mutation-rate=2 --mut-selector=2 --mutation=1 --replacement=0 \
+            --problem={pb} --buckets=20 --output-mat 2>/dev/null > {dir}/output_mat_{i}.csv"\
+            .format(dir=datadir, i=i, pb=p)
+        # print(cmd)
+        print(i,end=" ",flush=True)
+        os.system(cmd)
+
+
+    matrices=[]
+    for root, dirs, files in os.walk(datadir):
+        for filename in files:
+            matrices.append( np.genfromtxt(datadir+"/"+filename,delimiter=',') )
+
+    agg = matrices[0]
+    for mat in matrices[1:]:
+        agg += mat
+
+    # print(agg)
+
+    plt.rcParams["figure.figsize"] = (3,3)
+    plt.gca().pcolor(agg, edgecolors='grey', cmap="Blues")
+    plt.gca().set_xlabel("Time budget")
+    plt.gca().set_ylabel("Target")
+    plt.savefig("aittain_map_{pb}.png".format(pb=p), bbox_inches='tight')
+
+    print(".")

eo/contrib/irace/expe/alpha/run_algo.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# Array of problems.
+# You may set something like: (0 2 5 17)
+problems=($(seq 0 18))
+
+# Capture anything passed to the script
+outdir="$1"
+algo="${@:2}"
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+i=1 # Loop counter.
+for pb in "${problems[@]}" ; do # Iterate over the problems array.
+    for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+        # Forge a directory/log file name
+        # (remove double dashs and replace spaces with underscore).
+        name="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+
+        # This is the command to be ran.
+        cmd="${exe} --problem=${pb} --seed=${seed} ${algo}"
+        echo ${cmd} # Print the command.
+
+        # Progress print.
+        echo -n "problem ${pb}, run ${seed}"
+
+        # Actually start the command.
+        ${cmd} > "${outdir}/${name}.dat" 2> "${outdir}/${name}.log"
+
+        # Check for the most common problem in the log file.
+        cat "${outdir}/${name}.log" | grep "illogical performance"
+
+        perc=$(echo "scale=2;${i}/(${#problems[@]}*${runs})*100" | bc)
+        echo -e " -- ${perc}%"
+        i=$((i+1))
+    done
+done
+
+echo "Done $((${#problems[@]}*${runs})) runs, results in ${outdir}"

eo/contrib/irace/expe/alpha/run_baseline.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+outdir="$(date --iso-8601=minutes)_results_baselines"
+mkdir ${outdir}
+
+algos=(
+# (λ+λ)EA
+"--full-log=1 --crossover-rate=0 --cross-selector=0 --crossover=0 --mutation-rate=0 --mut-selector=0 --mutation=1 --replacement=0"
+# (λ+λ)fEA
+"--full-log=1 --crossover-rate=0 --cross-selector=0 --crossover=0 --mutation-rate=0 --mut-selector=0 --mutation=5 --replacement=0"
+# (λ+λ)xGA
+"--full-log=1 --crossover-rate=2 --cross-selector=2 --crossover=2 --mutation-rate=2 --mut-selector=2 --mutation=1 --replacement=0"
+# (λ+λ)1ptGA
+"--full-log=1 --crossover-rate=2 --cross-selector=2 --crossover=5 --mutation-rate=2 --mut-selector=2 --mutation=1 --replacement=0"
+)
+
+
+i=1 # Loop counter.
+for algo in "${algos[@]}" ; do
+    echo "${algo}"
+
+    name="$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+    ./run_algo.sh ${outdir} ${algo} &> "expe_${name}.log"
+
+    perc=$(echo "scale=2;${i}/${#algos[@]}*100" | bc)
+    echo -e "${perc}%\n"
+    i=$((i+1))
+done
+
+echo "Done"

eo/contrib/irace/expe/alpha/run_elite.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+outdir="$(date --iso-8601=minutes)_results_elites"
+mkdir ${outdir}
+
+# FIXME
+algos=(
+"--crossover-rate=1 --cross-selector=2 --crossover=1  --mutation-rate=2 --mut-selector=2 --mutation=8  --replacement=8"
+"--crossover-rate=4 --cross-selector=5 --crossover=2  --mutation-rate=3 --mut-selector=4 --mutation=9  --replacement=9"
+"--crossover-rate=1 --cross-selector=3 --crossover=8  --mutation-rate=2 --mut-selector=6 --mutation=3  --replacement=2"
+"--crossover-rate=2 --cross-selector=1 --crossover=1  --mutation-rate=2 --mut-selector=6 --mutation=9  --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=3 --mutation=7  --replacement=0"
+"--crossover-rate=0 --cross-selector=3 --crossover=2  --mutation-rate=4 --mut-selector=2 --mutation=6  --replacement=0"
+"--crossover-rate=3 --cross-selector=0 --crossover=3  --mutation-rate=4 --mut-selector=3 --mutation=10 --replacement=0"
+"--crossover-rate=0 --cross-selector=1 --crossover=0  --mutation-rate=3 --mut-selector=2 --mutation=10 --replacement=0"
+"--crossover-rate=2 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=5 --mutation=10 --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=5 --mutation=9  --replacement=0"
+"--crossover-rate=3 --cross-selector=2 --crossover=10 --mutation-rate=4 --mut-selector=2 --mutation=10 --replacement=0"
+"--crossover-rate=2 --cross-selector=2 --crossover=5  --mutation-rate=4 --mut-selector=3 --mutation=9  --replacement=0"
+"--crossover-rate=3 --cross-selector=6 --crossover=2  --mutation-rate=4 --mut-selector=1 --mutation=10 --replacement=0"
+"--crossover-rate=1 --cross-selector=5 --crossover=9  --mutation-rate=4 --mut-selector=2 --mutation=8  --replacement=0"
+"--crossover-rate=2 --cross-selector=5 --crossover=2  --mutation-rate=4 --mut-selector=6 --mutation=8  --replacement=0"
+"--crossover-rate=2 --cross-selector=2 --crossover=10 --mutation-rate=4 --mut-selector=6 --mutation=10 --replacement=0"
+"--crossover-rate=3 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=5 --mutation=10 --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=1 --mutation=8  --replacement=0"
+"--crossover-rate=4 --cross-selector=2 --crossover=2  --mutation-rate=4 --mut-selector=6 --mutation=9  --replacement=0"
+)
+
+pb=0 # Loop counter.
+for algo in "${algos[@]}" ; do
+    echo "Problem ${pb}"
+    echo -n "Runs: "
+
+    for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+        # This is the command to be ran.
+        cmd="${exe} --full-log=1 --problem=${pb} --seed=${seed} ${algo}"
+        # echo ${cmd} # Print the command.
+
+        # Forge a directory/log file name
+        # (remove double dashs and replace spaces with underscore).
+        name_run="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+        # echo $name_run
+
+        # Progress print.
+        echo -n "${seed} "
+
+        # Actually start the command.
+        ${cmd} > "${outdir}/${name_run}.dat" 2> "${outdir}/${name_run}.log"
+
+        # Check for the most common problem in the log file.
+        cat "${outdir}/${name_run}.log" | grep "illogical performance"
+    done
+
+    echo ""
+    perc=$(echo "scale=2;${pb}/${#algos[@]}*100" | bc)
+    echo -e "${perc}%\n"
+    pb=$((pb+1))
+
+done
+
+echo "Done"

eo/contrib/irace/expe/alpha/run_elites_all.sh

@@ -0,0 +1,61 @@
+
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+outdir="$(date --iso-8601=minutes)_results_elites_all"
+mkdir -p ${outdir}
+mkdir -p ${outdir}/raw
+mkdir -p ${outdir}/raw/data
+mkdir -p ${outdir}/raw/logs
+
+n=0
+algoid=0
+for line in $(cat results_irace_100k.csv|cut -s -d"," -f4-10); do
+    echo ""
+    date
+
+    a=($(echo $line | sed "s/,/ /g"))
+    algo="--crossover-rate=${a[0]} --cross-selector=${a[1]} --crossover=${a[2]}  --mutation-rate=${a[3]} --mut-selector=${a[4]} --mutation=${a[5]}  --replacement=${a[6]}"
+
+    for pb in $(seq 0 18) ; do
+        perc=$(echo "scale=3;${n}/(285*18)*100.0" | bc)
+        echo "${perc}% : algo ${algoid}/285, problem ${pb}/18"
+        # echo -n "Runs: "
+
+        for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+            # This is the command to be ran.
+            cmd="${exe} --full-log=1 --problem=${pb} --seed=${seed} ${algo}"
+            # echo ${cmd} # Print the command.
+
+            # Forge a directory/log file name
+            # (remove double dashs and replace spaces with underscore).
+            name_run="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+            # echo $name_run
+
+            # Progress print.
+            # echo -n "${seed} "
+
+            # Actually start the command.
+            ${cmd} > "${outdir}/raw/data/${name_run}.dat" 2> "${outdir}/raw/logs/${name_run}.log"
+
+            # Check for the most common problem in the log file.
+            cat "${outdir}/raw/logs/${name_run}.log" | grep "illogical performance"
+        done # seed
+        # echo ""
+
+        n=$(($n+1))
+    done # pb
+
+    algoid=$(($algoid+1))
+done
+
+# Move IOH logs in the results directory.
+mv ./FastGA_* ${outdir}
+
+echo "Done"
+date

eo/contrib/irace/expe/alpha/run_irace.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+outdir="$(date --iso-8601=minutes)_results_irace"
+mkdir ${outdir}
+cd ${outdir}
+
+for p in $(seq 0 18) ; do
+    echo -n "Problem ${p}... "
+    res="results_problem_${p}"
+    mkdir ${res}
+    cd ${res}
+
+    # Fore some reason, irace absolutely need those files...
+    cp ../../irace-config/example.scen .
+    cp ../../irace-config/default.instances .
+    cp ../../release/fastga .
+    cat ../../irace-config/target-runner | sed "s/{{PROBLEM}}/${p}/" > ./target-runner
+    chmod u+x target-runner
+
+    # Generate the parameter list file.
+    ./fastga -h > fastga.param 2>/dev/null
+    # /usr/lib/R/site-library/irace/bin/irace --scenario example.scen 2>&1 | tee irace.log
+    /usr/lib/R/site-library/irace/bin/irace --scenario example.scen &> irace.log
+
+    cd ..
+    echo " done"
+done

eo/contrib/irace/expe/alpha/run_irace_all.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+date -Iseconds
+echo "STARTS"
+
+for r in $(seq 15); do
+    echo "Run $r/15";
+    date -Iseconds
+    ./run_irace_parallel-batch.sh
+    date -Iseconds
+done
+
+echo "DONE"
+date -Iseconds

eo/contrib/irace/expe/alpha/run_irace_parallel-batch.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+outdir="$(date --iso-8601=ns)_results_irace"
+mkdir ${outdir}
+cd ${outdir}
+
+run(){
+    p="$1"
+
+    echo "Problem ${p}"
+    res="results_problem_${p}"
+    mkdir ${res}
+    cd ${res}
+
+    # Fore some reason, irace absolutely need those files...
+    cp ../../irace-config/example.scen .
+    cp ../../irace-config/default.instances .
+    cp ../../release/fastga .
+    cat ../../irace-config/target-runner | sed "s/{{PROBLEM}}/${p}/" > ./target-runner
+    chmod u+x target-runner
+
+    # Generate the parameter list file.
+    ./fastga -h > fastga.param 2>/dev/null
+    # /usr/lib/R/site-library/irace/bin/irace --scenario example.scen 2>&1 | tee irace.log
+    /usr/lib/R/site-library/irace/bin/irace --scenario example.scen &> irace.log
+
+    cd ..
+    echo "Done problem ${p}"
+}
+
+N=5 # Somehow 5 is the fastest on my 4-cores machine.
+(
+for pb in $(seq 0 18); do 
+    ((i=i%N)); ((i++==0)) && wait
+    run "$pb" &
+done
+wait
+)

eo/contrib/irace/expe/alpha/run_randoms.sh

@@ -0,0 +1,61 @@
+
+#!/bin/bash
+
+# Number of runs (=seeds).
+runs=50
+
+# You most probably want to run on release builds.
+exe="./release/fastga"
+
+outdir="$(date --iso-8601=minutes)_results_randoms"
+mkdir -p ${outdir}
+mkdir -p ${outdir}/raw
+mkdir -p ${outdir}/raw/data
+mkdir -p ${outdir}/raw/logs
+
+n=1
+algoid=0
+for algoid in $(seq 0 100); do
+    echo ""
+    date
+
+    a=( $((RANDOM%5)) $((RANDOM%7)) $((RANDOM%11)) $((RANDOM%5)) $((RANDOM%7)) $((RANDOM%11)) $((RANDOM%11)) )
+    algo="--crossover-rate=${a[0]} --cross-selector=${a[1]} --crossover=${a[2]}  --mutation-rate=${a[3]} --mut-selector=${a[4]} --mutation=${a[5]}  --replacement=${a[6]}"
+
+    for pb in $(seq 0 18) ; do
+        perc=$(echo "scale=3;${n}/(100*18)*100.0" | bc)
+        echo "${perc}% : algo ${algoid}/100, problem ${pb}/18"
+        # echo -n "Runs: "
+
+        for seed in $(seq ${runs}) ; do # Iterates over runs/seeds.
+            # This is the command to be ran.
+            cmd="${exe} --full-log=1 --problem=${pb} --seed=${seed} ${algo}"
+            # echo ${cmd} # Print the command.
+
+            # Forge a directory/log file name
+            # (remove double dashs and replace spaces with underscore).
+            name_run="pb=${pb}_seed=${seed}_$(echo "${algo}" | sed 's/--//g' | sed 's/ /_/g')"
+            # echo $name_run
+
+            # Progress print.
+            # echo -n "${seed} "
+
+            # Actually start the command.
+            ${cmd} > "${outdir}/raw/data/${name_run}.dat" 2> "${outdir}/raw/logs/${name_run}.log"
+
+            # Check for the most common problem in the log file.
+            cat "${outdir}/raw/logs/${name_run}.log" | grep "illogical performance"
+        done # seed
+        # echo ""
+
+        n=$(($n+1))
+    done # pb
+
+    algoid=$(($algoid+1))
+done
+
+# Move IOH logs in the results directory.
+mv ./FastGA_* ${outdir}
+
+echo "Done"
+date

eo/contrib/irace/expe/beta/best_out_of_elites.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+#cmd eg : 
+#python3 best_out_of_elites.py ./fastga_results_all/fastga_results_planO/planO_maxExp=50000_maxEv=5n_2021-08-13T19:16+02:00_results_elites_all
+#python3 best_out_of_elites.py ./fastga_results_all/fastga_results_random/maxEv=10000_nbAlgo=15_2021-08-21T20:53+02:00_results_randoms
+
+
+#get the configuration of the best out of the elite
+# recommendation suggested by 15 independant runs of irace
+
+figdir=sys.argv[1] # directory of a result of one experiment
+#eg : ./fastga_results_all/fastga_results_plan1/plan1_maxExp\=100000_maxEv\=5n_2021-08-13T19\:04+02\:00_results_elites_all/
+#print(figdir.split('/')[-2], figdir.split('/'))
+if("plan" in figdir.split('/')[-2]):
+    print("Operator,","op. ,",",".join(map(str,range(1,20))))
+
+    column={"pc" : 101, "SelectC": 7, "Crossover" : 10, "pm": 101,"SelectM" : 7, "Mutation": 11, "Replacement" : 11, "pop-size": 50, "offspring-size" : 50}
+    nbparam=(len(os.listdir(os.path.join(figdir,"raw/data"))[0].split("_"))-1) #-1 car il y a le pb
+
+    if( nbparam<len(column)):
+        del column["pop-size"]
+        del column["offspring-size"]
+    configs=[(-1,-1)]*19 #tuple(auc,config)
+    res=np.zeros((nbparam,19))
+    for fastgadir in os.listdir(os.path.join(figdir,"raw/data")): #fastgadir : directory of 50 runs of an elite configuration
+        #cum=np.cumsum([0.1]*10)
+        average=[]
+        for fname in os.listdir(os.path.join(figdir,"raw/data",fastgadir)):
+            with open(os.path.join(figdir,"raw/data",fastgadir,fname)) as fd:
+                auc = float(fd.readlines()[0]) * -1
+            average.append(auc)
+        pb=int(fastgadir.split("_")[0].split("=")[1])
+        new_auc=np.mean(average)
+        if(configs[pb][0]<new_auc):
+            configs[pb]=(new_auc,fastgadir)
+
+
+    for pb in range(19):
+        config=configs[pb][1].split("_")
+        configparam=[p.split("=")[1] for p in config[1:]]
+        res[:,pb]=configparam
+
+    ind=0 #index of param_name
+    for param_name in column.keys():
+        #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+        print(param_name+","+str(column[param_name])+",", ",".join(map(str,res[ind])))
+        ind+=1
+        #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
+
+if("maxEv" in figdir.split('/')[-2]):
+    print("Operator,","op. ,",",".join(map(str,range(1,20))))
+
+    column={"pc" : 101, "SelectC": 7, "Crossover" : 10, "pm": 101,"SelectM" : 7, "Mutation": 11, "Replacement" : 11, "pop-size": 50, "offspring-size" : 50}
+    nbparam=(len(os.listdir(figdir)[0].split("_")))
+    if( nbparam<len(column)):
+        del column["pop-size"]
+        del column["offspring-size"]
+    configs=[(-1,-1)]*19 #tuple(auc,config)
+    bests=np.zeros((nbparam,19))
+    
+    for algodir in os.listdir(figdir): #algodir : directory of one random algo
+        for fname in os.listdir(os.path.join(figdir,algodir,"data")): #fname : directory of 50 runs of fastga for one pb
+            average=[]
+            for res in os.listdir(os.path.join(figdir,algodir,"data",fname)):
+                with open(os.path.join(figdir,algodir,"data",fname,res)) as fd:
+                    auc = float(fd.readlines()[0]) * -1
+                average.append(auc)
+            pb=int(fname.split("_")[0].split("=")[1])
+            new_auc=np.mean(average)
+            if(configs[pb][0]<new_auc):
+                configs[pb]=(new_auc,algodir)
+
+
+    for pb in range(19):
+        config=configs[pb][1].split("_")
+        configparam=[p.split("=")[1] for p in config]
+        bests[:,pb]=configparam
+
+    ind=0 #index of param_name
+    for param_name in column.keys():
+        #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+        print(param_name+","+str(column[param_name])+",", ",".join(map(str,bests[ind])))
+        ind+=1

eo/contrib/irace/expe/beta/csv_all.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+ldata="./fastga_results_all/"  #fastga_results_all
+figpath="./hist_and_csv/"  #hist_and_csv
+
+#get distribution of operators variants of all problems of each plan of fastga_results_all
+#fastga_results_all contains all experiments of all plans 
+
+ldir=$(echo $(ls ${ldata})) #list of directory of each plan
+for plan in ${ldir[@]} ; do #get the directory of each plan
+    lexperiment=$(echo $(ls ${ldata}/${plan}))
+
+    for experiments in ${lexperiment[@]} ; do
+        path="${ldata}/${plan}/${experiments}"
+
+        #----------------average aucs of each algo for each pb only for plan A,F,O ---------------
+        #myfig=${figpath}/auc_average_${plan}
+        #mkdir -p ${myfig}
+        #cmd="python3 parse_auc_average.py ${path} "
+        #$cmd > "${myfig}/auc_average_${experiments}.csv"
+        #--------------distribution of operators by pb and for all pb only for plan A,F,O ------
+        #myfig=${figpath}/distribution_op_${plan}
+        #mkdir -p ${myfig}
+        #cmd="python3 distribution_op_all.py ${path} ${myfig} "
+        #$cmd 
+        #--------------best out csv--------
+        cmd="python3 best_out_of_elites.py ${path}"
+        myfig=${figpath}/best_out_${plan}
+        mkdir -p ${myfig}
+        $cmd > ${myfig}/best_out_all_pb_${experiments}.csv
+        echo ${cmd}
+
+    done
+done
+
+#---------------distribution of operators of randoma algo------------------
+#rpath=${ldata}/fastga_results_random
+#cmd="python3 dist_op_random.py ${rpath} ${figpath}"
+#$cmd
+#---------------random---------------

eo/contrib/irace/expe/beta/csv_all_bests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+ldata=$1
+file_py=$2
+csvdir="csv_FAO"
+ldir=$(echo $(ls ${ldata}))
+for data in ${ldir[@]} ; do 
+    path="${ldata}/${data}"
+    cmd="python3 ${file_py} ${path}"
+    plan_name=$(echo ${data} | sed "s/data//")
+    mexp=$(echo ${data[@]} | cut -d _ -f2)
+    mevals=$(echo ${data[@]} | cut -d _ -f3)
+    ddate=$(echo ${data[@]} | cut -d _ -f4)
+    name="results_irace_plan${plan_name[@]:0:1}_${mexp}_${mevals}_${ddate}"
+    mkdir -p "${csvdir}/csv_plan${plan_name[@]:0:1}"
+    ${cmd} > "${csvdir}/csv_plan${plan_name[@]:0:1}/${name}.csv"
+done

eo/contrib/irace/expe/beta/dist_op_random.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+#cmd eg : python3 ./dist_op_random.py ./fastga_results_all/fastga_results_random/ ./hist_and_csv/
+#get the distribution of operators variants recommended by 15 random algo for each maxEv 
+#pc and pm 10 possibilities :   [0-0.1[  [0.1-0.2[ [0.2-0.3[  [0.3-0.4[  [0-0.5[  [0.5-0.6[ ...[0.9-1[
+#pop-size and offspring-size 10 possibilities : 0-5 5-10, 10-15 15-20  20-25  25-30 30-35- 35-40 40-45 45-50
+
+path=sys.argv[1] # directory of a result of one experiment
+#eg : ./fastga_results_all/fastga_results_random/
+figdir=sys.argv[2] #directory of where you want to store the data
+if("random" in path):
+    #column : [operator : nbpossibilities]
+    distdir=figdir+"/distribution_random"
+    try:
+        os.makedirs(distdir)
+    except FileExistsError:
+        pass
+
+    nbparam=9 #-1 car il y a le pb
+
+    res=[]
+            
+    for maxEvdir in os.listdir(path):
+        res.append({"crossover-rate":["pc" , np.zeros(10, dtype=int)], 
+    "cross-selector":["SelectC", np.zeros(7, dtype=int)],
+    "crossover":["Crossover" , np.zeros(10, dtype=int)],
+    "mutation-rate":["pm",np.zeros(10, dtype=int)], 
+    "mut-selector":["SelectM",np.zeros(10, dtype=int)],
+    "mutation":["Mutation", np.zeros(11, dtype=int)],
+    "replacement":["Replacement" , np.zeros(11, dtype=int)],
+    "pop-size":["pop-size", np.zeros(10, dtype=int)], 
+    "offspring-size":["offspring-size" , np.zeros(10, dtype=int)]})
+        for algodir in os.listdir(os.path.join(path,maxEvdir)): #fastgadir : directory of 50 runs of an elite configuration
+            algo=algodir.split("_")
+            for param in algo:
+                name,val=param.split("=")[0],float(param.split("=")[1])
+                if(name in {"pop-size" ,"offspring-size"}):
+                    if(val%5==0):
+                        res[-1][name][1][int(val//5) -1]+=1
+                    else:
+                        #print(res[-1][name][1],val//5)
+                        res[-1][name][1][int(val//5)]+=1
+                        
+                elif(name in {"crossover-rate","mutation-rate"} ):
+                    if(int(val*10)==10): #case of val=1
+                        res[-1][name][1][-1]+=1
+                    else :
+                        #print(int(float(val)*10), name,pb,val)
+                        res[-1][name][1][int(val*10)]+=1
+                else :
+                    res[-1][name][1][int(val)]+=1
+
+
+    ind=0
+    for maxEvdir in os.listdir(path):
+        name="distribution_random_"+maxEvdir.split("_")[0]+".csv" #the end of the path must be /
+        with open(os.path.join(distdir,name),"w+") as csvfile:
+            csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+        with open(os.path.join(distdir,name),"a") as csvfile:
+            for param_name in res[ind].keys():
+                #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+                csvfile.write(res[ind][param_name][0]+","+ ",".join(map(str,res[ind][param_name][1]))+",-"*(11-len(res[ind][param_name][1])) +"\n")
+                #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
+        ind+=1
+    #all problems
+    name ="distribution_all_random_"+path.split("/")[-1]+".csv"
+    with open(os.path.join(distdir,name),'w+') as csvfile:
+        csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+
+    with open(os.path.join(distdir,name),'a') as csvfile:
+        for param_name in res[0].keys():
+            #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+            csvfile.write(res[0][param_name][0]+","+ ",".join(map(str,np.sum([res[i][param_name][1] for i in range(ind-1)],0)))+",-"*(11-len(res[0][param_name][1])) +"\n") #res[0] only for getting the name of parameters
+            #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1)))) 

eo/contrib/irace/expe/beta/distribution_op_all.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+#get the distribution of operators variants recommended by 15 runs of irace for all problems and each problem
+#also get an csv file
+#pc and pm 10 possibilities :   [0-0.1[  [0.1-0.2[ [0.2-0.3[  [0.3-0.4[  [0-0.5[  [0.5-0.6[ ...[0.9-1[
+#pop-size and offspring-size 10 possibilities : 0-5 5-10, 10-15 15-20  20-25  25-30 30-35- 35-40 40-45 45-50
+
+path=sys.argv[1] # directory of a result of one experiment
+#eg : ./fastga_results_all/fastga_results_planO/planO_maxExp\=100000_maxEv\=5n_2021-08-13T19\:04+02\:00_results_elites_all/
+
+if("fastga_results_plan" in path):
+        #column : [operator : nbpossibilities]
+    distdir=sys.argv[2]
+    try:
+        os.makedirs(distdir)
+    except FileExistsError:
+        pass
+
+    nbparam=(len(os.listdir(os.path.join(path,"raw/data"))[0].split("_"))-1)
+
+    if( nbparam==7):
+        res=[{"crossover-rate":["pc" , np.zeros(10, dtype=int)], 
+        "cross-selector":["SelectC", np.zeros(7, dtype=int)],
+        "crossover":["Crossover" , np.zeros(10, dtype=int)],
+        "mutation-rate":["pm",np.zeros(10, dtype=int)], 
+        "mut-selector":["SelectM",np.zeros(7, dtype=int)],
+        "mutation":["Mutation", np.zeros(11, dtype=int)],
+        "replacement":["Replacement" ,np.zeros(11, dtype=int)]} for i in range(19)]
+    else:
+        res=[{"crossover-rate":["pc" , np.zeros(10, dtype=int)], 
+        "cross-selector":["SelectC", np.zeros(7, dtype=int)],
+        "crossover":["Crossover" , np.zeros(10, dtype=int)],
+        "mutation-rate":["pm",np.zeros(10, dtype=int)], 
+        "mut-selector":["SelectM",np.zeros(7, dtype=int)],
+        "mutation":["Mutation", np.zeros(11, dtype=int)],
+        "replacement":["Replacement" , np.zeros(11, dtype=int)],
+        "pop-size":["pop-size", np.zeros(10, dtype=int)], 
+        "offspring-size":["offspring-size" , np.zeros(10, dtype=int)]} for i in range(19)]
+        
+
+    for fastgadir in os.listdir(os.path.join(path,"raw/data")): #fastgadir : directory of 50 runs of an elite configuration
+        algo=fastgadir.split("_")
+        pb=int(fastgadir.split("_")[0].split("=")[1])
+        for param in algo[1:]:
+            name,val=param.split("=")[0],float(param.split("=")[1])
+            if(name in {"pop-size" ,"offspring-size"}):
+                if(val%5==0):
+                    res[pb][name][1][int(val//5) -1]+=1
+                else:
+                    #print(res[pb][name][1],val//5)
+                    res[pb][name][1][int(val//5)]+=1
+                    
+            elif(name in {"crossover-rate","mutation-rate"} ):
+                if(int(val*10)==10): #case of val=1
+                    res[pb][name][1][-1]+=1
+                else :
+                    #print(int(float(val)*10), name,pb,val)
+                    res[pb][name][1][int(val*10)]+=1
+            else :
+                res[pb][name][1][int(val)]+=1
+
+
+
+    for pb in range(19):
+        name="distribution_pb="+str(pb)+"_"+path.split("/")[-2]+".csv" #the end of the path must be /
+        with open(os.path.join(distdir,name),"w+") as csvfile:
+            csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+        with open(os.path.join(distdir,name),"a") as csvfile:
+            for param_name in res[pb].keys():
+                #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+                csvfile.write(res[pb][param_name][0]+","+ ",".join(map(str,res[pb][param_name][1]))+",-"*(11-len(res[pb][param_name][1])) +"\n")
+                #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
+
+    #all problems
+    name ="distribution_all_pb_"+path.split("/")[-1]+".csv"
+    with open(os.path.join(path,"raw",name),'w+') as csvfile:
+        csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+
+    with open(os.path.join(path,"raw",name),'a') as csvfile:
+        for param_name in res[0].keys():
+            #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+            csvfile.write(res[0][param_name][0]+","+ ",".join(map(str,np.sum([res[i][param_name][1] for i in range(19)],0)))+",-"*(11-len(res[0][param_name][1])) +"\n") #res[0] only for getting the name of parameters
+            #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1)))) 

eo/contrib/irace/expe/beta/fastga_elites_all.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+ldata=$1 
+file_sh=$2
+ldir=$(echo $(ls ${ldata}))
+fastga_dir="fastga_results_all"
+mkdir -p /scratchbeta/${USER}/${fatga_dir}
+#mkdir -p "/home/${USER}/${fastga_dir}/fastga_results_plan1"
+mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planF"
+mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planA"
+mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planO"
+
+for data in ${ldir[@]} ; do 
+    path_csv="${ldata}/${data}"
+    plan_name=$(echo ${data} | sed "s/results_irace_plan//")
+    mexp=$(echo ${data[@]} | cut -d _ -f4)
+    mexp_id=$(echo ${mexp} | cut -d = -f2)
+    mevals=$(echo ${data[@]} | cut -d _ -f5)
+    mevals_id=$(echo ${mevals} | cut -d = -f2)
+    path="/scratchbeta/${USER}/${fastga_dir}/fastga_results_plan${plan_name[@]:0:1}"
+    cmd="bash ${file_sh} ${path_csv} ${mexp_id} ${mevals_id} ${path}"
+    name="fastga${plan_name[@]:0:1}_${mexp}_${mevals}_$(date -Iseconds)_results_elites_all"
+    ${cmd} &> "${path}/output${plan_name[@]:0:1}_fastga_${mexp}_${mevals}_$(date -Iseconds).txt"
+done

eo/contrib/irace/expe/beta/hist_all.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+ldata="./fastga_results_all"  #fastga_results_all
+figpath="./hist_and_csv"  #hist_and_csv
+
+ldir=$(echo $(ls ${ldata})) #list of directory of each plan
+for plan in ${ldir[@]} ; do #get the directory of each plan
+    #------------hist by budget of a Plan (O,R or F)
+    #path="${ldata}/${plan}"   
+    #cmd="python3 hist_join.py ${path} ${figpath}"
+    #echo $cmd
+    #$cmd
+
+    #---------------------------hist by pb by budget---------------
+    path="${ldata}/${plan}"   
+    cmd="python3 hist_by_pb_budget_plan.py ${path} ${figpath}"
+    echo $cmd
+    $cmd
+done
+
+#---------------random------------------
+#rpath=${ldata}/fastga_results_random
+#cmd="python3 hist_join_random.py ${rpath} ${figpath}"
+#---------------random---------------
+
+#--------------------Choose a Budget irace and a budget fastga
+mexp=100000
+mevals=1000
+#-------------------histogram join each plan F,A,R,O and join all algorithms for the budget chosen
+cmd="python3 hist_by_FARO.py ${ldata} ${figdir} ${mexp} ${mevals}"
+$cmd
+#-------------------histogram by pb join each plan F,A,R,O and join all algorithms for the budget chosen
+cmd="python3 hist_by_FARO_pb.py ${ldata} ${figdir} ${mexp} ${mevals}"
+$cmd