Add scripts for parsing and archive link

eo/contrib/irace/expe/beta/best_out_of_elites.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+#cmd eg : 
+#python3 best_out_of_elites.py ./fastga_results_all/fastga_results_planO/planO_maxExp=50000_maxEv=5n_2021-08-13T19:16+02:00_results_elites_all
+#python3 best_out_of_elites.py ./fastga_results_all/fastga_results_random/maxEv=10000_nbAlgo=15_2021-08-21T20:53+02:00_results_randoms
+
+
+#get the configuration of the best out of the elite
+# recommendation suggested by 15 independant runs of irace
+
+figdir=sys.argv[1] # directory of a result of one experiment
+#eg : ./fastga_results_all/fastga_results_plan1/plan1_maxExp\=100000_maxEv\=5n_2021-08-13T19\:04+02\:00_results_elites_all/
+#print(figdir.split('/')[-2], figdir.split('/'))
+if("plan" in figdir.split('/')[-2]):
+    print("Operator,","op. ,",",".join(map(str,range(1,20))))
+
+    column={"pc" : 101, "SelectC": 7, "Crossover" : 10, "pm": 101,"SelectM" : 7, "Mutation": 11, "Replacement" : 11, "pop-size": 50, "offspring-size" : 50}
+    nbparam=(len(os.listdir(os.path.join(figdir,"raw/data"))[0].split("_"))-1) #-1 car il y a le pb
+
+    if( nbparam<len(column)):
+        del column["pop-size"]
+        del column["offspring-size"]
+    configs=[(-1,-1)]*19 #tuple(auc,config)
+    res=np.zeros((nbparam,19))
+    for fastgadir in os.listdir(os.path.join(figdir,"raw/data")): #fastgadir : directory of 50 runs of an elite configuration
+        #cum=np.cumsum([0.1]*10)
+        average=[]
+        for fname in os.listdir(os.path.join(figdir,"raw/data",fastgadir)):
+            with open(os.path.join(figdir,"raw/data",fastgadir,fname)) as fd:
+                auc = float(fd.readlines()[0]) * -1
+            average.append(auc)
+        pb=int(fastgadir.split("_")[0].split("=")[1])
+        new_auc=np.mean(average)
+        if(configs[pb][0]<new_auc):
+            configs[pb]=(new_auc,fastgadir)
+
+
+    for pb in range(19):
+        config=configs[pb][1].split("_")
+        configparam=[p.split("=")[1] for p in config[1:]]
+        res[:,pb]=configparam
+
+    ind=0 #index of param_name
+    for param_name in column.keys():
+        #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+        print(param_name+","+str(column[param_name])+",", ",".join(map(str,res[ind])))
+        ind+=1
+        #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
+
+if("maxEv" in figdir.split('/')[-2]):
+    print("Operator,","op. ,",",".join(map(str,range(1,20))))
+
+    column={"pc" : 101, "SelectC": 7, "Crossover" : 10, "pm": 101,"SelectM" : 7, "Mutation": 11, "Replacement" : 11, "pop-size": 50, "offspring-size" : 50}
+    nbparam=(len(os.listdir(figdir)[0].split("_")))
+    if( nbparam<len(column)):
+        del column["pop-size"]
+        del column["offspring-size"]
+    configs=[(-1,-1)]*19 #tuple(auc,config)
+    bests=np.zeros((nbparam,19))
+    
+    for algodir in os.listdir(figdir): #algodir : directory of one random algo
+        for fname in os.listdir(os.path.join(figdir,algodir,"data")): #fname : directory of 50 runs of fastga for one pb
+            average=[]
+            for res in os.listdir(os.path.join(figdir,algodir,"data",fname)):
+                with open(os.path.join(figdir,algodir,"data",fname,res)) as fd:
+                    auc = float(fd.readlines()[0]) * -1
+                average.append(auc)
+            pb=int(fname.split("_")[0].split("=")[1])
+            new_auc=np.mean(average)
+            if(configs[pb][0]<new_auc):
+                configs[pb]=(new_auc,algodir)
+
+
+    for pb in range(19):
+        config=configs[pb][1].split("_")
+        configparam=[p.split("=")[1] for p in config]
+        bests[:,pb]=configparam
+
+    ind=0 #index of param_name
+    for param_name in column.keys():
+        #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+        print(param_name+","+str(column[param_name])+",", ",".join(map(str,bests[ind])))
+        ind+=1

eo/contrib/irace/expe/beta/csv_all.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+ldata="./fastga_results_all/"  #fastga_results_all
+figpath="./hist_and_csv/"  #hist_and_csv
+
+#get distribution of operators variants of all problems of each plan of fastga_results_all
+#fastga_results_all contains all experiments of all plans 
+
+ldir=$(echo $(ls ${ldata})) #list of directory of each plan
+for plan in ${ldir[@]} ; do #get the directory of each plan
+    lexperiment=$(echo $(ls ${ldata}/${plan}))
+
+    for experiments in ${lexperiment[@]} ; do
+        path="${ldata}/${plan}/${experiments}"
+
+        #----------------average aucs of each algo for each pb only for plan A,F,O ---------------
+        #myfig=${figpath}/auc_average_${plan}
+        #mkdir -p ${myfig}
+        #cmd="python3 parse_auc_average.py ${path} "
+        #$cmd > "${myfig}/auc_average_${experiments}.csv"
+        #--------------distribution of operators by pb and for all pb only for plan A,F,O ------
+        #myfig=${figpath}/distribution_op_${plan}
+        #mkdir -p ${myfig}
+        #cmd="python3 distribution_op_all.py ${path} ${myfig} "
+        #$cmd 
+        #--------------best out csv--------
+        cmd="python3 best_out_of_elites.py ${path}"
+        myfig=${figpath}/best_out_${plan}
+        mkdir -p ${myfig}
+        $cmd > ${myfig}/best_out_all_pb_${experiments}.csv
+        echo ${cmd}
+
+    done
+done
+
+#---------------distribution of operators of randoma algo------------------
+#rpath=${ldata}/fastga_results_random
+#cmd="python3 dist_op_random.py ${rpath} ${figpath}"
+#$cmd
+#---------------random---------------

eo/contrib/irace/expe/beta/csv_all_bests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 ldata=$1
 file_py=$2
-csvdir="csv_FA"
+csvdir="csv_FAO"
 ldir=$(echo $(ls ${ldata}))
 for data in ${ldir[@]} ; do 
     path="${ldata}/${data}"

eo/contrib/irace/expe/beta/dist_op_random.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+#cmd eg : python3 ./dist_op_random.py ./fastga_results_all/fastga_results_random/ ./hist_and_csv/
+#get the distribution of operators variants recommended by 15 random algo for each maxEv 
+#pc and pm 10 possibilities :   [0-0.1[  [0.1-0.2[ [0.2-0.3[  [0.3-0.4[  [0-0.5[  [0.5-0.6[ ...[0.9-1[
+#pop-size and offspring-size 10 possibilities : 0-5 5-10, 10-15 15-20  20-25  25-30 30-35- 35-40 40-45 45-50
+
+path=sys.argv[1] # directory of a result of one experiment
+#eg : ./fastga_results_all/fastga_results_random/
+figdir=sys.argv[2] #directory of where you want to store the data
+if("random" in path):
+    #column : [operator : nbpossibilities]
+    distdir=figdir+"/distribution_random"
+    try:
+        os.makedirs(distdir)
+    except FileExistsError:
+        pass
+
+    nbparam=9 #-1 car il y a le pb
+
+    res=[]
+            
+    for maxEvdir in os.listdir(path):
+        res.append({"crossover-rate":["pc" , np.zeros(10, dtype=int)], 
+    "cross-selector":["SelectC", np.zeros(7, dtype=int)],
+    "crossover":["Crossover" , np.zeros(10, dtype=int)],
+    "mutation-rate":["pm",np.zeros(10, dtype=int)], 
+    "mut-selector":["SelectM",np.zeros(10, dtype=int)],
+    "mutation":["Mutation", np.zeros(11, dtype=int)],
+    "replacement":["Replacement" , np.zeros(11, dtype=int)],
+    "pop-size":["pop-size", np.zeros(10, dtype=int)], 
+    "offspring-size":["offspring-size" , np.zeros(10, dtype=int)]})
+        for algodir in os.listdir(os.path.join(path,maxEvdir)): #fastgadir : directory of 50 runs of an elite configuration
+            algo=algodir.split("_")
+            for param in algo:
+                name,val=param.split("=")[0],float(param.split("=")[1])
+                if(name in {"pop-size" ,"offspring-size"}):
+                    if(val%5==0):
+                        res[-1][name][1][int(val//5) -1]+=1
+                    else:
+                        #print(res[-1][name][1],val//5)
+                        res[-1][name][1][int(val//5)]+=1
+                        
+                elif(name in {"crossover-rate","mutation-rate"} ):
+                    if(int(val*10)==10): #case of val=1
+                        res[-1][name][1][-1]+=1
+                    else :
+                        #print(int(float(val)*10), name,pb,val)
+                        res[-1][name][1][int(val*10)]+=1
+                else :
+                    res[-1][name][1][int(val)]+=1
+
+
+    ind=0
+    for maxEvdir in os.listdir(path):
+        name="distribution_random_"+maxEvdir.split("_")[0]+".csv" #the end of the path must be /
+        with open(os.path.join(distdir,name),"w+") as csvfile:
+            csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+        with open(os.path.join(distdir,name),"a") as csvfile:
+            for param_name in res[ind].keys():
+                #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+                csvfile.write(res[ind][param_name][0]+","+ ",".join(map(str,res[ind][param_name][1]))+",-"*(11-len(res[ind][param_name][1])) +"\n")
+                #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
+        ind+=1
+    #all problems
+    name ="distribution_all_random_"+path.split("/")[-1]+".csv"
+    with open(os.path.join(distdir,name),'w+') as csvfile:
+        csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+
+    with open(os.path.join(distdir,name),'a') as csvfile:
+        for param_name in res[0].keys():
+            #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+            csvfile.write(res[0][param_name][0]+","+ ",".join(map(str,np.sum([res[i][param_name][1] for i in range(ind-1)],0)))+",-"*(11-len(res[0][param_name][1])) +"\n") #res[0] only for getting the name of parameters
+            #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1)))) 

eo/contrib/irace/expe/beta/distribution_op_all.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+#get the distribution of operators variants recommended by 15 runs of irace for all problems and each problem
+#also get an csv file
+#pc and pm 10 possibilities :   [0-0.1[  [0.1-0.2[ [0.2-0.3[  [0.3-0.4[  [0-0.5[  [0.5-0.6[ ...[0.9-1[
+#pop-size and offspring-size 10 possibilities : 0-5 5-10, 10-15 15-20  20-25  25-30 30-35- 35-40 40-45 45-50
+
+path=sys.argv[1] # directory of a result of one experiment
+#eg : ./fastga_results_all/fastga_results_planO/planO_maxExp\=100000_maxEv\=5n_2021-08-13T19\:04+02\:00_results_elites_all/
+
+if("fastga_results_plan" in path):
+        #column : [operator : nbpossibilities]
+    distdir=sys.argv[2]
+    try:
+        os.makedirs(distdir)
+    except FileExistsError:
+        pass
+
+    nbparam=(len(os.listdir(os.path.join(path,"raw/data"))[0].split("_"))-1)
+
+    if( nbparam==7):
+        res=[{"crossover-rate":["pc" , np.zeros(10, dtype=int)], 
+        "cross-selector":["SelectC", np.zeros(7, dtype=int)],
+        "crossover":["Crossover" , np.zeros(10, dtype=int)],
+        "mutation-rate":["pm",np.zeros(10, dtype=int)], 
+        "mut-selector":["SelectM",np.zeros(7, dtype=int)],
+        "mutation":["Mutation", np.zeros(11, dtype=int)],
+        "replacement":["Replacement" ,np.zeros(11, dtype=int)]} for i in range(19)]
+    else:
+        res=[{"crossover-rate":["pc" , np.zeros(10, dtype=int)], 
+        "cross-selector":["SelectC", np.zeros(7, dtype=int)],
+        "crossover":["Crossover" , np.zeros(10, dtype=int)],
+        "mutation-rate":["pm",np.zeros(10, dtype=int)], 
+        "mut-selector":["SelectM",np.zeros(7, dtype=int)],
+        "mutation":["Mutation", np.zeros(11, dtype=int)],
+        "replacement":["Replacement" , np.zeros(11, dtype=int)],
+        "pop-size":["pop-size", np.zeros(10, dtype=int)], 
+        "offspring-size":["offspring-size" , np.zeros(10, dtype=int)]} for i in range(19)]
+        
+
+    for fastgadir in os.listdir(os.path.join(path,"raw/data")): #fastgadir : directory of 50 runs of an elite configuration
+        algo=fastgadir.split("_")
+        pb=int(fastgadir.split("_")[0].split("=")[1])
+        for param in algo[1:]:
+            name,val=param.split("=")[0],float(param.split("=")[1])
+            if(name in {"pop-size" ,"offspring-size"}):
+                if(val%5==0):
+                    res[pb][name][1][int(val//5) -1]+=1
+                else:
+                    #print(res[pb][name][1],val//5)
+                    res[pb][name][1][int(val//5)]+=1
+                    
+            elif(name in {"crossover-rate","mutation-rate"} ):
+                if(int(val*10)==10): #case of val=1
+                    res[pb][name][1][-1]+=1
+                else :
+                    #print(int(float(val)*10), name,pb,val)
+                    res[pb][name][1][int(val*10)]+=1
+            else :
+                res[pb][name][1][int(val)]+=1
+
+
+
+    for pb in range(19):
+        name="distribution_pb="+str(pb)+"_"+path.split("/")[-2]+".csv" #the end of the path must be /
+        with open(os.path.join(distdir,name),"w+") as csvfile:
+            csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+        with open(os.path.join(distdir,name),"a") as csvfile:
+            for param_name in res[pb].keys():
+                #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+                csvfile.write(res[pb][param_name][0]+","+ ",".join(map(str,res[pb][param_name][1]))+",-"*(11-len(res[pb][param_name][1])) +"\n")
+                #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
+
+    #all problems
+    name ="distribution_all_pb_"+path.split("/")[-1]+".csv"
+    with open(os.path.join(path,"raw",name),'w+') as csvfile:
+        csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
+
+    with open(os.path.join(path,"raw",name),'a') as csvfile:
+        for param_name in res[0].keys():
+            #print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
+            csvfile.write(res[0][param_name][0]+","+ ",".join(map(str,np.sum([res[i][param_name][1] for i in range(19)],0)))+",-"*(11-len(res[0][param_name][1])) +"\n") #res[0] only for getting the name of parameters
+            #print(str(i)+",",",".join(map(str,np.mean(aucs[i],1)))) 

eo/contrib/irace/expe/beta/fastga_elites_all.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
-ldata=$1 # eg : ./csv_planF/ don t forget to end the path with /
-file_sh=$2 #eg : ./run_elites_planF
+ldata=$1 
+file_sh=$2
 ldir=$(echo $(ls ${ldata}))
 fastga_dir="fastga_results_all"
 mkdir -p /scratchbeta/${USER}/${fatga_dir}
 #mkdir -p "/home/${USER}/${fastga_dir}/fastga_results_plan1"
 mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planF"
 mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planA"
+mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planO"
 
 for data in ${ldir[@]} ; do 
-    path_csv="${ldata}${data}"
+    path_csv="${ldata}/${data}"
     plan_name=$(echo ${data} | sed "s/results_irace_plan//")
     mexp=$(echo ${data[@]} | cut -d _ -f4)
     mexp_id=$(echo ${mexp} | cut -d = -f2)

eo/contrib/irace/expe/beta/hist_all.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+ldata="./fastga_results_all"  #fastga_results_all
+figpath="./hist_and_csv"  #hist_and_csv
+
+ldir=$(echo $(ls ${ldata})) #list of directory of each plan
+for plan in ${ldir[@]} ; do #get the directory of each plan
+    #------------hist by budget of a Plan (O,R or F)
+    #path="${ldata}/${plan}"   
+    #cmd="python3 hist_join.py ${path} ${figpath}"
+    #echo $cmd
+    #$cmd
+
+    #---------------------------hist by pb by budget---------------
+    path="${ldata}/${plan}"   
+    cmd="python3 hist_by_pb_budget_plan.py ${path} ${figpath}"
+    echo $cmd
+    $cmd
+done
+
+#---------------random------------------
+#rpath=${ldata}/fastga_results_random
+#cmd="python3 hist_join_random.py ${rpath} ${figpath}"
+#---------------random---------------
+
+#--------------------Choose a Budget irace and a budget fastga
+mexp=100000
+mevals=1000
+#-------------------histogram join each plan F,A,R,O and join all algorithms for the budget chosen
+cmd="python3 hist_by_FARO.py ${ldata} ${figdir} ${mexp} ${mevals}"
+$cmd
+#-------------------histogram by pb join each plan F,A,R,O and join all algorithms for the budget chosen
+cmd="python3 hist_by_FARO_pb.py ${ldata} ${figdir} ${mexp} ${mevals}"
+$cmd

eo/contrib/irace/expe/beta/hist_by_FARO.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import mannwhitneyu
+
+##python3 hist_by_FARO.py ./fastga_results_all/ ./hist_and_csv/ 100000 1000
+#one plot for one experiment plan for the same budget fastga, and the same budget irace if there is a budget irace (A,F)
+path=sys.argv[1]
+figpath=sys.argv[2]
+maxExp=sys.argv[3]
+maxEv=sys.argv[4]
+
+indF=-1
+indFO=-1
+averageConfigs=[]
+name=[]
+for fastga in os.listdir(path): #ddir : directory of fastga_plan
+    if(fastga in {"fastga_results_planA","fastga_results_planF","fastga_results_planO"}):
+        for plan in os.listdir(os.path.join(path,fastga)):
+            print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
+            if("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan):
+                average=[]
+                
+                for fastgadir in os.listdir(os.path.join(path,fastga,plan,"raw","data")): #fastgadir : directory of 50 runs of a configuration
+                    for fname in os.listdir(os.path.join(path,fastga,plan,"raw","data",fastgadir)):
+                        with open(os.path.join(path,fastga,plan,"raw","data",fastgadir,fname)) as fd:
+                            auc = float(fd.readlines()[0]) *(-1)
+                        average.append(auc)
+                averageConfigs.append(average)
+                nameid=plan.split("_")[0][-1]
+                name.append("plan"+nameid+"_"+"_".join(plan.split("_")[1:3]))
+    if("random" in fastga):
+        for randir in os.listdir(os.path.join(path,fastga)):
+            #eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
+            average=[]
+            if("maxEv="+str(maxEv)+"_" in randir):
+                for ddir in os.listdir(os.path.join(path,fastga,randir)): #ddir : directory of one run_elites_all or more
+                    if("crossover" in ddir):
+                        #name.append("_".join(ddir.split("_")[1:3]))
+                        for fastgadir in os.listdir(os.path.join(path,fastga,randir,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
+                            for fname in os.listdir(os.path.join(path,fastga,randir,ddir,"data",fastgadir)):
+                                with open(os.path.join(path,fastga,randir,ddir,"data",fastgadir,fname)) as fd:
+                                    auc = float(fd.readlines()[0]) *(-1)
+                                average.append(auc)
+                                #hist[belonging(auc,cum)]+=1
+                averageConfigs.append(average)
+                name.append(randir.split("_")[0]+"_random")
+
+
+figdir=os.path.join(figpath,"hist_FARO_by_budget")
+try:
+    os.makedirs(figdir)
+except FileExistsError:
+    pass
+
+#_,pv=mannwhitneyu(averageConfigs[indFO],averageConfigs[indF])
+#print(name,len(averageConfigs))
+plt.figure()
+plt.hist(averageConfigs,bins=10,range=(0,1),align="mid",rwidth=0.9,label=name) #no label
+plt.xlabel("performances")
+plt.ylabel("Number of runs")
+plt.xlim(0,1)
+plt.ylim(0,8000)
+plt.yticks(range(0,8000,500))
+#plt.title("pvalue="+str(pv)+"\n medianeF="+str(np.median(averageConfigs[indF]))+", medianeFO="+str(np.median(averageConfigs[indFO])))
+plt.legend()
+plt.savefig(figdir+"/hist_planFARO"+"_maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+".png")
+plt.close()
+    

eo/contrib/irace/expe/beta/hist_by_FARO_pb.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+##python3 hist_by_FARO_pb.py ./fastga_results_all/ ./hist_and_csv/ 100000 1000
+#19 histograms by plan F,A ,R O
+path=sys.argv[1]
+figpath=sys.argv[2]
+maxExp=sys.argv[3]
+maxEv=sys.argv[4]
+
+hist_pb=[[] for i in range(19)]
+name=[]
+for fastga in os.listdir(path): #ddir : directory of fastga_plan
+    if(fastga in {"fastga_results_planA", "fastga_results_planF","fastga_results_planO"}):
+        for plan in os.listdir(os.path.join(path,fastga)):
+            #print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
+            #print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
+            if("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan):
+                nameid=fastga[-1]
+                name.append("plan"+nameid+"_".join(plan.split("_")[1:3]))
+                for fastgadir in os.listdir(os.path.join(path,fastga,plan,"raw","data")): #fastgadir : directory of 50 runs of a configuration
+                    pb=int(fastgadir.split("_")[0].split("=")[1])
+                    average_pb=[]
+                    for fname in os.listdir(os.path.join(path,fastga,plan,"raw","data",fastgadir)):
+                        with open(os.path.join(path,fastga,plan,"raw","data",fastgadir,fname)) as fd:
+                            auc = float(fd.readlines()[0]) *(-1)
+                        average_pb.append(auc)
+                    if(hist_pb[pb]==[]): #first algo
+                        hist_pb[pb].append(average_pb)
+                    elif(len(hist_pb[pb])!=len(name)):
+                        hist_pb[pb].append(average_pb)
+                    else:
+                        hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
+
+    if("random" in fastga):
+        for randir in os.listdir(os.path.join(path,fastga)):
+            #eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
+            if(("maxEv="+str(maxEv)+"_") in randir):
+                #print("maxEv="+str(maxEv) in randir,randir)
+                name.append(randir.split("_")[0]+"_random")
+                for ddir in os.listdir(os.path.join(path,fastga,randir)): #ddir : directory of one run_elites_all or more
+                    if("crossover" in ddir):
+                        #name.append("_".join(ddir.split("_")[1:3]))
+                        for fastgadir in os.listdir(os.path.join(path,fastga,randir,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
+                            average_pb=[]
+                            pb=int(fastgadir.split("_")[0].split("=")[1])
+                            for fname in os.listdir(os.path.join(path,fastga,randir,ddir,"data",fastgadir)):
+                                with open(os.path.join(path,fastga,randir,ddir,"data",fastgadir,fname)) as fd:
+                                    auc = float(fd.readlines()[0]) *(-1)
+                                average_pb.append(auc)
+                            #print(len(hist_pb[pb]),len(name), pb)
+                            if(hist_pb[pb]==[]): #first algo
+                                #print("entrer random vide")
+                                hist_pb[pb].append(average_pb)
+                            elif(len(hist_pb[pb])!=len(name)):
+                                #print("entrer random !=")
+                                hist_pb[pb].append(average_pb)
+                            else:
+                                hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
+
+
+figdir=os.path.join(figpath,"hist_by_FARO_pb_maxExp={}_maxEv={}".format(maxExp,maxEv))
+try:
+    os.makedirs(figdir)
+except FileExistsError:
+    pass
+#colors=['yellow', 'green',"blue","pink","purple","orange","magenta","gray","darkred","cyan","brown","olivedrab","thistle","stateblue"]
+print(name)
+for pb in range(19):
+    print(pb, len(hist_pb[pb]))
+    for i in hist_pb[pb]:
+        print(len(i))
+    plt.figure()
+    plt.hist(hist_pb[pb],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name) #no label color=colors[:len(name)]
+    #for aucs in range(len(hist_pb[pb])):
+        #plt.hist(hist_pb[pb][aucs],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name[aucs]) #no label
+    plt.xlabel("performances")
+    plt.ylabel("Number of runs")
+    plt.ylim(0,800)
+    plt.xlim(0,1)
+    plt.yticks(range(0,800,50))
+    #plt.xticks(np.cumsum([0.1]*10))
+    plt.legend()
+    plt.savefig(figdir+"/hist_FARO_pb={}_maxExp={}_maxEv={}.png".format(pb,maxExp,maxEv))
+    plt.close()

eo/contrib/irace/expe/beta/hist_by_pb_budget_plan.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+# python3 hist_by_pb_budget_plan.py ./fastga_results_all/fastga_results_planF/ ./hist_and_csv/
+#python3 hist_by_pb_budget_plan.py ./fastga_results_all/fastga_results_planO ./hist_and_csv
+#get 19 histograms with number of budget bars, same as hist_join but now is by pb
+
+#argv : list of elite results
+path=sys.argv[1]
+figpath=sys.argv[2]
+#plan_name=sys.argv[3]
+hist_pb=[[] for i in range(19)]
+name=[]
+if("random" in path):
+    plan_name="R"
+else:
+    plan_name=path.strip("/").split("/")[-1][-1]
+
+
+for plandir in os.listdir(path): #plandir: directory of an experiment of elite results 
+    if("results_elites_all" in plandir):
+        #eg : plan2_maxExp=10000_maxEv=1000_2021-08-20T1347+0200_results_elites_all
+        budget_irace=plandir.split("_")[1].split("=")[1]
+        budget_fastga=plandir.split("_")[2].split("=")[1]
+        name.append("plan="+plan_name+"_"+"".join(plandir.split("_")[1:3])) #plan=*_maxExp=*_maxEv=*
+        
+        for algodir in os.listdir(os.path.join(path,plandir,"raw","data")):
+            average_pb=[]
+            pb=int(algodir.split("_")[0].split("=")[1])
+            for algo in os.listdir(os.path.join(path,plandir,"raw","data",algodir)):
+                with open(os.path.join(path,plandir,"raw","data",algodir,algo)) as fd:
+                    auc = float(fd.readlines()[0]) *(-1)
+                average_pb.append(auc)
+            if(hist_pb[pb]==[]): #first algo
+                hist_pb[pb].append(average_pb)
+            elif(len(hist_pb[pb])!=len(name)):
+                hist_pb[pb].append(average_pb)
+            else:
+                hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
+
+    if("results_randoms" in plandir):
+        #eg : maxEv=1000_2021-08-20T1347+0200_results_random
+        budget_fastga=plandir.split("_")[0].split("=")[1]
+        name.append("plan="+plan_name+"_"+"".join(plandir.split("_")[0])) #plan=*_maxExp=*_maxEv=*
+        for algodir in os.listdir(os.path.join(path,plandir)):
+            
+            for algo in os.listdir(os.path.join(path,plandir,algodir,"data")):
+                pb=int(algo.split("_")[0].split("=")[1])
+                average_pb=[]
+                for fname in os.listdir(os.path.join(path,plandir,algodir,"data",algo)):
+                    with open(os.path.join(path,plandir,algodir,"data",algo,fname)) as fd:
+                        auc = float(fd.readlines()[0]) *(-1)
+                    average_pb.append(auc)
+                if(hist_pb[pb]==[]): #first algo
+                    print("entrer")
+                    hist_pb[pb].append(average_pb)
+                elif(len(hist_pb[pb])!=len(name)):
+                    hist_pb[pb].append(average_pb)
+                else:
+                    hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
+        
+
+
+print(path.split("/")[-1][-1])
+
+figdir=os.path.join(figpath,"hist_by_{}_pb_budget_plan".format(plan_name))
+#figdir=os.path.join(figpath,"hist_by_{}_pb_irace_maxEv={}".format(plan_name,1000))
+try:
+    os.makedirs(figdir)
+except FileExistsError:
+    pass
+
+
+for pb in range(19):
+    print(pb, len(hist_pb[pb]))
+    plt.figure()
+    plt.hist(hist_pb[pb],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name) #no label color=colors[:len(name)]
+    #for aucs in range(len(hist_pb[pb])):
+        #plt.hist(hist_pb[pb][aucs],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name[aucs]) #no label
+    plt.xlabel("performances")
+    plt.ylabel("Number of runs")
+    plt.ylim(0,750)
+    plt.yticks(range(0,750,50))
+    plt.xlim(0,1)
+    plt.legend()
+    plt.savefig(figdir+"/hist_plan={}_pb={}_budget.png".format(plan_name,pb))
+    plt.close()

eo/contrib/irace/expe/beta/hist_join.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import mannwhitneyu
+
+#cmd : python3 hist_join.py ./fastga_results_all/fastga_results_planO/ ./hist_and_csv/
+#histogram by plan for the budgets (irace and fastag)
+
+
+path=sys.argv[1] #argv : directory of a Plan (O, A, F)
+figpath=sys.argv[2] #path to store the histograms
+averageConfigs=[]
+name=[]
+if("fastga_results_plan" in path):
+    for ddir in os.listdir(path): #ddir : directory of one run_elites_all or more
+        if("plan" in ddir):
+            average=[]
+            name.append("_".join(ddir.split("_")[1:3]))
+            for fastgadir in os.listdir(os.path.join(path,ddir,"raw","data")): #fastgadir : directory of 50 runs of a configuration
+                for fname in os.listdir(os.path.join(path,ddir,"raw","data",fastgadir)):
+                    with open(os.path.join(path,ddir,"raw","data",fastgadir,fname)) as fd:
+                        auc = float(fd.readlines()[0]) *(-1)
+                    average.append(auc)
+                    #hist[belonging(auc,cum)]+=1
+            averageConfigs.append(average)
+                #print(hist)
+                #print(average)
+
+    figdir=os.path.join(figpath,"hist_join")
+    try:
+        os.makedirs(figdir)
+    except FileExistsError:
+        pass
+
+
+    print(name,len(averageConfigs))
+
+    """
+    idd0=name[0].split("_")[0].split("=")[1][:-3]+"k"
+    idd1=name[1].split("_")[0].split("=")[1][:-3]+"k"
+    idd2=name[2].split("_")[0].split("=")[1][:-3]+"k"
+
+    #only for Budget irace 10000, 50000, 100000 ie: only three experiment results
+    titlename="median"+idd0+"={:.3f}".format(np.median(averageConfigs[0]))+" , median"+idd1+"={:.3f}".format(np.median(averageConfigs[1]))+" , median"+idd2+"={:.3f}".format(np.median(averageConfigs[2]))
+    _,pv=mannwhitneyu(averageConfigs[0],averageConfigs[1])
+    titlename+="\n pvalue{}={:.3f}".format(idd0+idd1,pv)
+    _,pv=mannwhitneyu(averageConfigs[0],averageConfigs[2])
+    titlename+=" ,pvalue{}={:.3f}".format(idd0+idd2,pv)
+    _,pv=mannwhitneyu(averageConfigs[1],averageConfigs[2])
+    titlename+=" ,pvalue{}={:.3f}".format(idd1+idd2,pv)
+    print(titlename)
+    """
+    plt.figure()
+    plt.hist(averageConfigs,bins=10,range=(0,1),align="mid",rwidth=0.9,label=name) #no label
+    plt.xlabel("performances")
+    plt.ylabel("Number of runs")
+    plt.xlim(0,1)
+    plt.ylim(0,7000)
+    plt.yticks(range(0,7000,500))
+    #plt.title(titlename)
+    plt.legend()
+    plt.savefig(figdir+"/hist_plan"+path.strip("/")[-1]+"_by_budget.png")
+    #plt.savefig(figpath+"/hist_plan"+path.strip("/")[-1]+"_by_budgetI.png")
+    plt.close()
+    
+

eo/contrib/irace/expe/beta/hist_join_random.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+##python3 hist_random.py ./dastga_results_all/fastga_results_random ./hist_and_csv/
+#argv : list of elite results
+data=sys.argv[1]
+figpath=sys.argv[2]
+averageConfigs=[]
+name=[]
+for path in os.listdir(data):
+    #eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
+    average=[]
+    if("maxEv" in path):
+        for ddir in os.listdir(os.path.join(data,path)): #ddir : directory of one run_elites_all or more
+            if("crossover" in ddir):
+                #name.append("_".join(ddir.split("_")[1:3]))
+                for fastgadir in os.listdir(os.path.join(data,path,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
+                    for fname in os.listdir(os.path.join(data,path,ddir,"data",fastgadir)):
+                        with open(os.path.join(data,path,ddir,"data",fastgadir,fname)) as fd:
+                            auc = float(fd.readlines()[0]) *(-1)
+                        average.append(auc)
+                        #hist[belonging(auc,cum)]+=1
+        averageConfigs.append(average)
+        name.append(path.split("_")[0])
+
+figdir=os.path.join(figpath,"hist_join")
+try:
+    os.makedirs(figdir)
+except FileExistsError:
+    pass
+
+colors=['yellow', 'green',"blue","pink","purple","orange","magenta","gray","darkred","cyan","brown","olivedrab","thistle","stateblue"]
+plt.figure()
+plt.hist(averageConfigs,bins=10,range=(0,1),align="mid",rwidth=0.5,label=name) #no label
+plt.xlabel("performances")
+plt.ylabel("Number of runs")
+plt.ylim([0,8000])
+plt.xlim(0,1)
+plt.yticks(range(0,8000,500))
+#plt.xticks(np.cumsum([0.1]*10))
+plt.legend()
+plt.savefig(figdir+"/hist_random_by_budget.png")
+plt.close()

eo/contrib/irace/expe/beta/irace_files_pA/forbidden.txt

@@ -0,0 +1,13 @@
+## Template for specifying forbidden parameter configurations in irace.
+##
+## This filename must be specified via the --forbidden-file command-line option
+## (or forbiddenFile in scenario.txt).
+##
+## The format is one constraint per line. Each constraint is a logical
+## expression (in R syntax). If a parameter configuration
+## is generated that makes the logical expression evaluate to TRUE,
+## then the configuration is discarded.
+##
+## Examples of valid logical operators are: ==  !=  >=  <=  >  <  &  |  !  %in%
+(replacement %in% c(2,3,4,5,6,7,8,9,10)) & (offspringsize > popsize)
+(replacement %in% c(1)) & (offspringsize < popsize)

eo/contrib/irace/expe/beta/irace_files_pF/forbidden.txt

@@ -0,0 +1,15 @@
+## Template for specifying forbidden parameter configurations in irace.
+##
+## This filename must be specified via the --forbidden-file command-line option
+## (or forbiddenFile in scenario.txt).
+##
+## The format is one constraint per line. Each constraint is a logical
+## expression (in R syntax). If a parameter configuration
+## is generated that makes the logical expression evaluate to TRUE,
+## then the configuration is discarded.
+##
+## Examples of valid logical operators are: ==  !=  >=  <=  >  <  &  |  !  %in%
+(replacement %in% c(2,3,4,5,6,7,8,9,10)) & (offspringsize > popsize)
+(replacement %in% c(1)) & (offspringsize < popsize)
+#(as.numeric(replacement) == 2) & (offspringsize > popsize)
+#(as.numeric(replacement) == 3) & (offspringsize > popsize)

eo/contrib/irace/expe/beta/irace_files_pO/default.instances

@@ -0,0 +1,48 @@
+## This is an example of specifying instances with a file.
+
+# Each line is an instance relative to trainInstancesDir
+# (see scenario.txt.tmpl) and an optional sequence of instance-specific
+# parameters that will be passed to target-runnerx when invoked on that
+# instance.
+
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40

eo/contrib/irace/expe/beta/irace_files_pO/example.scen

@@ -0,0 +1,228 @@
+###################################################### -*- mode: r -*- #####
+## Scenario setup for Iterated Race (irace).
+############################################################################
+
+## To use the default value of a parameter of iRace, simply do not set
+## the parameter (comment it out in this file, and do not give any
+## value on the command line).
+
+## File that contains the description of the parameters of the target
+## algorithm.
+parameterFile = "./fastga.param"
+
+## Directory where the programs will be run.
+execDir = "."
+
+## File to save tuning results as an R dataset, either absolute path or
+## relative to execDir.
+# logFile = "./irace.Rdata"
+
+## Previously saved log file to recover the execution of irace, either
+## absolute path or relative to the current directory.  If empty or NULL,
+## recovery is not performed.
+# recoveryFile = ""
+
+## Directory where training instances are located; either absolute path or
+## relative to current directory. If no trainInstancesFiles is provided,
+## all the files in trainInstancesDir will be listed as instances.
+trainInstancesDir = "."
+
+## File that contains a list of training instances and optionally
+## additional parameters for them. If trainInstancesDir is provided, irace
+## will search for the files in this folder.
+trainInstancesFile = "./default.instances"
+
+## File that contains a table of initial configurations. If empty or NULL,
+## all initial configurations are randomly generated.
+# configurationsFile = ""
+
+## File that contains a list of logical expressions that cannot be TRUE
+## for any evaluated configuration. If empty or NULL, do not use forbidden
+## expressions.
+# forbiddenFile = ""
+
+## Script called for each configuration that executes the target algorithm
+## to be tuned. See templates.
+targetRunner = "./target-runner"
+
+## Number of times to retry a call to targetRunner if the call failed.
+# targetRunnerRetries = 0
+
+## Optional data passed to targetRunner. This is ignored by the default
+## targetRunner function, but it may be used by custom targetRunner
+## functions to pass persistent data around.
+# targetRunnerData = ""
+
+## Optional R function to provide custom parallelization of targetRunner.
+# targetRunnerParallel = ""
+
+## Optional script or R function that provides a numeric value for each
+## configuration. See templates/target-evaluator.tmpl
+# targetEvaluator = ""
+
+## Maximum number of runs (invocations of targetRunner) that will be
+## performed. It determines the maximum budget of experiments for the
+## tuning.
+maxExperiments = 0 #100000
+
+
+## Maximum total execution time in seconds for the executions of
+## targetRunner. targetRunner must return two values: cost and time.
+# maxTime = 60
+
+## Fraction (smaller than 1) of the budget used to estimate the mean
+## computation time of a configuration. Only used when maxTime > 0
+# budgetEstimation = 0.02
+
+## Maximum number of decimal places that are significant for numerical
+## (real) parameters.
+digits = 2
+
+## Debug level of the output of irace. Set this to 0 to silence all debug
+## messages. Higher values provide more verbose debug messages.
+# debugLevel = 0
+
+## Number of iterations.
+# nbIterations = 0
+
+## Number of runs of the target algorithm per iteration.
+# nbExperimentsPerIteration = 0
+
+## Randomly sample the training instances or use them in the order given.
+# sampleInstances = 1
+
+## Statistical test used for elimination. Default test is always F-test
+## unless capping is enabled, in which case the default test is t-test.
+## Valid values are: F-test (Friedman test), t-test (pairwise t-tests with
+## no correction), t-test-bonferroni (t-test with Bonferroni's correction
+## for multiple comparisons), t-test-holm (t-test with Holm's correction
+## for multiple comparisons).
+# testType = "F-test"
+
+## Number of instances evaluated before the first elimination test. It
+## must be a multiple of eachTest.
+# firstTest = 5
+
+## Number of instances evaluated between elimination tests.
+# eachTest = 1
+
+## Minimum number of configurations needed to continue the execution of
+## each race (iteration).
+# minNbSurvival = 0
+
+## Number of configurations to be sampled and evaluated at each iteration.
+# nbConfigurations = 0
+
+## Parameter used to define the number of configurations sampled and
+## evaluated at each iteration.
+# mu = 5
+
+## Confidence level for the elimination test.
+# confidence = 0.95
+
+## If the target algorithm is deterministic, configurations will be
+## evaluated only once per instance.
+# deterministic = 0
+
+## Seed of the random number generator (by default, generate a random
+## seed).
+# seed = NA
+
+## Number of calls to targetRunner to execute in parallel. Values 0 or 1
+## mean no parallelization.
+# parallel = 0
+
+## Enable/disable load-balancing when executing experiments in parallel.
+## Load-balancing makes better use of computing resources, but increases
+## communication overhead. If this overhead is large, disabling
+## load-balancing may be faster.
+# loadBalancing = 1
+
+## Enable/disable MPI. Use Rmpi to execute targetRunner in parallel
+## (parameter parallel is the number of slaves).
+# mpi = 0
+
+## Specify how irace waits for jobs to finish when targetRunner submits
+## jobs to a batch cluster: sge, pbs, torque or slurm. targetRunner must
+## submit jobs to the cluster using, for example, qsub.
+# batchmode = 0
+
+## Enable/disable the soft restart strategy that avoids premature
+## convergence of the probabilistic model.
+# softRestart = 1
+
+## Soft restart threshold value for numerical parameters. If NA, NULL or
+## "", it is computed as 10^-digits.
+# softRestartThreshold = ""
+
+## Directory where testing instances are located, either absolute or
+## relative to current directory.
+# testInstancesDir = ""
+
+## File containing a list of test instances and optionally additional
+## parameters for them.
+# testInstancesFile = ""
+
+## Number of elite configurations returned by irace that will be tested if
+## test instances are provided.
+# testNbElites = 1
+
+## Enable/disable testing the elite configurations found at each
+## iteration.
+# testIterationElites = 0
+
+## Enable/disable elitist irace.
+# elitist = 1
+
+## Number of instances added to the execution list before previous
+## instances in elitist irace.
+# elitistNewInstances = 1
+
+## In elitist irace, maximum number per race of elimination tests that do
+## not eliminate a configuration. Use 0 for no limit.
+# elitistLimit = 2
+
+## User-defined R function that takes a configuration generated by irace
+## and repairs it.
+# repairConfiguration = ""
+
+## Enable the use of adaptive capping, a technique designed for minimizing
+## the computation time of configurations. This is only available when
+## elitist is active.
+# capping = 0
+
+## Measure used to obtain the execution bound from the performance of the
+## elite configurations: median, mean, worst, best.
+# cappingType = "median"
+
+## Method to calculate the mean performance of elite configurations:
+## candidate or instance.
+# boundType = "candidate"
+
+## Maximum execution bound for targetRunner. It must be specified when
+## capping is enabled.
+# boundMax = 0
+
+## Precision used for calculating the execution time. It must be specified
+## when capping is enabled.
+# boundDigits = 0
+
+## Penalization constant for timed out executions (executions that reach
+## boundMax execution time).
+# boundPar = 1
+
+## Replace the configuration cost of bounded executions with boundMax.
+# boundAsTimeout = 1
+
+## Percentage of the configuration budget used to perform a postselection
+## race of the best configurations of each iteration after the execution
+## of irace.
+# postselection = 0
+
+## Enable/disable AClib mode. This option enables compatibility with
+## GenericWrapper4AC as targetRunner script.
+# aclib = 0
+
+## END of scenario file
+############################################################################
+

eo/contrib/irace/expe/beta/irace_files_pO/fastga.param

@@ -0,0 +1,10 @@
+# name	switch	type	range
+# continuator	"--continuator="	c	(0)
+crossoverrate	"--crossover-rate="	r	(0,1)
+crossselector	"--cross-selector="	c	(0,1,2,3,4,5,6)
+# aftercrossselector	"--aftercross-selector="	c	(0)
+crossover	"--crossover="	        c	(0,1,2,3,4,5,6,7,8,9)
+mutationrate	"--mutation-rate="	r	(0,1)
+mutselector	"--mut-selector="	c	(0,1,2,3,4,5,6)
+mutation	"--mutation="	        c	(0,1,2,3,4,5,6,7,8,9,10)
+replacement     "--replacement="        c	(0,1,2,3,4,5,6,7,8,9,10)

eo/contrib/irace/expe/beta/irace_files_pO/target-runner

@@ -0,0 +1,88 @@
+#!/bin/bash
+###############################################################################
+# This script is the command that is executed every run.
+# Check the examples in examples/
+#
+# This script is run in the execution directory (execDir, --exec-dir).
+#
+# PARAMETERS:
+# $1 is the candidate configuration number
+# $2 is the instance ID
+# $3 is the seed
+# $4 is the instance name
+# The rest ($* after `shift 4') are parameters to the run
+#
+# RETURN VALUE:
+# This script should print one numerical value: the cost that must be minimized.
+# Exit with 0 if no error, with 1 in case of error
+###############################################################################
+error() {
+    echo "`TZ=UTC date`: $0: error: $@"
+    exit 1
+}
+
+
+EXE="./fastga"
+LOG_DIR="irace_logs"
+
+FIXED_PARAMS="--problem=0"
+MAX_EVALS=100
+#
+CONFIG_ID=$1
+INSTANCE_ID=$2 
+SEED=$3
+INSTANCE=$(echo $4 | sed 's/\//\n/g'|tail -n 1)
+CROSSOVER_RATE=$5
+CROSSOVER_SELECTOR=$6
+CROSSOVER=$7
+MUTATION_RATE=$8
+MUT_SELECTOR=$9
+MUTATION=${10}
+REPLACEMENT=${11}
+shift 11 || error "Not enough parameters"
+
+INSTANCE_PARAMS=$*
+
+# STDOUT=${LOG_DIR}/c${CONFIG_ID}_i${INSTANCE_ID}_s${SEED}.stdout
+# STDERR=${LOG_DIR}/c${CONFIG_ID}_i${INSTANCE_ID}_s${SEED}.stderr
+STDOUT="/dev/null"
+STDERR="/dev/null"
+
+if [ ! -x "${EXE}" ]; then
+    error "${EXE}: not found or not executable (pwd: $(pwd))"
+fi
+
+# If the program just prints a number, we can use 'exec' to avoid
+# creating another process, but there can be no other commands after exec.
+#exec $EXE ${FIXED_PARAMS} -i $INSTANCE ${INSTANCE_PARAMS} 
+# exit 1
+# 
+# Otherwise, save the output to a file, and parse the result from it.
+# (If you wish to ignore segmentation faults you can use '{}' around
+# the command.)
+cmd="$EXE ${FIXED_PARAMS} --instance=${INSTANCE} --seed=${SEED} ${CROSSOVER_RATE} ${CROSSOVER_SELECTOR} ${CROSSOVER} ${MUTATION_RATE} ${MUT_SELECTOR} ${MUTATION} ${REPLACEMENT}"
+# NOTE: irace seems to capture both stderr and stdout, so you should not output to stderr
+echo ${cmd} > ${STDERR}
+$cmd 2> ${STDERR} | tee ${STDOUT}
+
+# The following code is useless if the binary only output a single number on stdout.
+
+# This may be used to introduce a delay if there are filesystem
+# issues.
+# SLEEPTIME=1
+# while [ ! -s "${STDOUT}" ]; do
+#     sleep $SLEEPTIME
+#     let "SLEEPTIME += 1"
+# done
+
+# This is an example of reading a number from the output.
+# It assumes that the objective value is the first number in
+# the first column of the last line of the output.
+# if [ -s "${STDOUT}" ]; then
+#     COST=$(tail -n 1 ${STDOUT} | grep -e '^[[:space:]]*[+-]\?[0-9]' | cut -f1)
+#     echo "$COST"
+#     rm -f "${STDOUT}" "${STDERR}"
+#     exit 0
+# else
+#     error "${STDOUT}: No such file or directory"
+# fi

eo/contrib/irace/expe/beta/mwtestU.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+import sys
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import mannwhitneyu
+
+##cmd eg : 
+# python3 hist_by_2_4_5.py ./fastga_results_all/ ./hist_and_csv/ 100000 1000
+
+#get the Mann Whitney test U results between the plan F and plan R
+#  (change ligne 23 and 44 for other plan, and the maxExp, maxEv for other budget)
+
+path=sys.argv[1] 
+figpath=sys.argv[2] #directory to store the data
+maxExp=sys.argv[3]
+maxEv=sys.argv[4]
+
+hist_pb=[[] for i in range(19)]
+name=[]
+randind=-1
+for fastga in os.listdir(path): #ddir : directory of fastga_plan
+    if(fastga in {"fastga_results_planF"}):
+        for plan in os.listdir(os.path.join(path,fastga)):
+            print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
+            #print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
+            if("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan):
+                name.append("_".join(plan.split("_")[:3]))
+                for fastgadir in os.listdir(os.path.join(path,fastga,plan,"raw","data")): #fastgadir : directory of 50 runs of a configuration
+                    pb=int(fastgadir.split("_")[0].split("=")[1])
+                    average_pb=[]
+                    for fname in os.listdir(os.path.join(path,fastga,plan,"raw","data",fastgadir)):
+                        with open(os.path.join(path,fastga,plan,"raw","data",fastgadir,fname)) as fd:
+                            auc = float(fd.readlines()[0]) 
+                        average_pb.append(auc)
+                    if(hist_pb[pb]==[]): #first algo
+                        hist_pb[pb].append(average_pb)
+                    elif(len(hist_pb[pb])!=len(name)):
+                        hist_pb[pb].append(average_pb)
+                    else:
+                        hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
+
+
+    if("random" in fastga):
+        for randir in os.listdir(os.path.join(path,fastga)):
+            #eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
+            if(("maxEv="+str(maxEv)+"_") in randir):
+                print("maxEv="+str(maxEv) in randir,randir)
+                name.append(randir.split("_")[0]+"_random")
+                randind=len(name)-1
+                print(randind,name)
+                for ddir in os.listdir(os.path.join(path,fastga,randir)): #ddir : directory of one run_elites_all or more
+                    if("crossover" in ddir):
+                        for fastgadir in os.listdir(os.path.join(path,fastga,randir,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
+                            average_pb=[]
+                            pb=int(fastgadir.split("_")[0].split("=")[1])
+                            for fname in os.listdir(os.path.join(path,fastga,randir,ddir,"data",fastgadir)):
+                                with open(os.path.join(path,fastga,randir,ddir,"data",fastgadir,fname)) as fd:
+                                    auc = float(fd.readlines()[0]) 
+                                average_pb.append(auc)
+                            #print(len(hist_pb[pb]),len(name), pb)
+                            if(hist_pb[pb]==[]): #first algo
+                                #print("entrer random vide")
+                                hist_pb[pb].append(average_pb)
+                            elif(len(hist_pb[pb])!=len(name)):
+                                #print("entrer random !=")
+                                hist_pb[pb].append(average_pb)
+                            else:
+                                hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
+
+
+figdir=os.path.join(figpath,"mwtestU_FR")
+try:
+    os.makedirs(figdir)
+except FileExistsError:
+    pass
+#colors=['yellow', 'green',"blue","pink","purple","orange","magenta","gray","darkred","cyan","brown","olivedrab","thistle","stateblue"]
+print(name)
+
+filename="mwtestU_maxExp={}_maxEv={}_FR.csv".format(maxExp,maxEv)
+with open(os.path.join(figdir,filename),'w+') as csvfile:
+    csvfile.write(" ,"+",".join(map(str,range(0,19)))+"\n")
+meanvalue=[]
+pvalue=[]
+meanR=[]
+meanF=[]
+mdianR=[]
+mdianF=[]
+mdianvalue=[]
+iqrR=[]
+iqrF=[]
+stdR=[]
+stdF=[]
+iqrvalue=[]
+pstd=[]
+
+for pb in range(19):
+    #hR,lR,_=plt.hist(hist_pb[pb][randind],bins=10,range=(-1,0),align="mid",label=name) #no label color=colors[:len(name)]
+    #hF,lF,_=plt.hist(hist_pb[pb][np.abs(1-randind)],bins=10,range=(-1,0),align="mid",label=name) #no label color=colors[:len(name)]
+    _,pv=mannwhitneyu(hist_pb[pb][np.abs(1-randind)],hist_pb[pb][randind])
+    print(_,pv)
+    #meanvalue.append(np.mean(np.array(hF)*np.array(lF[:len(lF)-1]))-np.mean(np.array(hR)*np.array(lR[:len(lR)-1])))
+    pstd.append(np.std(hist_pb[pb][np.abs(1-randind)])-np.std(hist_pb[pb][randind]))
+    stdF.append(np.std(hist_pb[pb][np.abs(1-randind)]))
+    stdR.append(np.std(hist_pb[pb][randind]))
+    meanF.append(np.mean(hist_pb[pb][np.abs(1-randind)]))
+    meanR.append(np.mean(hist_pb[pb][randind]))
+    mdianF.append(np.median(hist_pb[pb][np.abs(1-randind)]))
+    mdianR.append(np.median(hist_pb[pb][randind]))
+    mdianvalue.append(np.median(hist_pb[pb][np.abs(1-randind)])-np.median(hist_pb[pb][randind]))
+    meanvalue.append(np.mean(hist_pb[pb][np.abs(1-randind)])-np.mean(hist_pb[pb][randind]))
+    pvalue.append(pv)
+    Q1 = np.percentile(hist_pb[pb][np.abs(1-randind)], 25, interpolation = 'midpoint')
+    # Third quartile (Q3)
+    Q3 = np.percentile(hist_pb[pb][np.abs(1-randind)], 75, interpolation = 'midpoint')
+    # Interquaritle range (IQR)
+    iqrF.append( Q3 - Q1)
+    Q1 = np.percentile(hist_pb[pb][randind], 25, interpolation = 'midpoint')
+    # Third quartile (Q3)
+    Q3 = np.percentile(hist_pb[pb][randind], 75, interpolation = 'midpoint')
+    # Interquaritle range (IQR)
+    iqrR.append( Q3 - Q1)
+    print(_,pv)
+iqrvalue=np.array(iqrF)-np.array(iqrR)
+with open(os.path.join(figdir,filename),'a') as csvfile:
+    csvfile.write("mF-mR,"+",".join(map(str,meanvalue))+"\n")
+    csvfile.write("p_value,"+",".join(map(str,pvalue))+"\n")
+    csvfile.write("mF,"+",".join(map(str,meanF))+"\n")
+    csvfile.write("mR,"+",".join(map(str,meanR))+"\n")
+    csvfile.write("medianF-medianR,"+",".join(map(str,mdianvalue))+"\n")
+    csvfile.write("medianF,"+",".join(map(str,mdianF))+"\n")
+    csvfile.write("medianR,"+",".join(map(str,mdianR))+"\n")
+    csvfile.write("stdF-stdR,"+",".join(map(str,mdianvalue))+"\n")
+    csvfile.write("stdF,"+",".join(map(str,stdF))+"\n")
+    csvfile.write("stdR,"+",".join(map(str,stdR))+"\n")
+    csvfile.write("iqrF,"+",".join(map(str,iqrF))+"\n")
+    csvfile.write("iqrR,"+",".join(map(str,iqrR))+"\n")
+    csvfile.write("iqrF-iqrR,"+",".join(map(str,iqrvalue))+"\n")
+
+    

eo/contrib/irace/expe/beta/parseO_irace_bests.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+#parse data1
+import os
+import re
+import sys
+#print("pb,ecdf,id,crossover-rate,cross-selector,crossover,mutation-rate,mut-selector,mutation,replacement") #plan1
+print("pb,ecdf,id,crossover-rate,cross-selector,crossover,mutation-rate,mut-selector,mutation,replacement")
+
+
+#give the path of one experiment 
+argv=sys.argv[1]
+for datadir in os.listdir(argv):
+    #if(os.path.isdir(os.path.join(argv,datadir))): check if argv/datadir is a directory
+    if(datadir.find("results_irace")>=0): #check if the directory is one JOB
+        for pb_dir in os.listdir(os.path.join(argv,datadir)):
+            if "results_problem" in pb_dir:
+                pb_id=pb_dir.replace("results_problem_","")
+                with open(os.path.join("./",argv,datadir,pb_dir,"irace.log")) as fd:
+                    data = fd.readlines()
+
+                    # Find the last best configuration
+                    bests = [line.strip() for line in data if "Best-so-far" in line]
+                    #print(datadir,bests)
+                    best = bests[-1].split()
+                    best_id, best_perf = best[2], best[5]
+                    # print(best_id,best_perf)
+
+                    # Filter the config detail
+                    configs = [line.strip() for line in data if "--crossover-rate=" in line and best_id in line]
+                    # print(configs)
+
+                    # Format as CSV
+                    algo = re.sub("\-\-\S*=", ",", configs[0])
+                    csv_line = pb_id + "," + best_perf + "," + algo
+                    print(csv_line.replace(" ",""))