Merge pull request #2 from AlixZh/feat+num_foundry
[fastga] Add scripts for parsing and archive link
This commit is contained in:
Johann Dreo
GitHub
commit
9e672182a8
30 changed files with 1570 additions and 18 deletions
86
eo/contrib/irace/expe/beta/best_out_of_elites.py
Executable file
86
eo/contrib/irace/expe/beta/best_out_of_elites.py
Executable file
|
|
@ -0,0 +1,86 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
#cmd eg :
|
||||||
|
#python3 best_out_of_elites.py ./fastga_results_all/fastga_results_planO/planO_maxExp=50000_maxEv=5n_2021-08-13T19:16+02:00_results_elites_all
|
||||||
|
#python3 best_out_of_elites.py ./fastga_results_all/fastga_results_random/maxEv=10000_nbAlgo=15_2021-08-21T20:53+02:00_results_randoms
|
||||||
|
|
||||||
|
|
||||||
|
#get the configuration of the best out of the elite
|
||||||
|
# recommendation suggested by 15 independant runs of irace
|
||||||
|
|
||||||
|
figdir=sys.argv[1] # directory of a result of one experiment
|
||||||
|
#eg : ./fastga_results_all/fastga_results_plan1/plan1_maxExp\=100000_maxEv\=5n_2021-08-13T19\:04+02\:00_results_elites_all/
|
||||||
|
#print(figdir.split('/')[-2], figdir.split('/'))
|
||||||
|
if("plan" in figdir.split('/')[-2]):
|
||||||
|
print("Operator,","op. ,",",".join(map(str,range(1,20))))
|
||||||
|
|
||||||
|
column={"pc" : 101, "SelectC": 7, "Crossover" : 10, "pm": 101,"SelectM" : 7, "Mutation": 11, "Replacement" : 11, "pop-size": 50, "offspring-size" : 50}
|
||||||
|
nbparam=(len(os.listdir(os.path.join(figdir,"raw/data"))[0].split("_"))-1) #-1 car il y a le pb
|
||||||
|
|
||||||
|
if( nbparam<len(column)):
|
||||||
|
del column["pop-size"]
|
||||||
|
del column["offspring-size"]
|
||||||
|
configs=[(-1,-1)]*19 #tuple(auc,config)
|
||||||
|
res=np.zeros((nbparam,19))
|
||||||
|
for fastgadir in os.listdir(os.path.join(figdir,"raw/data")): #fastgadir : directory of 50 runs of an elite configuration
|
||||||
|
#cum=np.cumsum([0.1]*10)
|
||||||
|
average=[]
|
||||||
|
for fname in os.listdir(os.path.join(figdir,"raw/data",fastgadir)):
|
||||||
|
with open(os.path.join(figdir,"raw/data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) * -1
|
||||||
|
average.append(auc)
|
||||||
|
pb=int(fastgadir.split("_")[0].split("=")[1])
|
||||||
|
new_auc=np.mean(average)
|
||||||
|
if(configs[pb][0]<new_auc):
|
||||||
|
configs[pb]=(new_auc,fastgadir)
|
||||||
|
|
||||||
|
|
||||||
|
for pb in range(19):
|
||||||
|
config=configs[pb][1].split("_")
|
||||||
|
configparam=[p.split("=")[1] for p in config[1:]]
|
||||||
|
res[:,pb]=configparam
|
||||||
|
|
||||||
|
ind=0 #index of param_name
|
||||||
|
for param_name in column.keys():
|
||||||
|
#print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
|
||||||
|
print(param_name+","+str(column[param_name])+",", ",".join(map(str,res[ind])))
|
||||||
|
ind+=1
|
||||||
|
#print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
|
||||||
|
|
||||||
|
if("maxEv" in figdir.split('/')[-2]):
|
||||||
|
print("Operator,","op. ,",",".join(map(str,range(1,20))))
|
||||||
|
|
||||||
|
column={"pc" : 101, "SelectC": 7, "Crossover" : 10, "pm": 101,"SelectM" : 7, "Mutation": 11, "Replacement" : 11, "pop-size": 50, "offspring-size" : 50}
|
||||||
|
nbparam=(len(os.listdir(figdir)[0].split("_")))
|
||||||
|
if( nbparam<len(column)):
|
||||||
|
del column["pop-size"]
|
||||||
|
del column["offspring-size"]
|
||||||
|
configs=[(-1,-1)]*19 #tuple(auc,config)
|
||||||
|
bests=np.zeros((nbparam,19))
|
||||||
|
|
||||||
|
for algodir in os.listdir(figdir): #algodir : directory of one random algo
|
||||||
|
for fname in os.listdir(os.path.join(figdir,algodir,"data")): #fname : directory of 50 runs of fastga for one pb
|
||||||
|
average=[]
|
||||||
|
for res in os.listdir(os.path.join(figdir,algodir,"data",fname)):
|
||||||
|
with open(os.path.join(figdir,algodir,"data",fname,res)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) * -1
|
||||||
|
average.append(auc)
|
||||||
|
pb=int(fname.split("_")[0].split("=")[1])
|
||||||
|
new_auc=np.mean(average)
|
||||||
|
if(configs[pb][0]<new_auc):
|
||||||
|
configs[pb]=(new_auc,algodir)
|
||||||
|
|
||||||
|
|
||||||
|
for pb in range(19):
|
||||||
|
config=configs[pb][1].split("_")
|
||||||
|
configparam=[p.split("=")[1] for p in config]
|
||||||
|
bests[:,pb]=configparam
|
||||||
|
|
||||||
|
ind=0 #index of param_name
|
||||||
|
for param_name in column.keys():
|
||||||
|
#print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
|
||||||
|
print(param_name+","+str(column[param_name])+",", ",".join(map(str,bests[ind])))
|
||||||
|
ind+=1
|
||||||
40
eo/contrib/irace/expe/beta/csv_all.sh
Executable file
40
eo/contrib/irace/expe/beta/csv_all.sh
Executable file
|
|
@ -0,0 +1,40 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
ldata="./fastga_results_all/" #fastga_results_all
|
||||||
|
figpath="./hist_and_csv/" #hist_and_csv
|
||||||
|
|
||||||
|
#get distribution of operators variants of all problems of each plan of fastga_results_all
|
||||||
|
#fastga_results_all contains all experiments of all plans
|
||||||
|
|
||||||
|
ldir=$(echo $(ls ${ldata})) #list of directory of each plan
|
||||||
|
for plan in ${ldir[@]} ; do #get the directory of each plan
|
||||||
|
lexperiment=$(echo $(ls ${ldata}/${plan}))
|
||||||
|
|
||||||
|
for experiments in ${lexperiment[@]} ; do
|
||||||
|
path="${ldata}/${plan}/${experiments}"
|
||||||
|
|
||||||
|
#----------------average aucs of each algo for each pb only for plan A,F,O ---------------
|
||||||
|
#myfig=${figpath}/auc_average_${plan}
|
||||||
|
#mkdir -p ${myfig}
|
||||||
|
#cmd="python3 parse_auc_average.py ${path} "
|
||||||
|
#$cmd > "${myfig}/auc_average_${experiments}.csv"
|
||||||
|
#--------------distribution of operators by pb and for all pb only for plan A,F,O ------
|
||||||
|
#myfig=${figpath}/distribution_op_${plan}
|
||||||
|
#mkdir -p ${myfig}
|
||||||
|
#cmd="python3 distribution_op_all.py ${path} ${myfig} "
|
||||||
|
#$cmd
|
||||||
|
#--------------best out csv--------
|
||||||
|
cmd="python3 best_out_of_elites.py ${path}"
|
||||||
|
myfig=${figpath}/best_out_${plan}
|
||||||
|
mkdir -p ${myfig}
|
||||||
|
$cmd > ${myfig}/best_out_all_pb_${experiments}.csv
|
||||||
|
echo ${cmd}
|
||||||
|
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
#---------------distribution of operators of randoma algo------------------
|
||||||
|
#rpath=${ldata}/fastga_results_random
|
||||||
|
#cmd="python3 dist_op_random.py ${rpath} ${figpath}"
|
||||||
|
#$cmd
|
||||||
|
#---------------random---------------
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
ldata=$1
|
ldata=$1
|
||||||
file_py=$2
|
file_py=$2
|
||||||
csvdir="csv_FA"
|
csvdir="csv_FAO"
|
||||||
ldir=$(echo $(ls ${ldata}))
|
ldir=$(echo $(ls ${ldata}))
|
||||||
for data in ${ldir[@]} ; do
|
for data in ${ldir[@]} ; do
|
||||||
path="${ldata}/${data}"
|
path="${ldata}/${data}"
|
||||||
|
|
|
||||||
78
eo/contrib/irace/expe/beta/dist_op_random.py
Executable file
78
eo/contrib/irace/expe/beta/dist_op_random.py
Executable file
|
|
@ -0,0 +1,78 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
#cmd eg : python3 ./dist_op_random.py ./fastga_results_all/fastga_results_random/ ./hist_and_csv/
|
||||||
|
#get the distribution of operators variants recommended by 15 random algo for each maxEv
|
||||||
|
#pc and pm 10 possibilities : [0-0.1[ [0.1-0.2[ [0.2-0.3[ [0.3-0.4[ [0-0.5[ [0.5-0.6[ ...[0.9-1[
|
||||||
|
#pop-size and offspring-size 10 possibilities : 0-5 5-10, 10-15 15-20 20-25 25-30 30-35- 35-40 40-45 45-50
|
||||||
|
|
||||||
|
path=sys.argv[1] # directory of a result of one experiment
|
||||||
|
#eg : ./fastga_results_all/fastga_results_random/
|
||||||
|
figdir=sys.argv[2] #directory of where you want to store the data
|
||||||
|
if("random" in path):
|
||||||
|
#column : [operator : nbpossibilities]
|
||||||
|
distdir=figdir+"/distribution_random"
|
||||||
|
try:
|
||||||
|
os.makedirs(distdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
nbparam=9 #-1 car il y a le pb
|
||||||
|
|
||||||
|
res=[]
|
||||||
|
|
||||||
|
for maxEvdir in os.listdir(path):
|
||||||
|
res.append({"crossover-rate":["pc" , np.zeros(10, dtype=int)],
|
||||||
|
"cross-selector":["SelectC", np.zeros(7, dtype=int)],
|
||||||
|
"crossover":["Crossover" , np.zeros(10, dtype=int)],
|
||||||
|
"mutation-rate":["pm",np.zeros(10, dtype=int)],
|
||||||
|
"mut-selector":["SelectM",np.zeros(10, dtype=int)],
|
||||||
|
"mutation":["Mutation", np.zeros(11, dtype=int)],
|
||||||
|
"replacement":["Replacement" , np.zeros(11, dtype=int)],
|
||||||
|
"pop-size":["pop-size", np.zeros(10, dtype=int)],
|
||||||
|
"offspring-size":["offspring-size" , np.zeros(10, dtype=int)]})
|
||||||
|
for algodir in os.listdir(os.path.join(path,maxEvdir)): #fastgadir : directory of 50 runs of an elite configuration
|
||||||
|
algo=algodir.split("_")
|
||||||
|
for param in algo:
|
||||||
|
name,val=param.split("=")[0],float(param.split("=")[1])
|
||||||
|
if(name in {"pop-size" ,"offspring-size"}):
|
||||||
|
if(val%5==0):
|
||||||
|
res[-1][name][1][int(val//5) -1]+=1
|
||||||
|
else:
|
||||||
|
#print(res[-1][name][1],val//5)
|
||||||
|
res[-1][name][1][int(val//5)]+=1
|
||||||
|
|
||||||
|
elif(name in {"crossover-rate","mutation-rate"} ):
|
||||||
|
if(int(val*10)==10): #case of val=1
|
||||||
|
res[-1][name][1][-1]+=1
|
||||||
|
else :
|
||||||
|
#print(int(float(val)*10), name,pb,val)
|
||||||
|
res[-1][name][1][int(val*10)]+=1
|
||||||
|
else :
|
||||||
|
res[-1][name][1][int(val)]+=1
|
||||||
|
|
||||||
|
|
||||||
|
ind=0
|
||||||
|
for maxEvdir in os.listdir(path):
|
||||||
|
name="distribution_random_"+maxEvdir.split("_")[0]+".csv" #the end of the path must be /
|
||||||
|
with open(os.path.join(distdir,name),"w+") as csvfile:
|
||||||
|
csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
|
||||||
|
with open(os.path.join(distdir,name),"a") as csvfile:
|
||||||
|
for param_name in res[ind].keys():
|
||||||
|
#print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
|
||||||
|
csvfile.write(res[ind][param_name][0]+","+ ",".join(map(str,res[ind][param_name][1]))+",-"*(11-len(res[ind][param_name][1])) +"\n")
|
||||||
|
#print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
|
||||||
|
ind+=1
|
||||||
|
#all problems
|
||||||
|
name ="distribution_all_random_"+path.split("/")[-1]+".csv"
|
||||||
|
with open(os.path.join(distdir,name),'w+') as csvfile:
|
||||||
|
csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
|
||||||
|
|
||||||
|
with open(os.path.join(distdir,name),'a') as csvfile:
|
||||||
|
for param_name in res[0].keys():
|
||||||
|
#print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
|
||||||
|
csvfile.write(res[0][param_name][0]+","+ ",".join(map(str,np.sum([res[i][param_name][1] for i in range(ind-1)],0)))+",-"*(11-len(res[0][param_name][1])) +"\n") #res[0] only for getting the name of parameters
|
||||||
|
#print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
|
||||||
87
eo/contrib/irace/expe/beta/distribution_op_all.py
Executable file
87
eo/contrib/irace/expe/beta/distribution_op_all.py
Executable file
|
|
@ -0,0 +1,87 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
#get the distribution of operators variants recommended by 15 runs of irace for all problems and each problem
|
||||||
|
#also get an csv file
|
||||||
|
#pc and pm 10 possibilities : [0-0.1[ [0.1-0.2[ [0.2-0.3[ [0.3-0.4[ [0-0.5[ [0.5-0.6[ ...[0.9-1[
|
||||||
|
#pop-size and offspring-size 10 possibilities : 0-5 5-10, 10-15 15-20 20-25 25-30 30-35- 35-40 40-45 45-50
|
||||||
|
|
||||||
|
path=sys.argv[1] # directory of a result of one experiment
|
||||||
|
#eg : ./fastga_results_all/fastga_results_planO/planO_maxExp\=100000_maxEv\=5n_2021-08-13T19\:04+02\:00_results_elites_all/
|
||||||
|
|
||||||
|
if("fastga_results_plan" in path):
|
||||||
|
#column : [operator : nbpossibilities]
|
||||||
|
distdir=sys.argv[2]
|
||||||
|
try:
|
||||||
|
os.makedirs(distdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
nbparam=(len(os.listdir(os.path.join(path,"raw/data"))[0].split("_"))-1)
|
||||||
|
|
||||||
|
if( nbparam==7):
|
||||||
|
res=[{"crossover-rate":["pc" , np.zeros(10, dtype=int)],
|
||||||
|
"cross-selector":["SelectC", np.zeros(7, dtype=int)],
|
||||||
|
"crossover":["Crossover" , np.zeros(10, dtype=int)],
|
||||||
|
"mutation-rate":["pm",np.zeros(10, dtype=int)],
|
||||||
|
"mut-selector":["SelectM",np.zeros(7, dtype=int)],
|
||||||
|
"mutation":["Mutation", np.zeros(11, dtype=int)],
|
||||||
|
"replacement":["Replacement" ,np.zeros(11, dtype=int)]} for i in range(19)]
|
||||||
|
else:
|
||||||
|
res=[{"crossover-rate":["pc" , np.zeros(10, dtype=int)],
|
||||||
|
"cross-selector":["SelectC", np.zeros(7, dtype=int)],
|
||||||
|
"crossover":["Crossover" , np.zeros(10, dtype=int)],
|
||||||
|
"mutation-rate":["pm",np.zeros(10, dtype=int)],
|
||||||
|
"mut-selector":["SelectM",np.zeros(7, dtype=int)],
|
||||||
|
"mutation":["Mutation", np.zeros(11, dtype=int)],
|
||||||
|
"replacement":["Replacement" , np.zeros(11, dtype=int)],
|
||||||
|
"pop-size":["pop-size", np.zeros(10, dtype=int)],
|
||||||
|
"offspring-size":["offspring-size" , np.zeros(10, dtype=int)]} for i in range(19)]
|
||||||
|
|
||||||
|
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,"raw/data")): #fastgadir : directory of 50 runs of an elite configuration
|
||||||
|
algo=fastgadir.split("_")
|
||||||
|
pb=int(fastgadir.split("_")[0].split("=")[1])
|
||||||
|
for param in algo[1:]:
|
||||||
|
name,val=param.split("=")[0],float(param.split("=")[1])
|
||||||
|
if(name in {"pop-size" ,"offspring-size"}):
|
||||||
|
if(val%5==0):
|
||||||
|
res[pb][name][1][int(val//5) -1]+=1
|
||||||
|
else:
|
||||||
|
#print(res[pb][name][1],val//5)
|
||||||
|
res[pb][name][1][int(val//5)]+=1
|
||||||
|
|
||||||
|
elif(name in {"crossover-rate","mutation-rate"} ):
|
||||||
|
if(int(val*10)==10): #case of val=1
|
||||||
|
res[pb][name][1][-1]+=1
|
||||||
|
else :
|
||||||
|
#print(int(float(val)*10), name,pb,val)
|
||||||
|
res[pb][name][1][int(val*10)]+=1
|
||||||
|
else :
|
||||||
|
res[pb][name][1][int(val)]+=1
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
for pb in range(19):
|
||||||
|
name="distribution_pb="+str(pb)+"_"+path.split("/")[-2]+".csv" #the end of the path must be /
|
||||||
|
with open(os.path.join(distdir,name),"w+") as csvfile:
|
||||||
|
csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
|
||||||
|
with open(os.path.join(distdir,name),"a") as csvfile:
|
||||||
|
for param_name in res[pb].keys():
|
||||||
|
#print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
|
||||||
|
csvfile.write(res[pb][param_name][0]+","+ ",".join(map(str,res[pb][param_name][1]))+",-"*(11-len(res[pb][param_name][1])) +"\n")
|
||||||
|
#print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
|
||||||
|
|
||||||
|
#all problems
|
||||||
|
name ="distribution_all_pb_"+path.split("/")[-1]+".csv"
|
||||||
|
with open(os.path.join(path,"raw",name),'w+') as csvfile:
|
||||||
|
csvfile.write("Op index, "+",".join(map(str,range(0,11)))+"\n")
|
||||||
|
|
||||||
|
with open(os.path.join(path,"raw",name),'a') as csvfile:
|
||||||
|
for param_name in res[0].keys():
|
||||||
|
#print(map(str,res[ind]),res[ind], ",".join(map(str,res[ind])))
|
||||||
|
csvfile.write(res[0][param_name][0]+","+ ",".join(map(str,np.sum([res[i][param_name][1] for i in range(19)],0)))+",-"*(11-len(res[0][param_name][1])) +"\n") #res[0] only for getting the name of parameters
|
||||||
|
#print(str(i)+",",",".join(map(str,np.mean(aucs[i],1))))
|
||||||
|
|
@ -1,15 +1,16 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
ldata=$1 # eg : ./csv_planF/ don t forget to end the path with /
|
ldata=$1
|
||||||
file_sh=$2 #eg : ./run_elites_planF
|
file_sh=$2
|
||||||
ldir=$(echo $(ls ${ldata}))
|
ldir=$(echo $(ls ${ldata}))
|
||||||
fastga_dir="fastga_results_all"
|
fastga_dir="fastga_results_all"
|
||||||
mkdir -p /scratchbeta/${USER}/${fatga_dir}
|
mkdir -p /scratchbeta/${USER}/${fatga_dir}
|
||||||
#mkdir -p "/home/${USER}/${fastga_dir}/fastga_results_plan1"
|
#mkdir -p "/home/${USER}/${fastga_dir}/fastga_results_plan1"
|
||||||
mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planF"
|
mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planF"
|
||||||
mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planA"
|
mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planA"
|
||||||
|
mkdir -p "/scratchbeta/${USER}/${fastga_dir}/fastga_results_planO"
|
||||||
|
|
||||||
for data in ${ldir[@]} ; do
|
for data in ${ldir[@]} ; do
|
||||||
path_csv="${ldata}${data}"
|
path_csv="${ldata}/${data}"
|
||||||
plan_name=$(echo ${data} | sed "s/results_irace_plan//")
|
plan_name=$(echo ${data} | sed "s/results_irace_plan//")
|
||||||
mexp=$(echo ${data[@]} | cut -d _ -f4)
|
mexp=$(echo ${data[@]} | cut -d _ -f4)
|
||||||
mexp_id=$(echo ${mexp} | cut -d = -f2)
|
mexp_id=$(echo ${mexp} | cut -d = -f2)
|
||||||
|
|
|
||||||
34
eo/contrib/irace/expe/beta/hist_all.sh
Executable file
34
eo/contrib/irace/expe/beta/hist_all.sh
Executable file
|
|
@ -0,0 +1,34 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
ldata="./fastga_results_all" #fastga_results_all
|
||||||
|
figpath="./hist_and_csv" #hist_and_csv
|
||||||
|
|
||||||
|
ldir=$(echo $(ls ${ldata})) #list of directory of each plan
|
||||||
|
for plan in ${ldir[@]} ; do #get the directory of each plan
|
||||||
|
#------------hist by budget of a Plan (O,R or F)
|
||||||
|
#path="${ldata}/${plan}"
|
||||||
|
#cmd="python3 hist_join.py ${path} ${figpath}"
|
||||||
|
#echo $cmd
|
||||||
|
#$cmd
|
||||||
|
|
||||||
|
#---------------------------hist by pb by budget---------------
|
||||||
|
path="${ldata}/${plan}"
|
||||||
|
cmd="python3 hist_by_pb_budget_plan.py ${path} ${figpath}"
|
||||||
|
echo $cmd
|
||||||
|
$cmd
|
||||||
|
done
|
||||||
|
|
||||||
|
#---------------random------------------
|
||||||
|
#rpath=${ldata}/fastga_results_random
|
||||||
|
#cmd="python3 hist_join_random.py ${rpath} ${figpath}"
|
||||||
|
#---------------random---------------
|
||||||
|
|
||||||
|
#--------------------Choose a Budget irace and a budget fastga
|
||||||
|
mexp=100000
|
||||||
|
mevals=1000
|
||||||
|
#-------------------histogram join each plan F,A,R,O and join all algorithms for the budget chosen
|
||||||
|
cmd="python3 hist_by_FARO.py ${ldata} ${figdir} ${mexp} ${mevals}"
|
||||||
|
$cmd
|
||||||
|
#-------------------histogram by pb join each plan F,A,R,O and join all algorithms for the budget chosen
|
||||||
|
cmd="python3 hist_by_FARO_pb.py ${ldata} ${figdir} ${mexp} ${mevals}"
|
||||||
|
$cmd
|
||||||
71
eo/contrib/irace/expe/beta/hist_by_FARO.py
Executable file
71
eo/contrib/irace/expe/beta/hist_by_FARO.py
Executable file
|
|
@ -0,0 +1,71 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from scipy.stats import mannwhitneyu
|
||||||
|
|
||||||
|
##python3 hist_by_FARO.py ./fastga_results_all/ ./hist_and_csv/ 100000 1000
|
||||||
|
#one plot for one experiment plan for the same budget fastga, and the same budget irace if there is a budget irace (A,F)
|
||||||
|
path=sys.argv[1]
|
||||||
|
figpath=sys.argv[2]
|
||||||
|
maxExp=sys.argv[3]
|
||||||
|
maxEv=sys.argv[4]
|
||||||
|
|
||||||
|
indF=-1
|
||||||
|
indFO=-1
|
||||||
|
averageConfigs=[]
|
||||||
|
name=[]
|
||||||
|
for fastga in os.listdir(path): #ddir : directory of fastga_plan
|
||||||
|
if(fastga in {"fastga_results_planA","fastga_results_planF","fastga_results_planO"}):
|
||||||
|
for plan in os.listdir(os.path.join(path,fastga)):
|
||||||
|
print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
|
||||||
|
if("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan):
|
||||||
|
average=[]
|
||||||
|
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,fastga,plan,"raw","data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
for fname in os.listdir(os.path.join(path,fastga,plan,"raw","data",fastgadir)):
|
||||||
|
with open(os.path.join(path,fastga,plan,"raw","data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average.append(auc)
|
||||||
|
averageConfigs.append(average)
|
||||||
|
nameid=plan.split("_")[0][-1]
|
||||||
|
name.append("plan"+nameid+"_"+"_".join(plan.split("_")[1:3]))
|
||||||
|
if("random" in fastga):
|
||||||
|
for randir in os.listdir(os.path.join(path,fastga)):
|
||||||
|
#eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
|
||||||
|
average=[]
|
||||||
|
if("maxEv="+str(maxEv)+"_" in randir):
|
||||||
|
for ddir in os.listdir(os.path.join(path,fastga,randir)): #ddir : directory of one run_elites_all or more
|
||||||
|
if("crossover" in ddir):
|
||||||
|
#name.append("_".join(ddir.split("_")[1:3]))
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,fastga,randir,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
for fname in os.listdir(os.path.join(path,fastga,randir,ddir,"data",fastgadir)):
|
||||||
|
with open(os.path.join(path,fastga,randir,ddir,"data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average.append(auc)
|
||||||
|
#hist[belonging(auc,cum)]+=1
|
||||||
|
averageConfigs.append(average)
|
||||||
|
name.append(randir.split("_")[0]+"_random")
|
||||||
|
|
||||||
|
|
||||||
|
figdir=os.path.join(figpath,"hist_FARO_by_budget")
|
||||||
|
try:
|
||||||
|
os.makedirs(figdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
#_,pv=mannwhitneyu(averageConfigs[indFO],averageConfigs[indF])
|
||||||
|
#print(name,len(averageConfigs))
|
||||||
|
plt.figure()
|
||||||
|
plt.hist(averageConfigs,bins=10,range=(0,1),align="mid",rwidth=0.9,label=name) #no label
|
||||||
|
plt.xlabel("performances")
|
||||||
|
plt.ylabel("Number of runs")
|
||||||
|
plt.xlim(0,1)
|
||||||
|
plt.ylim(0,8000)
|
||||||
|
plt.yticks(range(0,8000,500))
|
||||||
|
#plt.title("pvalue="+str(pv)+"\n medianeF="+str(np.median(averageConfigs[indF]))+", medianeFO="+str(np.median(averageConfigs[indFO])))
|
||||||
|
plt.legend()
|
||||||
|
plt.savefig(figdir+"/hist_planFARO"+"_maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+".png")
|
||||||
|
plt.close()
|
||||||
|
|
||||||
88
eo/contrib/irace/expe/beta/hist_by_FARO_pb.py
Executable file
88
eo/contrib/irace/expe/beta/hist_by_FARO_pb.py
Executable file
|
|
@ -0,0 +1,88 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
##python3 hist_by_FARO_pb.py ./fastga_results_all/ ./hist_and_csv/ 100000 1000
|
||||||
|
#19 histograms by plan F,A ,R O
|
||||||
|
path=sys.argv[1]
|
||||||
|
figpath=sys.argv[2]
|
||||||
|
maxExp=sys.argv[3]
|
||||||
|
maxEv=sys.argv[4]
|
||||||
|
|
||||||
|
hist_pb=[[] for i in range(19)]
|
||||||
|
name=[]
|
||||||
|
for fastga in os.listdir(path): #ddir : directory of fastga_plan
|
||||||
|
if(fastga in {"fastga_results_planA", "fastga_results_planF","fastga_results_planO"}):
|
||||||
|
for plan in os.listdir(os.path.join(path,fastga)):
|
||||||
|
#print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
|
||||||
|
#print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
|
||||||
|
if("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan):
|
||||||
|
nameid=fastga[-1]
|
||||||
|
name.append("plan"+nameid+"_".join(plan.split("_")[1:3]))
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,fastga,plan,"raw","data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
pb=int(fastgadir.split("_")[0].split("=")[1])
|
||||||
|
average_pb=[]
|
||||||
|
for fname in os.listdir(os.path.join(path,fastga,plan,"raw","data",fastgadir)):
|
||||||
|
with open(os.path.join(path,fastga,plan,"raw","data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average_pb.append(auc)
|
||||||
|
if(hist_pb[pb]==[]): #first algo
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
elif(len(hist_pb[pb])!=len(name)):
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
else:
|
||||||
|
hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
|
||||||
|
|
||||||
|
if("random" in fastga):
|
||||||
|
for randir in os.listdir(os.path.join(path,fastga)):
|
||||||
|
#eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
|
||||||
|
if(("maxEv="+str(maxEv)+"_") in randir):
|
||||||
|
#print("maxEv="+str(maxEv) in randir,randir)
|
||||||
|
name.append(randir.split("_")[0]+"_random")
|
||||||
|
for ddir in os.listdir(os.path.join(path,fastga,randir)): #ddir : directory of one run_elites_all or more
|
||||||
|
if("crossover" in ddir):
|
||||||
|
#name.append("_".join(ddir.split("_")[1:3]))
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,fastga,randir,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
average_pb=[]
|
||||||
|
pb=int(fastgadir.split("_")[0].split("=")[1])
|
||||||
|
for fname in os.listdir(os.path.join(path,fastga,randir,ddir,"data",fastgadir)):
|
||||||
|
with open(os.path.join(path,fastga,randir,ddir,"data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average_pb.append(auc)
|
||||||
|
#print(len(hist_pb[pb]),len(name), pb)
|
||||||
|
if(hist_pb[pb]==[]): #first algo
|
||||||
|
#print("entrer random vide")
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
elif(len(hist_pb[pb])!=len(name)):
|
||||||
|
#print("entrer random !=")
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
else:
|
||||||
|
hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
|
||||||
|
|
||||||
|
|
||||||
|
figdir=os.path.join(figpath,"hist_by_FARO_pb_maxExp={}_maxEv={}".format(maxExp,maxEv))
|
||||||
|
try:
|
||||||
|
os.makedirs(figdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
#colors=['yellow', 'green',"blue","pink","purple","orange","magenta","gray","darkred","cyan","brown","olivedrab","thistle","stateblue"]
|
||||||
|
print(name)
|
||||||
|
for pb in range(19):
|
||||||
|
print(pb, len(hist_pb[pb]))
|
||||||
|
for i in hist_pb[pb]:
|
||||||
|
print(len(i))
|
||||||
|
plt.figure()
|
||||||
|
plt.hist(hist_pb[pb],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name) #no label color=colors[:len(name)]
|
||||||
|
#for aucs in range(len(hist_pb[pb])):
|
||||||
|
#plt.hist(hist_pb[pb][aucs],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name[aucs]) #no label
|
||||||
|
plt.xlabel("performances")
|
||||||
|
plt.ylabel("Number of runs")
|
||||||
|
plt.ylim(0,800)
|
||||||
|
plt.xlim(0,1)
|
||||||
|
plt.yticks(range(0,800,50))
|
||||||
|
#plt.xticks(np.cumsum([0.1]*10))
|
||||||
|
plt.legend()
|
||||||
|
plt.savefig(figdir+"/hist_FARO_pb={}_maxExp={}_maxEv={}.png".format(pb,maxExp,maxEv))
|
||||||
|
plt.close()
|
||||||
90
eo/contrib/irace/expe/beta/hist_by_pb_budget_plan.py
Executable file
90
eo/contrib/irace/expe/beta/hist_by_pb_budget_plan.py
Executable file
|
|
@ -0,0 +1,90 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
# python3 hist_by_pb_budget_plan.py ./fastga_results_all/fastga_results_planF/ ./hist_and_csv/
|
||||||
|
#python3 hist_by_pb_budget_plan.py ./fastga_results_all/fastga_results_planO ./hist_and_csv
|
||||||
|
#get 19 histograms with number of budget bars, same as hist_join but now is by pb
|
||||||
|
|
||||||
|
#argv : list of elite results
|
||||||
|
path=sys.argv[1]
|
||||||
|
figpath=sys.argv[2]
|
||||||
|
#plan_name=sys.argv[3]
|
||||||
|
hist_pb=[[] for i in range(19)]
|
||||||
|
name=[]
|
||||||
|
if("random" in path):
|
||||||
|
plan_name="R"
|
||||||
|
else:
|
||||||
|
plan_name=path.strip("/").split("/")[-1][-1]
|
||||||
|
|
||||||
|
|
||||||
|
for plandir in os.listdir(path): #plandir: directory of an experiment of elite results
|
||||||
|
if("results_elites_all" in plandir):
|
||||||
|
#eg : plan2_maxExp=10000_maxEv=1000_2021-08-20T1347+0200_results_elites_all
|
||||||
|
budget_irace=plandir.split("_")[1].split("=")[1]
|
||||||
|
budget_fastga=plandir.split("_")[2].split("=")[1]
|
||||||
|
name.append("plan="+plan_name+"_"+"".join(plandir.split("_")[1:3])) #plan=*_maxExp=*_maxEv=*
|
||||||
|
|
||||||
|
for algodir in os.listdir(os.path.join(path,plandir,"raw","data")):
|
||||||
|
average_pb=[]
|
||||||
|
pb=int(algodir.split("_")[0].split("=")[1])
|
||||||
|
for algo in os.listdir(os.path.join(path,plandir,"raw","data",algodir)):
|
||||||
|
with open(os.path.join(path,plandir,"raw","data",algodir,algo)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average_pb.append(auc)
|
||||||
|
if(hist_pb[pb]==[]): #first algo
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
elif(len(hist_pb[pb])!=len(name)):
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
else:
|
||||||
|
hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
|
||||||
|
|
||||||
|
if("results_randoms" in plandir):
|
||||||
|
#eg : maxEv=1000_2021-08-20T1347+0200_results_random
|
||||||
|
budget_fastga=plandir.split("_")[0].split("=")[1]
|
||||||
|
name.append("plan="+plan_name+"_"+"".join(plandir.split("_")[0])) #plan=*_maxExp=*_maxEv=*
|
||||||
|
for algodir in os.listdir(os.path.join(path,plandir)):
|
||||||
|
|
||||||
|
for algo in os.listdir(os.path.join(path,plandir,algodir,"data")):
|
||||||
|
pb=int(algo.split("_")[0].split("=")[1])
|
||||||
|
average_pb=[]
|
||||||
|
for fname in os.listdir(os.path.join(path,plandir,algodir,"data",algo)):
|
||||||
|
with open(os.path.join(path,plandir,algodir,"data",algo,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average_pb.append(auc)
|
||||||
|
if(hist_pb[pb]==[]): #first algo
|
||||||
|
print("entrer")
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
elif(len(hist_pb[pb])!=len(name)):
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
else:
|
||||||
|
hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
print(path.split("/")[-1][-1])
|
||||||
|
|
||||||
|
figdir=os.path.join(figpath,"hist_by_{}_pb_budget_plan".format(plan_name))
|
||||||
|
#figdir=os.path.join(figpath,"hist_by_{}_pb_irace_maxEv={}".format(plan_name,1000))
|
||||||
|
try:
|
||||||
|
os.makedirs(figdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
for pb in range(19):
|
||||||
|
print(pb, len(hist_pb[pb]))
|
||||||
|
plt.figure()
|
||||||
|
plt.hist(hist_pb[pb],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name) #no label color=colors[:len(name)]
|
||||||
|
#for aucs in range(len(hist_pb[pb])):
|
||||||
|
#plt.hist(hist_pb[pb][aucs],bins=10,range=(0,1),align="mid",rwidth=0.9,edgecolor="red",label=name[aucs]) #no label
|
||||||
|
plt.xlabel("performances")
|
||||||
|
plt.ylabel("Number of runs")
|
||||||
|
plt.ylim(0,750)
|
||||||
|
plt.yticks(range(0,750,50))
|
||||||
|
plt.xlim(0,1)
|
||||||
|
plt.legend()
|
||||||
|
plt.savefig(figdir+"/hist_plan={}_pb={}_budget.png".format(plan_name,pb))
|
||||||
|
plt.close()
|
||||||
68
eo/contrib/irace/expe/beta/hist_join.py
Executable file
68
eo/contrib/irace/expe/beta/hist_join.py
Executable file
|
|
@ -0,0 +1,68 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from scipy.stats import mannwhitneyu
|
||||||
|
|
||||||
|
#cmd : python3 hist_join.py ./fastga_results_all/fastga_results_planO/ ./hist_and_csv/
|
||||||
|
#histogram by plan for the budgets (irace and fastag)
|
||||||
|
|
||||||
|
|
||||||
|
path=sys.argv[1] #argv : directory of a Plan (O, A, F)
|
||||||
|
figpath=sys.argv[2] #path to store the histograms
|
||||||
|
averageConfigs=[]
|
||||||
|
name=[]
|
||||||
|
if("fastga_results_plan" in path):
|
||||||
|
for ddir in os.listdir(path): #ddir : directory of one run_elites_all or more
|
||||||
|
if("plan" in ddir):
|
||||||
|
average=[]
|
||||||
|
name.append("_".join(ddir.split("_")[1:3]))
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,ddir,"raw","data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
for fname in os.listdir(os.path.join(path,ddir,"raw","data",fastgadir)):
|
||||||
|
with open(os.path.join(path,ddir,"raw","data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average.append(auc)
|
||||||
|
#hist[belonging(auc,cum)]+=1
|
||||||
|
averageConfigs.append(average)
|
||||||
|
#print(hist)
|
||||||
|
#print(average)
|
||||||
|
|
||||||
|
figdir=os.path.join(figpath,"hist_join")
|
||||||
|
try:
|
||||||
|
os.makedirs(figdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
print(name,len(averageConfigs))
|
||||||
|
|
||||||
|
"""
|
||||||
|
idd0=name[0].split("_")[0].split("=")[1][:-3]+"k"
|
||||||
|
idd1=name[1].split("_")[0].split("=")[1][:-3]+"k"
|
||||||
|
idd2=name[2].split("_")[0].split("=")[1][:-3]+"k"
|
||||||
|
|
||||||
|
#only for Budget irace 10000, 50000, 100000 ie: only three experiment results
|
||||||
|
titlename="median"+idd0+"={:.3f}".format(np.median(averageConfigs[0]))+" , median"+idd1+"={:.3f}".format(np.median(averageConfigs[1]))+" , median"+idd2+"={:.3f}".format(np.median(averageConfigs[2]))
|
||||||
|
_,pv=mannwhitneyu(averageConfigs[0],averageConfigs[1])
|
||||||
|
titlename+="\n pvalue{}={:.3f}".format(idd0+idd1,pv)
|
||||||
|
_,pv=mannwhitneyu(averageConfigs[0],averageConfigs[2])
|
||||||
|
titlename+=" ,pvalue{}={:.3f}".format(idd0+idd2,pv)
|
||||||
|
_,pv=mannwhitneyu(averageConfigs[1],averageConfigs[2])
|
||||||
|
titlename+=" ,pvalue{}={:.3f}".format(idd1+idd2,pv)
|
||||||
|
print(titlename)
|
||||||
|
"""
|
||||||
|
plt.figure()
|
||||||
|
plt.hist(averageConfigs,bins=10,range=(0,1),align="mid",rwidth=0.9,label=name) #no label
|
||||||
|
plt.xlabel("performances")
|
||||||
|
plt.ylabel("Number of runs")
|
||||||
|
plt.xlim(0,1)
|
||||||
|
plt.ylim(0,7000)
|
||||||
|
plt.yticks(range(0,7000,500))
|
||||||
|
#plt.title(titlename)
|
||||||
|
plt.legend()
|
||||||
|
plt.savefig(figdir+"/hist_plan"+path.strip("/")[-1]+"_by_budget.png")
|
||||||
|
#plt.savefig(figpath+"/hist_plan"+path.strip("/")[-1]+"_by_budgetI.png")
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
46
eo/contrib/irace/expe/beta/hist_join_random.py
Executable file
46
eo/contrib/irace/expe/beta/hist_join_random.py
Executable file
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
##python3 hist_random.py ./dastga_results_all/fastga_results_random ./hist_and_csv/
|
||||||
|
#argv : list of elite results
|
||||||
|
data=sys.argv[1]
|
||||||
|
figpath=sys.argv[2]
|
||||||
|
averageConfigs=[]
|
||||||
|
name=[]
|
||||||
|
for path in os.listdir(data):
|
||||||
|
#eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
|
||||||
|
average=[]
|
||||||
|
if("maxEv" in path):
|
||||||
|
for ddir in os.listdir(os.path.join(data,path)): #ddir : directory of one run_elites_all or more
|
||||||
|
if("crossover" in ddir):
|
||||||
|
#name.append("_".join(ddir.split("_")[1:3]))
|
||||||
|
for fastgadir in os.listdir(os.path.join(data,path,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
for fname in os.listdir(os.path.join(data,path,ddir,"data",fastgadir)):
|
||||||
|
with open(os.path.join(data,path,ddir,"data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0]) *(-1)
|
||||||
|
average.append(auc)
|
||||||
|
#hist[belonging(auc,cum)]+=1
|
||||||
|
averageConfigs.append(average)
|
||||||
|
name.append(path.split("_")[0])
|
||||||
|
|
||||||
|
figdir=os.path.join(figpath,"hist_join")
|
||||||
|
try:
|
||||||
|
os.makedirs(figdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
colors=['yellow', 'green',"blue","pink","purple","orange","magenta","gray","darkred","cyan","brown","olivedrab","thistle","stateblue"]
|
||||||
|
plt.figure()
|
||||||
|
plt.hist(averageConfigs,bins=10,range=(0,1),align="mid",rwidth=0.5,label=name) #no label
|
||||||
|
plt.xlabel("performances")
|
||||||
|
plt.ylabel("Number of runs")
|
||||||
|
plt.ylim([0,8000])
|
||||||
|
plt.xlim(0,1)
|
||||||
|
plt.yticks(range(0,8000,500))
|
||||||
|
#plt.xticks(np.cumsum([0.1]*10))
|
||||||
|
plt.legend()
|
||||||
|
plt.savefig(figdir+"/hist_random_by_budget.png")
|
||||||
|
plt.close()
|
||||||
13
eo/contrib/irace/expe/beta/irace_files_pA/forbidden.txt
Executable file
13
eo/contrib/irace/expe/beta/irace_files_pA/forbidden.txt
Executable file
|
|
@ -0,0 +1,13 @@
|
||||||
|
## Template for specifying forbidden parameter configurations in irace.
|
||||||
|
##
|
||||||
|
## This filename must be specified via the --forbidden-file command-line option
|
||||||
|
## (or forbiddenFile in scenario.txt).
|
||||||
|
##
|
||||||
|
## The format is one constraint per line. Each constraint is a logical
|
||||||
|
## expression (in R syntax). If a parameter configuration
|
||||||
|
## is generated that makes the logical expression evaluate to TRUE,
|
||||||
|
## then the configuration is discarded.
|
||||||
|
##
|
||||||
|
## Examples of valid logical operators are: == != >= <= > < & | ! %in%
|
||||||
|
(replacement %in% c(2,3,4,5,6,7,8,9,10)) & (offspringsize > popsize)
|
||||||
|
(replacement %in% c(1)) & (offspringsize < popsize)
|
||||||
15
eo/contrib/irace/expe/beta/irace_files_pF/forbidden.txt
Executable file
15
eo/contrib/irace/expe/beta/irace_files_pF/forbidden.txt
Executable file
|
|
@ -0,0 +1,15 @@
|
||||||
|
## Template for specifying forbidden parameter configurations in irace.
|
||||||
|
##
|
||||||
|
## This filename must be specified via the --forbidden-file command-line option
|
||||||
|
## (or forbiddenFile in scenario.txt).
|
||||||
|
##
|
||||||
|
## The format is one constraint per line. Each constraint is a logical
|
||||||
|
## expression (in R syntax). If a parameter configuration
|
||||||
|
## is generated that makes the logical expression evaluate to TRUE,
|
||||||
|
## then the configuration is discarded.
|
||||||
|
##
|
||||||
|
## Examples of valid logical operators are: == != >= <= > < & | ! %in%
|
||||||
|
(replacement %in% c(2,3,4,5,6,7,8,9,10)) & (offspringsize > popsize)
|
||||||
|
(replacement %in% c(1)) & (offspringsize < popsize)
|
||||||
|
#(as.numeric(replacement) == 2) & (offspringsize > popsize)
|
||||||
|
#(as.numeric(replacement) == 3) & (offspringsize > popsize)
|
||||||
48
eo/contrib/irace/expe/beta/irace_files_pO/default.instances
Executable file
48
eo/contrib/irace/expe/beta/irace_files_pO/default.instances
Executable file
|
|
@ -0,0 +1,48 @@
|
||||||
|
## This is an example of specifying instances with a file.
|
||||||
|
|
||||||
|
# Each line is an instance relative to trainInstancesDir
|
||||||
|
# (see scenario.txt.tmpl) and an optional sequence of instance-specific
|
||||||
|
# parameters that will be passed to target-runnerx when invoked on that
|
||||||
|
# instance.
|
||||||
|
|
||||||
|
0
|
||||||
|
1
|
||||||
|
2
|
||||||
|
3
|
||||||
|
4
|
||||||
|
5
|
||||||
|
6
|
||||||
|
7
|
||||||
|
8
|
||||||
|
9
|
||||||
|
10
|
||||||
|
11
|
||||||
|
12
|
||||||
|
13
|
||||||
|
14
|
||||||
|
15
|
||||||
|
16
|
||||||
|
17
|
||||||
|
18
|
||||||
|
19
|
||||||
|
20
|
||||||
|
21
|
||||||
|
22
|
||||||
|
23
|
||||||
|
24
|
||||||
|
25
|
||||||
|
26
|
||||||
|
27
|
||||||
|
28
|
||||||
|
29
|
||||||
|
30
|
||||||
|
31
|
||||||
|
32
|
||||||
|
33
|
||||||
|
34
|
||||||
|
35
|
||||||
|
36
|
||||||
|
37
|
||||||
|
38
|
||||||
|
39
|
||||||
|
40
|
||||||
228
eo/contrib/irace/expe/beta/irace_files_pO/example.scen
Executable file
228
eo/contrib/irace/expe/beta/irace_files_pO/example.scen
Executable file
|
|
@ -0,0 +1,228 @@
|
||||||
|
###################################################### -*- mode: r -*- #####
|
||||||
|
## Scenario setup for Iterated Race (irace).
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
## To use the default value of a parameter of iRace, simply do not set
|
||||||
|
## the parameter (comment it out in this file, and do not give any
|
||||||
|
## value on the command line).
|
||||||
|
|
||||||
|
## File that contains the description of the parameters of the target
|
||||||
|
## algorithm.
|
||||||
|
parameterFile = "./fastga.param"
|
||||||
|
|
||||||
|
## Directory where the programs will be run.
|
||||||
|
execDir = "."
|
||||||
|
|
||||||
|
## File to save tuning results as an R dataset, either absolute path or
|
||||||
|
## relative to execDir.
|
||||||
|
# logFile = "./irace.Rdata"
|
||||||
|
|
||||||
|
## Previously saved log file to recover the execution of irace, either
|
||||||
|
## absolute path or relative to the current directory. If empty or NULL,
|
||||||
|
## recovery is not performed.
|
||||||
|
# recoveryFile = ""
|
||||||
|
|
||||||
|
## Directory where training instances are located; either absolute path or
|
||||||
|
## relative to current directory. If no trainInstancesFiles is provided,
|
||||||
|
## all the files in trainInstancesDir will be listed as instances.
|
||||||
|
trainInstancesDir = "."
|
||||||
|
|
||||||
|
## File that contains a list of training instances and optionally
|
||||||
|
## additional parameters for them. If trainInstancesDir is provided, irace
|
||||||
|
## will search for the files in this folder.
|
||||||
|
trainInstancesFile = "./default.instances"
|
||||||
|
|
||||||
|
## File that contains a table of initial configurations. If empty or NULL,
|
||||||
|
## all initial configurations are randomly generated.
|
||||||
|
# configurationsFile = ""
|
||||||
|
|
||||||
|
## File that contains a list of logical expressions that cannot be TRUE
|
||||||
|
## for any evaluated configuration. If empty or NULL, do not use forbidden
|
||||||
|
## expressions.
|
||||||
|
# forbiddenFile = ""
|
||||||
|
|
||||||
|
## Script called for each configuration that executes the target algorithm
|
||||||
|
## to be tuned. See templates.
|
||||||
|
targetRunner = "./target-runner"
|
||||||
|
|
||||||
|
## Number of times to retry a call to targetRunner if the call failed.
|
||||||
|
# targetRunnerRetries = 0
|
||||||
|
|
||||||
|
## Optional data passed to targetRunner. This is ignored by the default
|
||||||
|
## targetRunner function, but it may be used by custom targetRunner
|
||||||
|
## functions to pass persistent data around.
|
||||||
|
# targetRunnerData = ""
|
||||||
|
|
||||||
|
## Optional R function to provide custom parallelization of targetRunner.
|
||||||
|
# targetRunnerParallel = ""
|
||||||
|
|
||||||
|
## Optional script or R function that provides a numeric value for each
|
||||||
|
## configuration. See templates/target-evaluator.tmpl
|
||||||
|
# targetEvaluator = ""
|
||||||
|
|
||||||
|
## Maximum number of runs (invocations of targetRunner) that will be
|
||||||
|
## performed. It determines the maximum budget of experiments for the
|
||||||
|
## tuning.
|
||||||
|
maxExperiments = 0 #100000
|
||||||
|
|
||||||
|
|
||||||
|
## Maximum total execution time in seconds for the executions of
|
||||||
|
## targetRunner. targetRunner must return two values: cost and time.
|
||||||
|
# maxTime = 60
|
||||||
|
|
||||||
|
## Fraction (smaller than 1) of the budget used to estimate the mean
|
||||||
|
## computation time of a configuration. Only used when maxTime > 0
|
||||||
|
# budgetEstimation = 0.02
|
||||||
|
|
||||||
|
## Maximum number of decimal places that are significant for numerical
|
||||||
|
## (real) parameters.
|
||||||
|
digits = 2
|
||||||
|
|
||||||
|
## Debug level of the output of irace. Set this to 0 to silence all debug
|
||||||
|
## messages. Higher values provide more verbose debug messages.
|
||||||
|
# debugLevel = 0
|
||||||
|
|
||||||
|
## Number of iterations.
|
||||||
|
# nbIterations = 0
|
||||||
|
|
||||||
|
## Number of runs of the target algorithm per iteration.
|
||||||
|
# nbExperimentsPerIteration = 0
|
||||||
|
|
||||||
|
## Randomly sample the training instances or use them in the order given.
|
||||||
|
# sampleInstances = 1
|
||||||
|
|
||||||
|
## Statistical test used for elimination. Default test is always F-test
|
||||||
|
## unless capping is enabled, in which case the default test is t-test.
|
||||||
|
## Valid values are: F-test (Friedman test), t-test (pairwise t-tests with
|
||||||
|
## no correction), t-test-bonferroni (t-test with Bonferroni's correction
|
||||||
|
## for multiple comparisons), t-test-holm (t-test with Holm's correction
|
||||||
|
## for multiple comparisons).
|
||||||
|
# testType = "F-test"
|
||||||
|
|
||||||
|
## Number of instances evaluated before the first elimination test. It
|
||||||
|
## must be a multiple of eachTest.
|
||||||
|
# firstTest = 5
|
||||||
|
|
||||||
|
## Number of instances evaluated between elimination tests.
|
||||||
|
# eachTest = 1
|
||||||
|
|
||||||
|
## Minimum number of configurations needed to continue the execution of
|
||||||
|
## each race (iteration).
|
||||||
|
# minNbSurvival = 0
|
||||||
|
|
||||||
|
## Number of configurations to be sampled and evaluated at each iteration.
|
||||||
|
# nbConfigurations = 0
|
||||||
|
|
||||||
|
## Parameter used to define the number of configurations sampled and
|
||||||
|
## evaluated at each iteration.
|
||||||
|
# mu = 5
|
||||||
|
|
||||||
|
## Confidence level for the elimination test.
|
||||||
|
# confidence = 0.95
|
||||||
|
|
||||||
|
## If the target algorithm is deterministic, configurations will be
|
||||||
|
## evaluated only once per instance.
|
||||||
|
# deterministic = 0
|
||||||
|
|
||||||
|
## Seed of the random number generator (by default, generate a random
|
||||||
|
## seed).
|
||||||
|
# seed = NA
|
||||||
|
|
||||||
|
## Number of calls to targetRunner to execute in parallel. Values 0 or 1
|
||||||
|
## mean no parallelization.
|
||||||
|
# parallel = 0
|
||||||
|
|
||||||
|
## Enable/disable load-balancing when executing experiments in parallel.
|
||||||
|
## Load-balancing makes better use of computing resources, but increases
|
||||||
|
## communication overhead. If this overhead is large, disabling
|
||||||
|
## load-balancing may be faster.
|
||||||
|
# loadBalancing = 1
|
||||||
|
|
||||||
|
## Enable/disable MPI. Use Rmpi to execute targetRunner in parallel
|
||||||
|
## (parameter parallel is the number of slaves).
|
||||||
|
# mpi = 0
|
||||||
|
|
||||||
|
## Specify how irace waits for jobs to finish when targetRunner submits
|
||||||
|
## jobs to a batch cluster: sge, pbs, torque or slurm. targetRunner must
|
||||||
|
## submit jobs to the cluster using, for example, qsub.
|
||||||
|
# batchmode = 0
|
||||||
|
|
||||||
|
## Enable/disable the soft restart strategy that avoids premature
|
||||||
|
## convergence of the probabilistic model.
|
||||||
|
# softRestart = 1
|
||||||
|
|
||||||
|
## Soft restart threshold value for numerical parameters. If NA, NULL or
|
||||||
|
## "", it is computed as 10^-digits.
|
||||||
|
# softRestartThreshold = ""
|
||||||
|
|
||||||
|
## Directory where testing instances are located, either absolute or
|
||||||
|
## relative to current directory.
|
||||||
|
# testInstancesDir = ""
|
||||||
|
|
||||||
|
## File containing a list of test instances and optionally additional
|
||||||
|
## parameters for them.
|
||||||
|
# testInstancesFile = ""
|
||||||
|
|
||||||
|
## Number of elite configurations returned by irace that will be tested if
|
||||||
|
## test instances are provided.
|
||||||
|
# testNbElites = 1
|
||||||
|
|
||||||
|
## Enable/disable testing the elite configurations found at each
|
||||||
|
## iteration.
|
||||||
|
# testIterationElites = 0
|
||||||
|
|
||||||
|
## Enable/disable elitist irace.
|
||||||
|
# elitist = 1
|
||||||
|
|
||||||
|
## Number of instances added to the execution list before previous
|
||||||
|
## instances in elitist irace.
|
||||||
|
# elitistNewInstances = 1
|
||||||
|
|
||||||
|
## In elitist irace, maximum number per race of elimination tests that do
|
||||||
|
## not eliminate a configuration. Use 0 for no limit.
|
||||||
|
# elitistLimit = 2
|
||||||
|
|
||||||
|
## User-defined R function that takes a configuration generated by irace
|
||||||
|
## and repairs it.
|
||||||
|
# repairConfiguration = ""
|
||||||
|
|
||||||
|
## Enable the use of adaptive capping, a technique designed for minimizing
|
||||||
|
## the computation time of configurations. This is only available when
|
||||||
|
## elitist is active.
|
||||||
|
# capping = 0
|
||||||
|
|
||||||
|
## Measure used to obtain the execution bound from the performance of the
|
||||||
|
## elite configurations: median, mean, worst, best.
|
||||||
|
# cappingType = "median"
|
||||||
|
|
||||||
|
## Method to calculate the mean performance of elite configurations:
|
||||||
|
## candidate or instance.
|
||||||
|
# boundType = "candidate"
|
||||||
|
|
||||||
|
## Maximum execution bound for targetRunner. It must be specified when
|
||||||
|
## capping is enabled.
|
||||||
|
# boundMax = 0
|
||||||
|
|
||||||
|
## Precision used for calculating the execution time. It must be specified
|
||||||
|
## when capping is enabled.
|
||||||
|
# boundDigits = 0
|
||||||
|
|
||||||
|
## Penalization constant for timed out executions (executions that reach
|
||||||
|
## boundMax execution time).
|
||||||
|
# boundPar = 1
|
||||||
|
|
||||||
|
## Replace the configuration cost of bounded executions with boundMax.
|
||||||
|
# boundAsTimeout = 1
|
||||||
|
|
||||||
|
## Percentage of the configuration budget used to perform a postselection
|
||||||
|
## race of the best configurations of each iteration after the execution
|
||||||
|
## of irace.
|
||||||
|
# postselection = 0
|
||||||
|
|
||||||
|
## Enable/disable AClib mode. This option enables compatibility with
|
||||||
|
## GenericWrapper4AC as targetRunner script.
|
||||||
|
# aclib = 0
|
||||||
|
|
||||||
|
## END of scenario file
|
||||||
|
############################################################################
|
||||||
|
|
||||||
10
eo/contrib/irace/expe/beta/irace_files_pO/fastga.param
Executable file
10
eo/contrib/irace/expe/beta/irace_files_pO/fastga.param
Executable file
|
|
@ -0,0 +1,10 @@
|
||||||
|
# name switch type range
|
||||||
|
# continuator "--continuator=" c (0)
|
||||||
|
crossoverrate "--crossover-rate=" r (0,1)
|
||||||
|
crossselector "--cross-selector=" c (0,1,2,3,4,5,6)
|
||||||
|
# aftercrossselector "--aftercross-selector=" c (0)
|
||||||
|
crossover "--crossover=" c (0,1,2,3,4,5,6,7,8,9)
|
||||||
|
mutationrate "--mutation-rate=" r (0,1)
|
||||||
|
mutselector "--mut-selector=" c (0,1,2,3,4,5,6)
|
||||||
|
mutation "--mutation=" c (0,1,2,3,4,5,6,7,8,9,10)
|
||||||
|
replacement "--replacement=" c (0,1,2,3,4,5,6,7,8,9,10)
|
||||||
88
eo/contrib/irace/expe/beta/irace_files_pO/target-runner
Executable file
88
eo/contrib/irace/expe/beta/irace_files_pO/target-runner
Executable file
|
|
@ -0,0 +1,88 @@
|
||||||
|
#!/bin/bash
|
||||||
|
###############################################################################
|
||||||
|
# This script is the command that is executed every run.
|
||||||
|
# Check the examples in examples/
|
||||||
|
#
|
||||||
|
# This script is run in the execution directory (execDir, --exec-dir).
|
||||||
|
#
|
||||||
|
# PARAMETERS:
|
||||||
|
# $1 is the candidate configuration number
|
||||||
|
# $2 is the instance ID
|
||||||
|
# $3 is the seed
|
||||||
|
# $4 is the instance name
|
||||||
|
# The rest ($* after `shift 4') are parameters to the run
|
||||||
|
#
|
||||||
|
# RETURN VALUE:
|
||||||
|
# This script should print one numerical value: the cost that must be minimized.
|
||||||
|
# Exit with 0 if no error, with 1 in case of error
|
||||||
|
###############################################################################
|
||||||
|
error() {
|
||||||
|
echo "`TZ=UTC date`: $0: error: $@"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
EXE="./fastga"
|
||||||
|
LOG_DIR="irace_logs"
|
||||||
|
|
||||||
|
FIXED_PARAMS="--problem=0"
|
||||||
|
MAX_EVALS=100
|
||||||
|
#
|
||||||
|
CONFIG_ID=$1
|
||||||
|
INSTANCE_ID=$2
|
||||||
|
SEED=$3
|
||||||
|
INSTANCE=$(echo $4 | sed 's/\//\n/g'|tail -n 1)
|
||||||
|
CROSSOVER_RATE=$5
|
||||||
|
CROSSOVER_SELECTOR=$6
|
||||||
|
CROSSOVER=$7
|
||||||
|
MUTATION_RATE=$8
|
||||||
|
MUT_SELECTOR=$9
|
||||||
|
MUTATION=${10}
|
||||||
|
REPLACEMENT=${11}
|
||||||
|
shift 11 || error "Not enough parameters"
|
||||||
|
|
||||||
|
INSTANCE_PARAMS=$*
|
||||||
|
|
||||||
|
# STDOUT=${LOG_DIR}/c${CONFIG_ID}_i${INSTANCE_ID}_s${SEED}.stdout
|
||||||
|
# STDERR=${LOG_DIR}/c${CONFIG_ID}_i${INSTANCE_ID}_s${SEED}.stderr
|
||||||
|
STDOUT="/dev/null"
|
||||||
|
STDERR="/dev/null"
|
||||||
|
|
||||||
|
if [ ! -x "${EXE}" ]; then
|
||||||
|
error "${EXE}: not found or not executable (pwd: $(pwd))"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# If the program just prints a number, we can use 'exec' to avoid
|
||||||
|
# creating another process, but there can be no other commands after exec.
|
||||||
|
#exec $EXE ${FIXED_PARAMS} -i $INSTANCE ${INSTANCE_PARAMS}
|
||||||
|
# exit 1
|
||||||
|
#
|
||||||
|
# Otherwise, save the output to a file, and parse the result from it.
|
||||||
|
# (If you wish to ignore segmentation faults you can use '{}' around
|
||||||
|
# the command.)
|
||||||
|
cmd="$EXE ${FIXED_PARAMS} --instance=${INSTANCE} --seed=${SEED} ${CROSSOVER_RATE} ${CROSSOVER_SELECTOR} ${CROSSOVER} ${MUTATION_RATE} ${MUT_SELECTOR} ${MUTATION} ${REPLACEMENT}"
|
||||||
|
# NOTE: irace seems to capture both stderr and stdout, so you should not output to stderr
|
||||||
|
echo ${cmd} > ${STDERR}
|
||||||
|
$cmd 2> ${STDERR} | tee ${STDOUT}
|
||||||
|
|
||||||
|
# The following code is useless if the binary only output a single number on stdout.
|
||||||
|
|
||||||
|
# This may be used to introduce a delay if there are filesystem
|
||||||
|
# issues.
|
||||||
|
# SLEEPTIME=1
|
||||||
|
# while [ ! -s "${STDOUT}" ]; do
|
||||||
|
# sleep $SLEEPTIME
|
||||||
|
# let "SLEEPTIME += 1"
|
||||||
|
# done
|
||||||
|
|
||||||
|
# This is an example of reading a number from the output.
|
||||||
|
# It assumes that the objective value is the first number in
|
||||||
|
# the first column of the last line of the output.
|
||||||
|
# if [ -s "${STDOUT}" ]; then
|
||||||
|
# COST=$(tail -n 1 ${STDOUT} | grep -e '^[[:space:]]*[+-]\?[0-9]' | cut -f1)
|
||||||
|
# echo "$COST"
|
||||||
|
# rm -f "${STDOUT}" "${STDERR}"
|
||||||
|
# exit 0
|
||||||
|
# else
|
||||||
|
# error "${STDOUT}: No such file or directory"
|
||||||
|
# fi
|
||||||
140
eo/contrib/irace/expe/beta/mwtestU.py
Executable file
140
eo/contrib/irace/expe/beta/mwtestU.py
Executable file
|
|
@ -0,0 +1,140 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from scipy.stats import mannwhitneyu
|
||||||
|
|
||||||
|
##cmd eg :
|
||||||
|
# python3 hist_by_2_4_5.py ./fastga_results_all/ ./hist_and_csv/ 100000 1000
|
||||||
|
|
||||||
|
#get the Mann Whitney test U results between the plan F and plan R
|
||||||
|
# (change ligne 23 and 44 for other plan, and the maxExp, maxEv for other budget)
|
||||||
|
|
||||||
|
path=sys.argv[1]
|
||||||
|
figpath=sys.argv[2] #directory to store the data
|
||||||
|
maxExp=sys.argv[3]
|
||||||
|
maxEv=sys.argv[4]
|
||||||
|
|
||||||
|
hist_pb=[[] for i in range(19)]
|
||||||
|
name=[]
|
||||||
|
randind=-1
|
||||||
|
for fastga in os.listdir(path): #ddir : directory of fastga_plan
|
||||||
|
if(fastga in {"fastga_results_planF"}):
|
||||||
|
for plan in os.listdir(os.path.join(path,fastga)):
|
||||||
|
print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
|
||||||
|
#print("maxExp="+str(maxExp)+"_maxEv="+str(maxEv) in plan,plan,"maxExp="+str(maxExp)+"_maxEv="+str(maxEv))
|
||||||
|
if("maxExp="+str(maxExp)+"_maxEv="+str(maxEv)+"_" in plan):
|
||||||
|
name.append("_".join(plan.split("_")[:3]))
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,fastga,plan,"raw","data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
pb=int(fastgadir.split("_")[0].split("=")[1])
|
||||||
|
average_pb=[]
|
||||||
|
for fname in os.listdir(os.path.join(path,fastga,plan,"raw","data",fastgadir)):
|
||||||
|
with open(os.path.join(path,fastga,plan,"raw","data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0])
|
||||||
|
average_pb.append(auc)
|
||||||
|
if(hist_pb[pb]==[]): #first algo
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
elif(len(hist_pb[pb])!=len(name)):
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
else:
|
||||||
|
hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
|
||||||
|
|
||||||
|
|
||||||
|
if("random" in fastga):
|
||||||
|
for randir in os.listdir(os.path.join(path,fastga)):
|
||||||
|
#eg path: maxEv=100_nbAlgo=15_2021-08-20T1511+0200_results_randoms
|
||||||
|
if(("maxEv="+str(maxEv)+"_") in randir):
|
||||||
|
print("maxEv="+str(maxEv) in randir,randir)
|
||||||
|
name.append(randir.split("_")[0]+"_random")
|
||||||
|
randind=len(name)-1
|
||||||
|
print(randind,name)
|
||||||
|
for ddir in os.listdir(os.path.join(path,fastga,randir)): #ddir : directory of one run_elites_all or more
|
||||||
|
if("crossover" in ddir):
|
||||||
|
for fastgadir in os.listdir(os.path.join(path,fastga,randir,ddir,"data")): #fastgadir : directory of 50 runs of a configuration
|
||||||
|
average_pb=[]
|
||||||
|
pb=int(fastgadir.split("_")[0].split("=")[1])
|
||||||
|
for fname in os.listdir(os.path.join(path,fastga,randir,ddir,"data",fastgadir)):
|
||||||
|
with open(os.path.join(path,fastga,randir,ddir,"data",fastgadir,fname)) as fd:
|
||||||
|
auc = float(fd.readlines()[0])
|
||||||
|
average_pb.append(auc)
|
||||||
|
#print(len(hist_pb[pb]),len(name), pb)
|
||||||
|
if(hist_pb[pb]==[]): #first algo
|
||||||
|
#print("entrer random vide")
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
elif(len(hist_pb[pb])!=len(name)):
|
||||||
|
#print("entrer random !=")
|
||||||
|
hist_pb[pb].append(average_pb)
|
||||||
|
else:
|
||||||
|
hist_pb[pb][len(name)-1]+=average_pb #another algo for the same plan
|
||||||
|
|
||||||
|
|
||||||
|
figdir=os.path.join(figpath,"mwtestU_FR")
|
||||||
|
try:
|
||||||
|
os.makedirs(figdir)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
#colors=['yellow', 'green',"blue","pink","purple","orange","magenta","gray","darkred","cyan","brown","olivedrab","thistle","stateblue"]
|
||||||
|
print(name)
|
||||||
|
|
||||||
|
filename="mwtestU_maxExp={}_maxEv={}_FR.csv".format(maxExp,maxEv)
|
||||||
|
with open(os.path.join(figdir,filename),'w+') as csvfile:
|
||||||
|
csvfile.write(" ,"+",".join(map(str,range(0,19)))+"\n")
|
||||||
|
meanvalue=[]
|
||||||
|
pvalue=[]
|
||||||
|
meanR=[]
|
||||||
|
meanF=[]
|
||||||
|
mdianR=[]
|
||||||
|
mdianF=[]
|
||||||
|
mdianvalue=[]
|
||||||
|
iqrR=[]
|
||||||
|
iqrF=[]
|
||||||
|
stdR=[]
|
||||||
|
stdF=[]
|
||||||
|
iqrvalue=[]
|
||||||
|
pstd=[]
|
||||||
|
|
||||||
|
for pb in range(19):
|
||||||
|
#hR,lR,_=plt.hist(hist_pb[pb][randind],bins=10,range=(-1,0),align="mid",label=name) #no label color=colors[:len(name)]
|
||||||
|
#hF,lF,_=plt.hist(hist_pb[pb][np.abs(1-randind)],bins=10,range=(-1,0),align="mid",label=name) #no label color=colors[:len(name)]
|
||||||
|
_,pv=mannwhitneyu(hist_pb[pb][np.abs(1-randind)],hist_pb[pb][randind])
|
||||||
|
print(_,pv)
|
||||||
|
#meanvalue.append(np.mean(np.array(hF)*np.array(lF[:len(lF)-1]))-np.mean(np.array(hR)*np.array(lR[:len(lR)-1])))
|
||||||
|
pstd.append(np.std(hist_pb[pb][np.abs(1-randind)])-np.std(hist_pb[pb][randind]))
|
||||||
|
stdF.append(np.std(hist_pb[pb][np.abs(1-randind)]))
|
||||||
|
stdR.append(np.std(hist_pb[pb][randind]))
|
||||||
|
meanF.append(np.mean(hist_pb[pb][np.abs(1-randind)]))
|
||||||
|
meanR.append(np.mean(hist_pb[pb][randind]))
|
||||||
|
mdianF.append(np.median(hist_pb[pb][np.abs(1-randind)]))
|
||||||
|
mdianR.append(np.median(hist_pb[pb][randind]))
|
||||||
|
mdianvalue.append(np.median(hist_pb[pb][np.abs(1-randind)])-np.median(hist_pb[pb][randind]))
|
||||||
|
meanvalue.append(np.mean(hist_pb[pb][np.abs(1-randind)])-np.mean(hist_pb[pb][randind]))
|
||||||
|
pvalue.append(pv)
|
||||||
|
Q1 = np.percentile(hist_pb[pb][np.abs(1-randind)], 25, interpolation = 'midpoint')
|
||||||
|
# Third quartile (Q3)
|
||||||
|
Q3 = np.percentile(hist_pb[pb][np.abs(1-randind)], 75, interpolation = 'midpoint')
|
||||||
|
# Interquaritle range (IQR)
|
||||||
|
iqrF.append( Q3 - Q1)
|
||||||
|
Q1 = np.percentile(hist_pb[pb][randind], 25, interpolation = 'midpoint')
|
||||||
|
# Third quartile (Q3)
|
||||||
|
Q3 = np.percentile(hist_pb[pb][randind], 75, interpolation = 'midpoint')
|
||||||
|
# Interquaritle range (IQR)
|
||||||
|
iqrR.append( Q3 - Q1)
|
||||||
|
print(_,pv)
|
||||||
|
iqrvalue=np.array(iqrF)-np.array(iqrR)
|
||||||
|
with open(os.path.join(figdir,filename),'a') as csvfile:
|
||||||
|
csvfile.write("mF-mR,"+",".join(map(str,meanvalue))+"\n")
|
||||||
|
csvfile.write("p_value,"+",".join(map(str,pvalue))+"\n")
|
||||||
|
csvfile.write("mF,"+",".join(map(str,meanF))+"\n")
|
||||||
|
csvfile.write("mR,"+",".join(map(str,meanR))+"\n")
|
||||||
|
csvfile.write("medianF-medianR,"+",".join(map(str,mdianvalue))+"\n")
|
||||||
|
csvfile.write("medianF,"+",".join(map(str,mdianF))+"\n")
|
||||||
|
csvfile.write("medianR,"+",".join(map(str,mdianR))+"\n")
|
||||||
|
csvfile.write("stdF-stdR,"+",".join(map(str,mdianvalue))+"\n")
|
||||||
|
csvfile.write("stdF,"+",".join(map(str,stdF))+"\n")
|
||||||
|
csvfile.write("stdR,"+",".join(map(str,stdR))+"\n")
|
||||||
|
csvfile.write("iqrF,"+",".join(map(str,iqrF))+"\n")
|
||||||
|
csvfile.write("iqrR,"+",".join(map(str,iqrR))+"\n")
|
||||||
|
csvfile.write("iqrF-iqrR,"+",".join(map(str,iqrvalue))+"\n")
|
||||||
|
|
||||||
|
|
||||||
35
eo/contrib/irace/expe/beta/parseO_irace_bests.py
Executable file
35
eo/contrib/irace/expe/beta/parseO_irace_bests.py
Executable file
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#parse data1
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
#print("pb,ecdf,id,crossover-rate,cross-selector,crossover,mutation-rate,mut-selector,mutation,replacement") #plan1
|
||||||
|
print("pb,ecdf,id,crossover-rate,cross-selector,crossover,mutation-rate,mut-selector,mutation,replacement")
|
||||||
|
|
||||||
|
|
||||||
|
#give the path of one experiment
|
||||||
|
argv=sys.argv[1]
|
||||||
|
for datadir in os.listdir(argv):
|
||||||
|
#if(os.path.isdir(os.path.join(argv,datadir))): check if argv/datadir is a directory
|
||||||
|
if(datadir.find("results_irace")>=0): #check if the directory is one JOB
|
||||||
|
for pb_dir in os.listdir(os.path.join(argv,datadir)):
|
||||||
|
if "results_problem" in pb_dir:
|
||||||
|
pb_id=pb_dir.replace("results_problem_","")
|
||||||
|
with open(os.path.join("./",argv,datadir,pb_dir,"irace.log")) as fd:
|
||||||
|
data = fd.readlines()
|
||||||
|
|
||||||
|
# Find the last best configuration
|
||||||
|
bests = [line.strip() for line in data if "Best-so-far" in line]
|
||||||
|
#print(datadir,bests)
|
||||||
|
best = bests[-1].split()
|
||||||
|
best_id, best_perf = best[2], best[5]
|
||||||
|
# print(best_id,best_perf)
|
||||||
|
|
||||||
|
# Filter the config detail
|
||||||
|
configs = [line.strip() for line in data if "--crossover-rate=" in line and best_id in line]
|
||||||
|
# print(configs)
|
||||||
|
|
||||||
|
# Format as CSV
|
||||||
|
algo = re.sub("\-\-\S*=", ",", configs[0])
|
||||||
|
csv_line = pb_id + "," + best_perf + "," + algo
|
||||||
|
print(csv_line.replace(" ",""))
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue