diff --git a/eo/contrib/irace/irace-config/example.scen b/eo/contrib/irace/irace-config/example.scen index 3c7e9beed..5f9fafe15 100644 --- a/eo/contrib/irace/irace-config/example.scen +++ b/eo/contrib/irace/irace-config/example.scen @@ -15,7 +15,7 @@ execDir = "." ## File to save tuning results as an R dataset, either absolute path or ## relative to execDir. -# logFile = "./irace.Rdata" +logFile = "" # "./irace.Rdata" ## Previously saved log file to recover the execution of irace, either ## absolute path or relative to the current directory. If empty or NULL, @@ -63,7 +63,7 @@ targetRunner = "./target-runner" ## Maximum number of runs (invocations of targetRunner) that will be ## performed. It determines the maximum budget of experiments for the ## tuning. (minimum: 180) -maxExperiments = 20000 # 100000 +maxExperiments = 100000 ## Maximum total execution time in seconds for the executions of ## targetRunner. targetRunner must return two values: cost and time. @@ -129,13 +129,13 @@ digits = 2 ## Number of calls to targetRunner to execute in parallel. Values 0 or 1 ## mean no parallelization. -# parallel = 0 +parallel = 0 ## Enable/disable load-balancing when executing experiments in parallel. ## Load-balancing makes better use of computing resources, but increases ## communication overhead. If this overhead is large, disabling ## load-balancing may be faster. -# loadBalancing = 1 +loadBalancing = 0 ## Enable/disable MPI. Use Rmpi to execute targetRunner in parallel ## (parameter parallel is the number of slaves). diff --git a/eo/contrib/irace/parse_baseline.py b/eo/contrib/irace/parse_baseline.py new file mode 100755 index 000000000..8f0193910 --- /dev/null +++ b/eo/contrib/irace/parse_baseline.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import sys + +print("algo,problem,seed,ECDF_AUC") + +algos_names = { + str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":1, "replacement":0}) : "EA", + str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":5, "replacement":0}) : "fEA", + str({"crossover-rate":2, "cross-selector":2, "crossover":2, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "xGA", + str({"crossover-rate":2, "cross-selector":2, "crossover":5, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "1ptGA", +} + +for fname in sys.argv[1:]: + + run = {} + for f in fname.strip(".dat").split("_"): + kv = f.split("=") + assert(len(kv)==2),str(kv)+" "+str(len(kv)) + key,idx = kv[0], int(kv[1]) + run[key] = idx + + with open(fname) as fd: + auc = int(fd.readlines()[0]) * -1 + + algo = str({"crossover-rate":run["crossover-rate"], "cross-selector":run["cross-selector"], "crossover":run["crossover"], "mutation-rate":run["mutation-rate"], "mut-selector":run["mut-selector"], "mutation":run["mutation"], "replacement":run["replacement"]}) + + print(algos_names[algo], run["pb"], run["seed"], auc, sep=",") diff --git a/eo/contrib/irace/parse_baseline_average.py b/eo/contrib/irace/parse_baseline_average.py new file mode 100755 index 000000000..2b71f7c4a --- /dev/null +++ b/eo/contrib/irace/parse_baseline_average.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +import sys + +print("problem,EA,fEA,xGA,1ptGA") + +algos_names = { + str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":1, "replacement":0}) : "EA", + str({"crossover-rate":0, "cross-selector":0, "crossover":0, "mutation-rate":0, "mut-selector":0, "mutation":5, "replacement":0}) : "fEA", + str({"crossover-rate":2, "cross-selector":2, "crossover":2, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "xGA", + str({"crossover-rate":2, "cross-selector":2, "crossover":5, "mutation-rate":2, "mut-selector":2, "mutation":1, "replacement":0}) : "1ptGA", +} + +data = {} + +# Parse +for fname in sys.argv[1:]: + + run = {} + for f in fname.strip(".dat").split("_"): + kv = f.split("=") + assert(len(kv)==2),str(kv)+" "+str(len(kv)) + key,idx = kv[0], int(kv[1]) + run[key] = idx + + with open(fname) as fd: + auc = int(fd.readlines()[0]) * -1 + + algo = str({"crossover-rate":run["crossover-rate"], "cross-selector":run["cross-selector"], "crossover":run["crossover"], "mutation-rate":run["mutation-rate"], "mut-selector":run["mut-selector"], "mutation":run["mutation"], "replacement":run["replacement"]}) + + if run["pb"] in data: + data[run["pb"]][algos_names[algo]].append(auc) + else: + data[run["pb"]] = { + "EA" : [], + "fEA" : [], + "xGA" : [], + "1ptGA": [], + } + data[run["pb"]][algos_names[algo]].append(auc) + + +# Print CSV +for pb in sorted(data.keys()): + print(pb, end="") + for algo in ["EA","fEA","xGA","1ptGA"]: + res = data[pb][algo] + print(",", sum(res)/len(res), end="", sep="") + print() diff --git a/eo/contrib/irace/parse_elites.py b/eo/contrib/irace/parse_elites.py new file mode 100755 index 000000000..518bc7245 --- /dev/null +++ b/eo/contrib/irace/parse_elites.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import sys + +print("algo,problem,seed,ECDF_AUC") + +for fname in sys.argv[1:]: + + run = {} + for f in fname.strip(".dat").split("_"): + kv = f.split("=") + assert(len(kv)==2),str(kv)+" "+str(len(kv)) + key,idx = kv[0], int(kv[1]) + run[key] = idx + + with open(fname) as fd: + auc = int(fd.readlines()[0]) * -1 + + algo = "pc={}_c={}_C={}_pm={}_m={}_M={}_R={}".format(run["crossover-rate"],run["cross-selector"],run["crossover"],run["mutation-rate"],run["mut-selector"],run["mutation"],run["replacement"]) + + print(algo, run["pb"], run["seed"], auc, sep=",")