Add experimental stuff

- add ecdf module
- add expe module
- add func module
- use Dump wrapper around obj. func. instead of iters.
- add random solvers in snp options.
- add no-plot option in snp.

ecdf.py

@@ -0,0 +1,137 @@
+import sys
+import csv
+import argparse
+import numpy as np
+import matplotlib.pyplot as plt
+from difflib import SequenceMatcher
+
+
+def guess_number_evals(filenames):
+    """Guess the number of evals from first file."""
+    with open(filenames[0], 'r') as fd:
+        nevals = len(fd.readlines())
+    return nevals
+
+
+def along_runtime(filenames, data):
+    for fid,filename in enumerate(filenames):
+        with open(filename, 'r') as fd:
+            strdata = csv.reader(fd, delimiter=';')
+            for i,row in enumerate(strdata):
+                evals = int(row[0])
+                val = float(row[1])
+                data[evals,fid] = val
+    return data
+
+
+def cumul(data, delta, optim = None, do_min = False):
+    # Keep only best values along columns.
+    for i in range(1,len(data)):
+        for j in range(len(data[i])):
+            data[i,j] = max( data[i,j], data[i-1,j] )
+
+    if not optim:
+        optim = data.max()
+
+    # Normalize.
+    norm = data/optim
+
+    # Threshold.
+    if do_min:
+        ecdf = (norm < delta)
+    else:
+        ecdf = (norm > delta)
+
+    # Sum across rows.
+    return ecdf.sum(axis=1)/data.shape[1]
+
+
+def parse(filenames, delta, nb_rows = None, optim = None, do_min = False):
+    if not nb_rows:
+        nb_rows = guess_number_evals(filenames)
+
+    data = np.zeros( (nb_rows+1, len(filenames)) )
+    data = along_runtime(filenames,data)
+    ert = cumul(data, delta, optim, do_min)
+    return ert
+
+
+def make_name(names, delta, erts, name_strip = [], do_min = False):
+    common = names[0]
+    for run in names:
+        match = SequenceMatcher(None, common, run).find_longest_match(0, len(common), 0, len(run))
+        common = common[match.a: match.a + match.size]
+
+    for strp in name_strip:
+        common = common.replace(strp,"")
+
+    name = u"{} $\Delta={}$".format(common,delta)
+
+    if name in erts:
+        i += 1
+        name += " ({})".format(i)
+
+    return name
+
+
+if __name__ == "__main__":
+
+
+    can = argparse.ArgumentParser()
+
+    can.add_argument("-e", "--evals", metavar="NB", default=None, type=int,
+            help="Max number of evaluations to consider")
+
+    # can.add_argument("-q", "--quality", action='store_true',
+    #         help="Produce Expected Quality ECDF, instead of Expected Runtime ECDF.")
+
+    can.add_argument("-m", "--min", action='store_true',
+            help="Minimization problem, instead of maximization.")
+
+    can.add_argument("-o", "--optimum", metavar="VAL", default=None, type=float,
+            help="Best value used for normalization (else, default to the max in the data).")
+
+    can.add_argument("-s", "--name-strip", metavar="STR", default=[],
+            type=str, action='append',
+            help="Remove this string from the labels.")
+
+    can.add_argument("-d", "--delta", metavar="PERC",
+            action='append', type=float, required=True,
+            help="Target(s), as a percentage of values normalized against optimum.")
+
+    can.add_argument("-r", "--runs", metavar="FILES", nargs='*', required=True, action='append')
+
+    the = can.parse_args()
+
+    print(the.name_strip)
+
+    erts = {}
+    names = []
+    i = 0
+    for runs in the.runs:
+        for delta in the.delta:
+            ert = parse(
+                    runs, delta,
+                    nb_rows = the.evals, optim = the.optimum, do_min = the.min
+                )
+
+            name = make_name(runs, delta, erts, the.name_strip, the.min)
+            erts[name] = ert
+
+    fig = plt.figure()
+    for name in erts:
+        plt.plot(erts[name], label=name)
+
+    plt.ylim([0,1])
+
+    if the.min:
+        comp = "<"
+    else:
+        comp=">"
+    # plt.ylabel(r"$P\left(f\left(\hat{x})\right)/"+str(the.optimum)+comp+r"\Delta\right)$")
+    plt.ylabel(r"$P\left(1/"+str(the.optimum)+r"\cdot f\left(\hat{x})\right)"+comp+r"\Delta\right)$")
+    plt.xlabel("Time (#function evals)")
+    plt.title("Expected RunTime Empirical Cumulative Density Function")
+    plt.legend()
+    plt.show()
+

expe.py

@@ -0,0 +1,46 @@
+
+if __name__ == "__main__":
+    import os
+    import subprocess
+
+    # can = argparse.ArgumentParser()
+    #
+    # can.add_argument("-n", "--nb-sensors", metavar="NB", default=3, type=int,
+    #         help="Number of sensors")
+    #
+    # can.add_argument("-r", "--sensor-range", metavar="RATIO", default=0.3, type=float,
+    #         help="Sensors' range (as a fraction of domain width)")
+    #
+    # can.add_argument("-w", "--domain-width", metavar="NB", default=30, type=int,
+    #         help="Domain width (a number of cells)")
+    #
+    # can.add_argument("-i", "--iters", metavar="NB", default=100, type=int,
+    #         help="Maximum number of iterations")
+    #
+    # the = can.parse_args()
+
+    const_args=" --nb-sensors 5 --sensor-range 0.2 --domain-width 50 --iters 10000"
+    solvers = ["num_greedy","bit_greedy","num_rand","bit_rand"]
+    nbruns = 100
+    outdir = "results"
+
+    if not os.path.exists(outdir):
+        os.mkdir(outdir)
+
+    for seed in range(nbruns):
+        procs = []
+        for solver in solvers:
+            print(seed,solver)
+            p = subprocess.Popen(
+                    "python3 snp.py "
+                    + const_args
+                    + " --no-plot --dir {} --seed {} --solver {}"
+                    .format(outdir,seed,solver),
+                    shell=True
+                )
+            procs.append(p)
+
+        for proc in procs:
+            proc.wait()
+
+

sho/func.py

@@ -0,0 +1,26 @@
+
+########################################################################
+# Wrappers around objective functions
+########################################################################
+
+class Dump:
+    """A wrapper around an objective function that
+    dumps a line in a file every time the objective function is called."""
+
+    def __init__(self, func, filename="run.csv", fmt="{it} ; {val} ; {sol}\n", sepsol=" , "):
+        self.func = func
+        self.filename = filename
+        self.fmt = fmt
+        self.sepsol = sepsol
+        self.counter = 0
+        # Erase previous file.
+        with open(self.filename, 'w') as fd:
+            fd.write("")
+
+    def __call__(self, sol):
+        val = self.func(sol)
+        self.counter += 1
+        with open(self.filename, 'a') as fd:
+            fmtsol = self.sepsol.join([str(i) for i in sol])
+            fd.write( self.fmt.format(it=self.counter, val=val, sol=fmtsol) )
+        return val

snp.py

@@ -1,7 +1,8 @@
+import os
 import numpy as np
 import matplotlib.pyplot as plt
 
-from sho import *
+from sho import algo, bit, func, iters, make, num, pb, plot
 
 ########################################################################
 # Interface
@@ -30,18 +31,23 @@ if __name__=="__main__":
     can.add_argument("-s", "--seed", metavar="VAL", default=None, type=int,
             help="Random pseudo-generator seed (none for current epoch)")
 
-    solvers = ["num_greedy","bit_greedy"]
+    solvers = ["num_greedy","bit_greedy","num_rand","bit_rand"]
     can.add_argument("-m", "--solver", metavar="NAME", choices=solvers, default="num_greedy",
             help="Solver to use, among: "+", ".join(solvers))
 
-    can.add_argument("-t", "--target", metavar="VAL", default=30*30, type=float,
+    # can.add_argument("-t", "--target", metavar="VAL", default=30*30, type=float,
-            help="Objective function value target")
+    #         help="Objective function value target")
+    #
+    # can.add_argument("-y", "--steady-delta", metavar="NB", default=50, type=float,
+    #         help="Stop if no improvement after NB iterations")
+    # can.add_argument("-e", "--steady-epsilon", metavar="DVAL", default=0, type=float,
+    #         help="Stop if the improvement of the objective function value is lesser than DVAL")
 
-    can.add_argument("-y", "--steady-delta", metavar="NB", default=50, type=float,
+    can.add_argument("-p", "--no-plot", action='store_true',
-            help="Stop if no improvement after NB iterations")
+            help="Do not display plots.")
-    can.add_argument("-e", "--steady-epsilon", metavar="DVAL", default=0, type=float,
-            help="Stop if the improvement of the objective function value is lesser than DVAL")
 
+    can.add_argument("-d", "--dir", metavar="DIR", default="", type=str,
+            help="Directory to which output written files.")
 
     the = can.parse_args()
 
@@ -66,29 +72,34 @@ if __name__=="__main__":
                 agains = [
                     make.iter(iters.max,
                         nb_it = the.iters),
-                    make.iter(iters.save,
+                    # make.iter(iters.save,
-                        filename = the.solver+".csv",
+                    #     filename = os.path.join(the.dir,the.solver+".csv"),
-                        fmt = "{it} ; {val} ; {sol}\n"),
+                    #     fmt = "{it} ; {val} ; {sol}\n"),
                     make.iter(iters.log,
                         fmt="\r{it} {val}"),
                     make.iter(iters.history,
                         history = history),
-                    make.iter(iters.target,
+                    # make.iter(iters.target,
-                        target = the.target),
+                    #     target = the.target),
-                    iters.steady(the.steady_delta, the.steady_epsilon)
+                    # iters.steady(the.steady_delta, the.steady_epsilon)
                 ]
             )
 
     # Erase the previous file.
-    with open(the.solver+".csv", 'w') as fd:
+    # with open(the.solver+".csv", 'w') as fd:
-        fd.write("# {} {}\n".format(the.solver,the.domain_width))
+    #     fd.write("# {} {}\n".format(the.solver,the.domain_width))
 
     val,sol,sensors = None,None,None
     if the.solver == "num_greedy":
-        val,sol = algo.greedy(
+        fdump = func.Dump(
                 make.func(num.cover_sum,
                     domain_width = the.domain_width,
                     sensor_range = the.sensor_range * the.domain_width),
+                filename = os.path.join(the.dir,"{s}_run_{i}.csv".format(s=the.solver, i=the.seed)),
+                fmt = "{it} ; {val} ; {sol}\n"
+            )
+        val,sol = algo.greedy(
+                fdump,
                 make.init(num.rand,
                     dim = d * the.nb_sensors,
                     scale = the.domain_width),
@@ -98,11 +109,34 @@ if __name__=="__main__":
             )
         sensors = num.to_sensors(sol)
 
+    if the.solver == "num_rand":
+        fdump = func.Dump(
+                make.func(num.cover_sum,
+                    domain_width = the.domain_width,
+                    sensor_range = the.sensor_range * the.domain_width),
+                filename = os.path.join(the.dir,"{s}_run_{i}.csv".format(s=the.solver, i=the.seed)),
+                fmt = "{it} ; {val} ; {sol}\n"
+            )
+        val,sol = algo.random(
+                fdump,
+                make.init(num.rand,
+                    dim = d * the.nb_sensors,
+                    scale = the.domain_width),
+                iters
+            )
+        sensors = num.to_sensors(sol)
+
+
     elif the.solver == "bit_greedy":
-        val,sol = algo.greedy(
+        fdump = func.Dump(
                 make.func(bit.cover_sum,
                     domain_width = the.domain_width,
                     sensor_range = the.sensor_range),
+                filename = os.path.join(the.dir,"{s}_run_{i}.csv".format(s=the.solver, i=the.seed)),
+                fmt = "{it} ; {val} ; {sol}\n"
+            )
+        val,sol = algo.greedy(
+                fdump,
                 make.init(bit.rand,
                     domain_width = the.domain_width,
                     nb_sensors = the.nb_sensors),
@@ -112,10 +146,29 @@ if __name__=="__main__":
             )
         sensors = bit.to_sensors(sol)
 
+    elif the.solver == "bit_rand":
+        fdump = func.Dump(
+                make.func(bit.cover_sum,
+                    domain_width = the.domain_width,
+                    sensor_range = the.sensor_range),
+                filename = os.path.join(the.dir,"{s}_run_{i}.csv".format(s=the.solver, i=the.seed)),
+                fmt = "{it} ; {val} ; {sol}\n"
+            )
+        val,sol = algo.random(
+                fdump,
+                make.init(bit.rand,
+                    domain_width = the.domain_width,
+                    nb_sensors = the.nb_sensors),
+                iters
+            )
+        sensors = bit.to_sensors(sol)
+
+
 
     # Fancy output.
     print("\n{} : {}".format(val,sensors))
 
+    if not the.no_plot:
         shape=(the.domain_width, the.domain_width)
 
         fig = plt.figure()