manual merge: deactivate t-eoParallel, that depends on OpenMP, whatever the build options

eo/CMakeLists.txt

@@ -121,6 +121,11 @@ ENDIF (ENABLE_CMAKE_TESTING)
 ADD_SUBDIRECTORY(doc)
 ADD_SUBDIRECTORY(src)
 ADD_SUBDIRECTORY(test)
+
+IF(WITH_MPI)
+    ADD_SUBDIRECTORY(test/mpi)
+ENDIF()
+
 ADD_SUBDIRECTORY(tutorial)
 
 ######################################################################################

eo/src/apply.h

@@ -36,12 +36,6 @@
 #include <omp.h>
 #endif
 
-# ifdef WITH_MPI
-# include <mpi/eoMpi.h>
-# include <mpi/eoMultiParallelApply.h>
-# include <mpi/eoTerminateJob.h>
-# endif // WITH_MPI
-
 /**
   Applies a unary function to a std::vector of things.
 
@@ -88,21 +82,6 @@ void apply(eoUF<EOT&, void>& _proc, std::vector<EOT>& _pop)
 #endif // !_OPENMP
 }
 
-#ifdef WITH_MPI
-template<class EOT>
-void parallelApply(
-        std::vector<EOT>& _pop,
-        eo::mpi::AssignmentAlgorithm& _algo,
-        int _masterRank,
-        eo::mpi::ParallelEvalStore<EOT> & _store )
-{
-    _store.data( _pop );
-    _algo.reinit( _pop.size() );
-    eo::mpi::ParallelApply<EOT> job( _algo, _masterRank, _store );
-    job.run();
-}
-#endif
-
 /**
   This is a variant of apply<EOT> which is called in parallel
   thanks to OpenMP.

eo/src/eoEasyEA.h

@@ -37,8 +37,6 @@
 #include <eoMergeReduce.h>
 #include <eoReplacement.h>
 
-
-
 template <class EOT> class eoIslandsEasyEA ;
 
 template <class EOT> class eoDistEvalEasyEA ;
@@ -247,7 +245,6 @@ template<class EOT> class eoEasyEA: public eoAlgo<EOT>
     virtual void operator()(eoPop<EOT>& _pop)
     {
 
-        eo::log << "[EasyEA] Call to operator()" << std::endl;
         if (isFirstCall)
         {
             size_t total_capacity = _pop.capacity() + offspring.capacity();
@@ -256,39 +253,26 @@ template<class EOT> class eoEasyEA: public eoAlgo<EOT>
             isFirstCall = false;
         }
 
-    // TODO TODOB delete all log traces
-    std::cout << "[EasyEA] After is first call." << std::endl;
-
         eoPop<EOT> empty_pop;
-        std::cout << "[EasyEA] After empty_pop." << std::endl;
-
-      popEval(empty_pop, _pop); // A first eval of pop.
-        std::cout << "[EasyEA] After pop_eval." << std::endl;
 
         do
         {
             try
             {
-        std::cout << "[EasyEA] Beginning try." << std::endl;
                 unsigned pSize = _pop.size();
-        std::cout << "[EasyEA] psize determinated." << std::endl;
+
                 offspring.clear(); // new offspring
-        std::cout << "[EasyEA] offspring cleared." << std::endl;
 
                 breed(_pop, offspring);
 
-        std::cout << "[EasyEA] After breed, evaluating pop." << std::endl;
                 popEval(_pop, offspring); // eval of parents + offspring if necessary
 
-        std::cout << "[EasyEA] After evaluation, replacing pop." << std::endl;
                 replace(_pop, offspring); // after replace, the new pop. is in _pop
-        std::cout << "[EasyEA] After replacing, continuator." << std::endl;
 
                 if (pSize > _pop.size())
                     throw std::runtime_error("Population shrinking!");
                 else if (pSize < _pop.size())
                     throw std::runtime_error("Population growing!");
-
             }
             catch (std::exception& e)
             {

eo/src/eoPopEvalFunc.h

@@ -30,6 +30,16 @@
 #include <eoEvalFunc.h>
 #include <apply.h>
 
+# ifdef WITH_MPI
+#include <mpi/eoMpi.h>
+#include <mpi/eoTerminateJob.h>
+#include <mpi/eoMpiAssignmentAlgorithm.h>
+#include <mpi/eoParallelApply.h>
+#include <utils/eoParallel.h>
+
+#include <cmath> // ceil
+# endif // WITH_MPI
+
 /** eoPopEvalFunc: This abstract class is for GLOBAL evaluators
  *    of a population after variation.
  *    It takes 2 populations (typically the parents and the offspring)
@@ -78,12 +88,43 @@ private:
 };
 
 #ifdef WITH_MPI
-// TODO TODOB commenter
+/**
+ * @brief Evaluator of a population of EOT which uses parallelization to evaluate individuals.
+ *
+ * This class implements an instance of eoPopEvalFunc that applies a private eoEvalFunc to
+ * all offspring, but in a parallel way. The original process becomes the central host from a network ("master"), and
+ * other machines disponible in the MPI network ("slaves") are used as evaluators. Population to evaluate is splitted in
+ * little packets of individuals, which are sent to the evaluators, that process the effective call to eval. Once all
+ * the individuals have been evaluated, they are returned to the master. The whole process is entirely invisible to the
+ * eyes of the user, who just has to launch a certain number of processes in MPI so as to have a result.
+ *
+ * The eoEvalFunc is no more directly given, but it is stored in the eo::mpi::ParallelApplyStore, which can be
+ * instanciated if no one is given at construction.
+ *
+ * The use of this class requires the user to have called the eo::mpi::Node::init function, at the beginning of its
+ * program.
+ *
+ * @ingroup Evaluation Parallel
+ *
+ * @author Benjamin Bouvier <benjamin.bouvier@gmail.com>
+ */
 template<class EOT>
 class eoParallelPopLoopEval : public eoPopEvalFunc<EOT>
 {
     public:
-        /** Ctor: set value of embedded eoEvalFunc */
+        /**
+         * @brief Constructor which creates the job store for the user.
+         *
+         * This constructor is the simplest to use, as it creates the store used by the parallel job, for the user.
+         * The user just precises the scheduling algorithm, the rank of the master and then gives its eval function and
+         * the size of a packet (how many individuals should be in a single message to evaluator).
+         *
+         * @param _assignAlgo The scheduling algorithm used to give orders to evaluators.
+         * @param _masterRank The MPI rank of the master.
+         * @param _eval The evaluation functor used to evaluate each individual in the population.
+         * @param _packetSize The number of individuals to send in one message to evaluator, and which are evaluated at
+         * a time.
+         */
         eoParallelPopLoopEval(
                 // Job parameters
                 eo::mpi::AssignmentAlgorithm& _assignAlgo,
@@ -94,53 +135,92 @@ class eoParallelPopLoopEval : public eoPopEvalFunc<EOT>
                 ) :
             assignAlgo( _assignAlgo ),
             masterRank( _masterRank ),
-            needToDeleteStore( true )
+            needToDeleteStore( true ) // we used new, we'll have to use delete (RAII)
         {
-            store = new eo::mpi::ParallelEvalStore<EOT>( _eval, _masterRank, _packetSize );
+            store = new eo::mpi::ParallelApplyStore<EOT>( _eval, _masterRank, _packetSize );
         }
 
+        /**
+         * @brief Constructor which allows the user to customize its job store.
+         *
+         * This constructor allows the user to customize the store, for instance by adding wrappers and other
+         * functionnalities, before using it for the parallelized evaluation.
+         *
+         * @param _assignAlgo The scheduling algorithm used to give orders to evaluators.
+         * @param _masterRank The MPI rank of the master.
+         * @param _store Pointer to a parallel eval store given by the user.
+         */
         eoParallelPopLoopEval(
                 // Job parameters
                 eo::mpi::AssignmentAlgorithm& _assignAlgo,
                 int _masterRank,
-                eo::mpi::ParallelEvalStore<EOT>* _store
+                eo::mpi::ParallelApplyStore<EOT>* _store
                 ) :
             assignAlgo( _assignAlgo ),
             masterRank( _masterRank ),
             store( _store ),
-            needToDeleteStore( false )
+            needToDeleteStore( false ) // we haven't used new for creating store, we don't care if we have to delete it (RAII).
         {
             // empty
         }
 
+        /**
+         * @brief Default destructor. Sends a message to all evaluators indicating that the global loop (eoEasyEA, for
+         * instance) is over.
+         */
         ~eoParallelPopLoopEval()
         {
+            // Only the master has to send the termination message
             if( eo::mpi::Node::comm().rank() == masterRank )
             {
                 eo::mpi::EmptyJob job( assignAlgo, masterRank );
                 job.run();
             }
 
+            // RAII
             if( needToDeleteStore )
             {
                 delete store;
             }
         }
 
-        /** Do the job: simple loop over the offspring */
+        /**
+         * @brief Parallel implementation of the operator().
+         *
+         * @param _parents Population of parents (ignored).
+         * @param _offspring Population of children, which will be evaluated.
+         */
         void operator()( eoPop<EOT> & _parents, eoPop<EOT> & _offspring )
         {
             (void)_parents;
-            parallelApply<EOT>(_offspring, assignAlgo, masterRank, *store);
+            // Reinits the store and the scheduling algorithm
+            store->data( _offspring );
+            // For static scheduling, it's mandatory to reinit attributions
+            int nbWorkers = assignAlgo.availableWorkers();
+            assignAlgo.reinit( nbWorkers );
+            if( ! eo::parallel.isDynamic() ) {
+                store->data()->packetSize = ceil( static_cast<double>( _offspring.size() ) / nbWorkers );
+            }
+            // Effectively launches the job.
+            eo::mpi::ParallelApply<EOT> job( assignAlgo, masterRank, *store );
+            job.run();
         }
 
     private:
+        // Scheduling algorithm
         eo::mpi::AssignmentAlgorithm & assignAlgo;
+        // Master MPI rank
         int masterRank;
 
-        eo::mpi::ParallelEvalStore<EOT>* store;
+        // Store
+        eo::mpi::ParallelApplyStore<EOT>* store;
+        // Do we have to delete the store by ourselves ?
         bool needToDeleteStore;
 };
+
+/**
+ * @example t-mpi-eval.cpp
+ */
 #endif
 
 /////////////////////////////////////////////////////////////

eo/src/mpi/eoMpi.cpp

@@ -1,6 +1,5 @@
 # include "eoMpi.h"
 
-// MpiNode* MpiNodeStore::singleton;
 namespace eo
 {
     namespace mpi

eo/src/mpi/eoMpi.h

@@ -1,10 +1,28 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EO_MPI_H__
 # define __EO_MPI_H__
 
-# include <vector>
+# include <vector>  // std::vector
-# include <map>
-# include <sys/time.h>
-# include <sys/resource.h>
 
 # include <utils/eoLogger.h>
 # include <utils/eoTimer.h>
@@ -14,42 +32,229 @@
 # include "eoMpiNode.h"
 # include "eoMpiAssignmentAlgorithm.h"
 
-// TODO TODOB comment!
-
 namespace eo
 {
+    /**
+     * @ingroup Parallel
+     * @defgroup MPI Message Passing Interface
+     * @brief See namespace eo::mpi to have all explanations about this module.
+     * @{
+     */
+
+    /**
+     * @brief MPI parallelization helpers for EO.
+     *
+     * This namespace contains parallelization functions which help to parallelize computations in EO. It is based on a
+     * generic algorithm, which is then customized with functors, corresponding to the algorithm main steps. These
+     * computations are centralized, i.e there is one central host whose role is to handle the steps of the algorithm ;
+     * we call it the "master". The other hosts just have to perform a "dummy" computation, which may be any kind of
+     * processing ; we call them, the "slaves", or less pejoratively, the "workers". Workers can communicate to each
+     * other, but they receive their orders from the Master and send him back some results. A worker can also be the
+     * master of a different parallelization process, as soon as it is a part of its work. Machines of the network, also
+     * called hosts, are identified by an unique number: their rank. At any time during the execution of the program,
+     * all the hosts know the total number of hosts.
+     *
+     * A parallelized Job is a set of tasks which are independant (i.e can be executed in random order without
+     * modifiying the result) and take a data input and compute a data output to be sent to the Master. The data can be
+     * of any type, however they have to be serialized to be sent over a network. It is sufficient that they can be
+     * serialized through boost.
+     *
+     * @todo For serialization purposes, don't depend upon boost. It would be easy to use only eoserial and send strings
+     * via mpi.
+     *
+     * The main steps of the algorithm are the following:
+     * - For the master:
+     *      - Have we done with the treatment we are doing ?
+     *      - If this is the case, we can quit.
+     *      - Otherwise, send an input data to some available worker.
+     *      - If there's no available worker, wait for a worker to be free.
+     *      - When receiving the response, handle it (eventually compute something on the output data, store it...).
+     *      - Go back to the first step.
+     * - For the worker, it is even easier:
+     *      - Wait for an order.
+     *      - If there's nothing to do, just quit.
+     *      - Otherwise, eventually retrieve data and do the work.
+     *      - Go back to the first step.
+     *
+     * There is of course some network adjustements to do and precisions to give there, but the main ideas are present. As the
+     * job is fully centralized, this is the master who tells the workers when to quit and when to work.
+     *
+     * The idea behind these MPI helpers is to be the most generic possible. If we look back at the steps of the
+     * algorithm, we found that the steps can be splitted into 2 parts: the first consists in the steps of any
+     * parallelization algorithm and the other consists in the specific parts of the algorithm. Ideally, the user should
+     * just have to implement the specific parts of the algorithm. We identified these parts to be:
+     * - For the master:
+     *      - What does mean to have terminated ? There are only two alternatives, in our binary world: either it is
+     *      terminated, or it is not. Hence we only need a function returning a boolean to know if we're done with the
+     *      computation : we'll call it IsFinished.
+     *      - What do we have to do when we send a task ? We don't have any a priori on the form of the sent data, or
+     *      the number of sent data. Moreover, as the tasks are all independant, we don't care of who will do the
+     *      computation, as soon as it's done. Knowing the rank of the worker will be sufficient to send him data. We
+     *      have identified another function, taking a single argument which is the rank of the worker: we'll call it
+     *      SendTask.
+     *      - What do we have to do when we receive a response from a worker ? One more time, we don't know which form
+     *      or structure can have the receive data, only the user can know. Also we let the user the charge to retrieve
+     *      the data ; he just has to know from who the master will retrieve the data. Here is another function, taking
+     *      a rank (the sender's one) as a function argument : this will be HandleResponse.
+     * - For the worker:
+     *      - What is the processing ? It can have any nature. We just need to be sure that a data is sent back to the
+     *      master, but it seems difficult to check that: it will be the role of the user to assert that data is sent by
+     *      the worker at the end of an execution. We've got identified our last function: ProcessTask.
+     *
+     * In term of implementation, it would be annoying to have only abstract classes with these 4 methods to implement. It
+     * would mean that if you want to alter just one of these 4 functions, you have to implement a new sub class, with a
+     * new constructor which could have the same signature. Besides, this fashion doesn't allow you to add dynamic
+     * functionalities, using the design pattern Decorator for instance, without implement a class for each type of
+     * decoration you want to add. For these reasons, we decided to transform function into functors ; the user can then
+     * wrap the existing, basic comportments into more sophisticated computations, whenever he wants, and without the
+     * notion of order. We retrieve here the power of extension given by the design pattern Decorator.
+     *
+     * Our 4 functors could have a big amount of data in common (see eoParallelApply to have an idea).
+     * So as to make it easy for the user to implement these 4 functors, we consider that these functors
+     * have to share a common data structure. This data structure is referenced (as a pointer) in the 4 functors, so the
+     * user doesn't need to pass a lot of parameters to each functor constructor.
+     *
+     * There are two kinds of jobs:
+     * - The job which are launched a fixed and well known amount of times, i.e both master and workers know how many
+     *   times they will be launched. They are "one shot jobs".
+     * - The job which are launched an unknown amount of times, for instance embedded in a while loop for which we don't
+     *   know the amount of repetitions (typically, eoEasyEA loop is a good example, as we don't know the continuator
+     *   condition). They are called "multi job".
+     * As the master tells the workers to quit, we have to differentiate these two kinds of jobs. When the job is of the
+     * kind "multi job", the workers would have to perform a while(true) loop so as to receive the orders ; but even if
+     * the master tells them to quit, they would begin another job and wait for another order, while the master would
+     * have quit: this would cause a deadlock and workers processes would be blocked, waiting for an order.
+     */
     namespace mpi
     {
+        /**
+         * @brief A timer which allows user to generate statistics about computation times.
+         */
         extern eoTimerStat timerStat;
 
+        /**
+         * @brief Tags used in MPI messages for framework communication
+         *
+         * These tags are used for framework communication and fits "channels", so as to differentiate when we're
+         * sending an order to a worker (Commands) or data (Messages). They are not reserved by the framework and can be
+         * used by the user, but he is not bound to.
+         *
+         * @ingroup MPI
+         */
         namespace Channel
         {
             const int Commands = 0;
             const int Messages = 1;
         }
 
+        /**
+         * @brief Simple orders used by the framework.
+         *
+         * These orders are sent by the master to the workers, to indicate to them if they should receive another task
+         * to do (Continue), if an one shot job is done (Finish) or if a multi job is done (Kill).
+         *
+         * @ingroup MPI
+         */
         namespace Message
         {
             const int Continue = 0;
-            const int Finish = 1; // TODO commentaire : différence entre finir une tâche et arrêter le worker à expliciter.
+            const int Finish = 1;
             const int Kill = 2;
         }
 
+        /**
+         * @brief If the job only has one master, the user can use this constant, so as not to worry with integer ids.
+         *
+         * @ingroup MPI
+         */
         const int DEFAULT_MASTER = 0;
 
+        /**
+         * @brief Base class for the 4 algorithm functors.
+         *
+         * This class can embed a data (JobData) and a wrapper, so as to make all the 4 functors wrappable.
+         * We can add a wrapper at initialization or at any time when executing the program.
+         *
+         * According to RAII, the boolean needDelete helps to know if we have to use the operator delete on the wrapper
+         * or not. Hence, if any functor is wrapped, user has just to put this boolean to true, to indicate to wrapper
+         * that it should call delete. This allows to mix wrapper initialized in the heap (with new) or in the stack.
+         *
+         * @param JobData a Data type, which can have any form. It can a struct, a single int, anything.
+         *
+         * @param Wrapped the type of the functor, which will be stored as a pointer under the name _wrapped.
+         * This allows to wrap directly the functor in functors of the same type
+         * here, instead of dealing with SharedDataFunction* that we would have to cast all the time.
+         * Doing also allows to handle the wrapped functor as the functor we're writing, when coding the wrappers,
+         * instead of doing some static_cast. For instance, if there are 2 functors subclasses, fA and fB, fA
+         * implementing doFa() and fB implementing doFb(), we could have the following code:
+         * @code
+         * struct fA_wrapper
+         * {
+         *   // some code
+         *   void doFa()
+         *   {
+         *      _wrapped->doFa();
+         *      std::cout << "I'm a fA wrapper!" << std::endl;
+         *      // if we didn't have the second template parameter, but a SharedDataFunction, we would have to do this:
+         *      static_cast<fA*>(_wrapped)->doFa();
+         *      // do other things (it's a wrapper)
+         *   }
+         * };
+         *
+         * struct fB_wrapper
+         * {
+         *  // some code
+         *  void doFb()
+         *  {
+         *      _wrapped->doFb(); // and not: static_cast<fB*>(_wrapped)->doFb();
+         *  }
+         * };
+         * @endcode
+         * This makes the code easier to write for the user.
+         *
+         * @ingroup MPI
+         */
         template< typename JobData, typename Wrapped >
         struct SharedDataFunction
         {
-            SharedDataFunction( Wrapped * w = 0 ) : _wrapped( w )
+            /**
+             * @brief Default constructor.
+             *
+             * The user is not bound to give a wrapped functor.
+             */
+            SharedDataFunction( Wrapped * w = 0 ) : _wrapped( w ), _needDelete( false )
             {
                 // empty
             }
 
+            /**
+             * @brief Destructor.
+             *
+             * Calls delete on the wrapped function, only if necessary.
+             */
+            virtual ~SharedDataFunction()
+            {
+                if( _wrapped && _wrapped->needDelete() )
+                {
+                    delete _wrapped;
+                }
+            }
+
+            /**
+             * @brief Setter for the wrapped function.
+             *
+             * It doesn't do anything on the current wrapped function, like deleting it.
+             */
             void wrapped( Wrapped * w )
             {
                 _wrapped = w;
             }
 
+            /**
+             * @brief Setter for the data present in the functor.
+             *
+             * Calls the setter on the functor and on the wrapped functors, in a Composite pattern fashion.
+             */
             void data( JobData* _d )
             {
                 d = _d;
@@ -59,11 +264,31 @@ namespace eo
                 }
             }
 
+            /**
+             * @brief Returns true if we need to use operator delete on this wrapper, false otherwise.
+             **/
+            bool needDelete() { return _needDelete; }
+            void needDelete( bool b ) { _needDelete = b; }
+
             protected:
             JobData* d;
-            Wrapped* _wrapped;
+            Wrapped* _wrapped; // Pointer and not a reference so as to be set at any time and to avoid affectation
+            bool _needDelete;
         };
 
+        /**
+         * @brief Functor (master side) used to send a task to the worker.
+         *
+         * The user doesn't have to know which worker will receive a task, so we just indicate to master the rank of the
+         * worker. The data used for computation have to be explicitly sent by the master to the worker, with indicated
+         * rank. Once this functor has been called, the worker is considered busy until it sends a return message to the
+         * master.
+         *
+         * This is a functor implementing void operator()(int), and also a shared data function, containing wrapper on its
+         * own type.
+         *
+         * @ingroup MPI
+         */
         template< typename JobData >
         struct SendTaskFunction : public eoUF<int, void>, public SharedDataFunction< JobData, SendTaskFunction<JobData> >
         {
@@ -74,9 +299,21 @@ namespace eo
                 // empty
             }
 
-            virtual ~SendTaskFunction() {}
+            virtual ~SendTaskFunction() {} // for inherited classes
         };
 
+        /**
+         * @brief Functor (master side) used to indicate what to do when receiving a response.
+         *
+         * The master calls this function as soon as it receives some data, in some channel. Thanks to MPI, we retrieve
+         * the rank of the data's sender. This functor is then called with this rank. There is no memoization of a link
+         * between sent data and rank, so the user has to implement it, if he needs it.
+         *
+         * This is a functor implementing void operator()(int), and also a shared data function, containing wrapper on
+         * its own type.
+         *
+         * @ingroup MPI
+         */
         template< typename JobData >
         struct HandleResponseFunction : public eoUF<int, void>, public SharedDataFunction< JobData, HandleResponseFunction<JobData> >
         {
@@ -87,9 +324,23 @@ namespace eo
                 // empty
             }
 
-            virtual ~HandleResponseFunction() {}
+            virtual ~HandleResponseFunction() {} // for inherited classes
         };
 
+        /**
+         * @brief Functor (worker side) implementing the processing to do.
+         *
+         * This is where the real computation happen.
+         * Whenever the master sends the command "Continue" to workers, which indicates the worker will receive a task,
+         * the worker calls this functor. The user has to explicitly retrieve the data, handle it and transmit it,
+         * processed, back to the master. If the worker does not send any data back to the master, the latter will
+         * consider the worker isn't done and a deadlock could occur.
+         *
+         * This is a functor implementing void operator()(), and also a shared data function, containing wrapper on its
+         * own type.
+         *
+         * @ingroup MPI
+         */
         template< typename JobData >
         struct ProcessTaskFunction : public eoF<void>, public SharedDataFunction< JobData, ProcessTaskFunction<JobData> >
         {
@@ -100,9 +351,21 @@ namespace eo
                 // empty
             }
 
-            virtual ~ProcessTaskFunction() {}
+            virtual ~ProcessTaskFunction() {} // for inherited classes
         };
 
+        /**
+         * @brief Functor (master side) indicating whether the job is done or not.
+         *
+         * The master loops on this functor to know when to stop. When this functor returns true, the master will wait
+         * for the last responses and properly stops the job. Whenever this functor returns false, the master will send
+         * tasks, until this functor returns true.
+         *
+         * This is a functor implementing bool operator()(), and also a shared function, containing wrapper on its own
+         * type.
+         *
+         * @ingroup MPI
+         */
         template< typename JobData >
         struct IsFinishedFunction : public eoF<bool>, public SharedDataFunction< JobData, IsFinishedFunction<JobData> >
         {
@@ -113,12 +376,33 @@ namespace eo
                 // empty
             }
 
-            virtual ~IsFinishedFunction() {}
+            virtual ~IsFinishedFunction() {} // for inherited classes
         };
 
+        /**
+         * @brief Contains all the required data and the functors to launch a job.
+         *
+         * Splitting the functors and data from the job in itself allows to use the same functors and data for multiples
+         * instances of the same job. You define your store once and can use it a lot of times during your program. If
+         * the store was included in the job, you'd have to give again all the functors and all the datas to each
+         * invokation of the job.
+         *
+         * Job store contains the 4 functors (pointers, so as to be able to wrap them ; references couldn't have
+         * permitted that) described above and the JobData used by all these functors. It contains
+         * also helpers to easily wrap the functors, getters and setters on all of them.
+         *
+         * The user has to implement data(), which is the getter for retrieving JobData. We don't have any idea of who
+         * owns the data, moreover it is impossible to initialize it in this generic JobStore, as we don't know its
+         * form. As a matter of fact, the user has to define this in the JobStore subclasses.
+         *
+         * @ingroup MPI
+         */
         template< typename JobData >
         struct JobStore
         {
+            /**
+             * @brief Default ctor with the 4 functors.
+             */
             JobStore(
                 SendTaskFunction<JobData>* stf,
                 HandleResponseFunction<JobData>* hrf,
@@ -130,21 +414,45 @@ namespace eo
                 // empty
             }
 
+            /**
+             * @brief Empty ctor, useful for not forcing users to call the other constructor.
+             *
+             * When using this constructor, the user have to care about the 4 functors pointers, otherwise null pointer
+             * segfaults have to be expected.
+             */
             JobStore()
             {
                 // empty
             }
 
+            /**
+             * @brief Default destructor.
+             *
+             * JobStore is the highest layer which calls needDelete on its functors.
+             */
+            ~JobStore()
+            {
+                if( _stf->needDelete() ) delete _stf;
+                if( _hrf->needDelete() ) delete _hrf;
+                if( _ptf->needDelete() ) delete _ptf;
+                if( _iff->needDelete() ) delete _iff;
+            }
+
+            // Getters
             SendTaskFunction<JobData> & sendTask() { return *_stf; }
             HandleResponseFunction<JobData> & handleResponse() { return *_hrf; }
             ProcessTaskFunction<JobData> & processTask() { return *_ptf; }
             IsFinishedFunction<JobData> & isFinished() { return *_iff; }
 
+            // Setters
             void sendTask( SendTaskFunction<JobData>* stf ) { _stf = stf; }
             void handleResponse( HandleResponseFunction<JobData>* hrf ) { _hrf = hrf; }
             void processTask( ProcessTaskFunction<JobData>* ptf ) { _ptf = ptf; }
             void isFinished( IsFinishedFunction<JobData>* iff ) { _iff = iff; }
 
+            /**
+             * @brief Helpers for wrapping send task functor.
+             */
             void wrapSendTask( SendTaskFunction<JobData>* stf )
             {
                 if( stf )
@@ -154,6 +462,9 @@ namespace eo
                 }
             }
 
+            /**
+             * @brief Helpers for wrapping handle response functor.
+             */
             void wrapHandleResponse( HandleResponseFunction<JobData>* hrf )
             {
                 if( hrf )
@@ -163,6 +474,9 @@ namespace eo
                 }
             }
 
+            /**
+             * @brief Helpers for wrapping process task functor.
+             */
             void wrapProcessTask( ProcessTaskFunction<JobData>* ptf )
             {
                 if( ptf )
@@ -172,6 +486,9 @@ namespace eo
                 }
             }
 
+            /**
+             * @brief Helpers for wrapping is finished functor.
+             */
             void wrapIsFinished( IsFinishedFunction<JobData>* iff )
             {
                 if( iff )
@@ -181,28 +498,65 @@ namespace eo
                 }
             }
 
-            // TODO commenter : laissé à la couche d'en dessous car impossible d'initialiser une donnée membre d'une classe mère depuis une classe fille.
             virtual JobData* data() = 0;
 
             protected:
 
-            // TODO commenter : Utiliser des pointeurs pour éviter d'écraser les fonctions wrappées
             SendTaskFunction< JobData >* _stf;
             HandleResponseFunction< JobData >* _hrf;
             ProcessTaskFunction< JobData >* _ptf;
             IsFinishedFunction< JobData >* _iff;
         };
 
+        /**
+         * @example t-mpi-wrapper.cpp
+         */
+
+        /**
+         * @brief Class implementing the centralized job algorithm.
+         *
+         * This class handles all the job algorithm. With its store and its assignment (scheduling) algorithm, it
+         * executes the general algorithm described above, adding some networking, so as to make the global process
+         * work. It initializes all the functors with the data, then launches the main loop, indicating to workers when
+         * they will have to work and when they will finish, by sending them a termination message (integer that can be
+         * customized). As the algorithm is centralized, it is also mandatory to indicate what is the MPI rank of the
+         * master process, hence the workers will know from who they should receive their commands.
+         *
+         * Any of the 3 master functors can launch exception, it will be catched and rethrown as a std::runtime_exception
+         * to the higher layers.
+         *
+         * @ingroup MPI
+         */
         template< class JobData >
         class Job
         {
             public:
+                /**
+                 * @brief Main constructor for Job.
+                 *
+                 * @param _algo The used assignment (scheduling) algorithm. It has to be initialized, with its maximum
+                 * possible number of workers (some workers referenced in this algorithm shouldn't be busy). See
+                 * AssignmentAlgorithm for more details.
+                 *
+                 * @param _masterRank The MPI rank of the master.
+                 * 
+                 * @param _workerStopCondition Number of the message which will cause the workers to terminate. It could
+                 * be one of the constants defined in eo::mpi::Commands, or any other integer. The user has to be sure
+                 * that a message containing this integer will be sent to each worker on the Commands channel, otherwise
+                 * deadlock will happen. Master sends Finish messages at the end of a simple job, but as a job can
+                 * happen multiples times (multi job), workers don't have to really finish on these messages but on
+                 * another message. This is here where you can configurate it. See also OneShotJob and MultiJob.
+                 *
+                 * @param store The JobStore containing functors and data for this job.
+                 */
                 Job( AssignmentAlgorithm& _algo,
                      int _masterRank,
+                     int _workerStopCondition,
                      JobStore<JobData> & store
                     ) :
                     assignmentAlgo( _algo ),
                     masterRank( _masterRank ),
+                    workerStopCondition( _workerStopCondition ),
                     comm( Node::comm() ),
                     // Functors
                     sendTask( store.sendTask() ),
@@ -220,6 +574,16 @@ namespace eo
 
             protected:
 
+                /**
+                 * @brief Finally block of the main algorithm
+                 *
+                 * Herb Sutter's trick for having a finally block, in a try/catch section: invoke a class at the
+                 * beginning of the try, its destructor will be called in every cases.
+                 *
+                 * This implements the end of the master algorithm:
+                 * - sends to all available workers that they are free,
+                 * - waits for last responses, handles them and sends termination messages to last workers.
+                 */
                 struct FinallyBlock
                 {
                     FinallyBlock(
@@ -279,6 +643,15 @@ namespace eo
                     bmpi::communicator & comm;
                 };
 
+                /**
+                 * @brief Master part of the job.
+                 *
+                 * Launches the parallelized job algorithm : while there is something to do (! IsFinished ), get a
+                 * worker who will be the assignee ; if no worker is available, wait for a response, handle it and reask
+                 * for an assignee. Then send the command and the task.
+                 * Once there is no more to do (IsFinished), indicate to all available workers that they're free, wait
+                 * for all the responses and send termination messages (see also FinallyBlock).
+                 */
                 void master( )
                 {
                     int totalWorkers = assignmentAlgo.availableWorkers();
@@ -324,6 +697,12 @@ namespace eo
                     }
                 }
 
+                /**
+                 * @brief Worker part of the algorithm.
+                 *
+                 * The algorithm is more much simpler: wait for an order; if it's termination message, leave. Otherwise,
+                 * prepare to work.
+                 */
                 void worker( )
                 {
                     int order;
@@ -339,7 +718,7 @@ namespace eo
 # ifndef NDEBUG
                         eo::log << "[W" << comm.rank() << "] Waiting for an order..." << std::endl;
 # endif
-                        if ( order == Message::Kill )
+                        if ( order == workerStopCondition )
                         {
 # ifndef NDEBUG
                             eo::log << "[W" << comm.rank() << "] Leaving worker task." << std::endl;
@@ -361,11 +740,18 @@ namespace eo
 
             public:
 
+                /**
+             * @brief Launches the job algorithm, according to the role of the host (roles are deduced from the
+                 * master rank indicated in the constructor).
+                 */
                 void run( )
                 {
                     ( _isMaster ) ? master( ) : worker( );
                 }
 
+                /**
+                 * @brief Returns true if the current host is the master, false otherwise.
+                 */
                 bool isMaster( )
                 {
                     return _isMaster;
@@ -375,6 +761,7 @@ namespace eo
 
                 AssignmentAlgorithm& assignmentAlgo;
                 int masterRank;
+                const int workerStopCondition;
                 bmpi::communicator& comm;
 
                 SendTaskFunction<JobData> & sendTask;
@@ -384,8 +771,62 @@ namespace eo
 
                 bool _isMaster;
         };
-    }
-}
 
+        /**
+         * @brief Job that will be launched only once.
+         *
+         * As explained in eo::mpi documentation, jobs can happen either a well known amount of times or an unknown
+         * amount of times. This class implements the general case when the job is launched a well known amount of
+         * times. The job will be terminated on both sides (master and worker) once the master would have said it.
+         *
+         * It uses the message Message::Finish as the termination message.
+         *
+         * @ingroup MPI
+         */
+        template< class JobData >
+        class OneShotJob : public Job< JobData >
+        {
+            public:
+                OneShotJob( AssignmentAlgorithm& algo,
+                            int masterRank,
+                            JobStore<JobData> & store )
+                    : Job<JobData>( algo, masterRank, Message::Finish, store )
+                {
+                    // empty
+                }
+        };
+
+        /**
+         * @brief Job that will be launched an unknown amount of times, in worker side.
+         *
+         * As explained in eo::mpi documentation, jobs can happen either a well known amount of times or an unknown
+         * amount of times. This class implements the general case when the job is launched an unknown amount of times, for
+         * instance in a while loop. The master will run many jobs (or the same job many times), but the workers will
+         * launch it only once.
+         *
+         * It uses the message Message::Kill as the termination message. This message can be launched with an EmptyJob,
+         * launched only by the master. If no Message::Kill is sent on the Channels::Commands, the worker will wait
+         * forever, which will cause a deadlock.
+         *
+         * @ingroup MPI
+         */
+        template< class JobData >
+        class MultiJob : public Job< JobData >
+        {
+            public:
+                MultiJob ( AssignmentAlgorithm& algo,
+                            int masterRank,
+                            JobStore<JobData> & store )
+                    : Job<JobData>( algo, masterRank, Message::Kill, store )
+                {
+                    // empty
+                }
+        };
+    }
+
+    /**
+     * @}
+     */
+}
 # endif // __EO_MPI_H__
 

eo/src/mpi/eoMpiAssignmentAlgorithm.h

@@ -1,28 +1,120 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __MPI_ASSIGNMENT_ALGORITHM_H__
 # define __MPI_ASSIGNMENT_ALGORITHM_H__
 
-# include <vector>
+# include <vector> // std::vector
 # include "eoMpiNode.h"
 
 namespace eo
 {
     namespace mpi
     {
+        /**
+         * @brief Constant indicating to use all the resting available workers, in assignment algorithms constructor
+         * using an interval.
+         *
+         * @ingroup MPI
+         */
         const int REST_OF_THE_WORLD = -1;
 
+        /**
+         * @brief Contains informations on the available workers and allows to find assignees for jobs.
+         *
+         * Available workers are workers who aren't processing anything. When they've received an order, workers switch
+         * from the state "available" to the state "busy", and the master has to wait for their response for considering
+         * them available again.
+         *
+         * @ingroup MPI
+         */
         struct AssignmentAlgorithm
         {
+            /**
+             * @brief Gets the rank of an available worker, so as to send it a task.
+             *
+             * @return The MPI rank of an available worker, or -1 if there is no available worker.
+             */
             virtual int get( ) = 0;
+
+            /**
+             * @brief Gets the number of total available workers.
+             *
+             * Before the first call, it is equal to the total number of present workers, as specified in the
+             * specific assignment algorithm constructor. It allows the Job class to know when all the responses have
+             * been received, by comparing this number to the total number of workers.
+             *
+             * @return Integer indicating how many workers are available.
+             */
             virtual int availableWorkers( ) = 0;
+
+            /**
+             * @brief Reinject the worker of indicated rank in the available state.
+             *
+             * @param wrkRank The MPI rank of the worker who has finished its job.
+             */
             virtual void confirm( int wrkRank ) = 0;
+
+            /**
+             * @brief Indicates who are the workers which do nothing.
+             *
+             * At the end of the algorithm, the master has to warn all the workers that it's done. All the workers mean,
+             * the workers which are currently processing data, and the other ones who could be waiting : the idles.
+             * This function indicates to the master which worker aren't doing anything.
+             *
+             * @return A std::vector containing all the MPI ranks of the idles workers.
+             */
             virtual std::vector<int> idles( ) = 0;
+
+            /**
+             * @brief Reinitializes the assignment algorithm with the right number of runs.
+             *
+             * In fact, this is only useful for static assignment algorithm, which has to be reinitialized every time
+             * it's used, in the case of a Multi Job. It's the user's responsability to call this function.
+             *
+             * @todo Not really clean. Find a better way to do it.
+             */
             virtual void reinit( int runs ) = 0;
         };
 
+        /**
+         * @brief Assignment (scheduling) algorithm which handles workers in a queue.
+         *
+         * With this assignment algorithm, workers are put in a queue and may be called an unlimited number of times.
+         * Whenever a worker returns, it is added to the queue, and it becomes available for the next call to get().
+         * The available workers are all located in the queue at any time, so the number of available workers is
+         * directly equal to the size of the queue.
+         *
+         * This kind of assignment is adapted for tasks whose execution time is stochastic or unknown, but without any
+         * warranty to be faster than other assignments.
+         *
+         * @ingroup MPI
+         */
         struct DynamicAssignmentAlgorithm : public AssignmentAlgorithm
         {
             public:
 
+                /**
+                 * @brief Uses all the hosts whose rank is higher to 1, inclusive, as workers.
+                 */
                 DynamicAssignmentAlgorithm( )
                 {
                     for(int i = 1; i < Node::comm().size(); ++i)
@@ -31,16 +123,33 @@ namespace eo
                     }
                 }
 
+                /**
+                 * @brief Uses the unique host with given rank as a worker.
+                 *
+                 * @param unique MPI rank of the unique worker.
+                 */
                 DynamicAssignmentAlgorithm( int unique )
                 {
                     availableWrk.push_back( unique );
                 }
 
+                /**
+                 * @brief Uses the workers whose ranks are present in the argument as workers.
+                 *
+                 * @param workers std::vector containing MPI ranks of workers.
+                 */
                 DynamicAssignmentAlgorithm( const std::vector<int> & workers )
                 {
                     availableWrk = workers;
                 }
 
+                /**
+                 * @brief Uses a range of ranks as workers.
+                 *
+                 * @param first The first worker to be included (inclusive)
+                 * @param last The last worker to be included (inclusive). If last == eo::mpi::REST_OF_THE_WORLD, all
+                 * hosts whose rank is higher than first are taken.
+                 */
                 DynamicAssignmentAlgorithm( int first, int last )
                 {
                     if( last == REST_OF_THE_WORLD )
@@ -90,14 +199,43 @@ namespace eo
                 std::vector< int > availableWrk;
         };
 
+        /**
+         * @brief Assignment algorithm which gives to each worker a precise number of tasks to do, in a round robin
+         * fashion.
+         *
+         * This scheduling algorithm attributes, at initialization or when calling reinit(), a fixed amount of runs to
+         * distribute to the workers. The amount of runs is then equally distributed between all workers ; if total
+         * number of runs is not a direct multiple of workers number, then remainding unaffected runs are distributed to
+         * workers from the first to the last, in a round-robin fashion.
+         *
+         * This scheduling should be used when the amount of runs can be computed or is fixed and when we guess that the
+         * duration of processing task will be the same for each run. There is no warranty that this algorithm is more
+         * or less efficient that another one. When having a doubt, use DynamicAssignmentAlgorithm.
+         *
+         * @ingroup MPI
+         */
         struct StaticAssignmentAlgorithm : public AssignmentAlgorithm
         {
             public:
+                /**
+                 * @brief Uses a given precise set of workers.
+                 *
+                 * @param workers std::vector of MPI ranks of workers which will be used.
+                 * @param runs Fixed amount of runs, strictly positive.
+                 */
                 StaticAssignmentAlgorithm( std::vector<int>& workers, int runs )
                 {
                     init( workers, runs );
                 }
 
+                /**
+                 * @brief Uses a range of workers.
+                 *
+                 * @param first The first MPI rank of worker to use
+                 * @param last The last MPI rank of worker to use. If it's equal to REST_OF_THE_WORLD, then all the
+                 * workers from the first one are taken as workers.
+                 * @param runs Fixed amount of runs, strictly positive.
+                 */
                 StaticAssignmentAlgorithm( int first, int last, int runs )
                 {
                     std::vector<int> workers;
@@ -114,16 +252,29 @@ namespace eo
                     init( workers, runs );
                 }
 
-                StaticAssignmentAlgorithm( int runs )
+                /**
+                 * @brief Uses all the hosts whose rank is higher than 1 (inclusive) as workers.
+                 *
+                 * @param runs Fixed amount of runs, strictly positive. If it's not set, you'll have to call reinit()
+                 * later.
+                 */
+                StaticAssignmentAlgorithm( int runs = 0 )
                 {
                     std::vector<int> workers;
                     for(int i = 1; i < Node::comm().size(); ++i)
                     {
                         workers.push_back( i );
                     }
+
                     init( workers, runs );
                 }
 
+                /**
+                 * @brief Uses an unique host as worker.
+                 *
+                 * @param unique The MPI rank of the host which will be the worker.
+                 * @param runs Fixed amount of runs, strictly positive.
+                 */
                 StaticAssignmentAlgorithm( int unique, int runs )
                 {
                     std::vector<int> workers;
@@ -132,12 +283,31 @@ namespace eo
                 }
 
             private:
+                /**
+                 * @brief Initializes the static scheduling.
+                 *
+                 * Gives to each worker an equal attribution number, equal to runs / workers.size(), eventually plus one
+                 * if number of workers is not a divisor of runs.
+                 *
+                 * @param workers Vector of hosts' ranks
+                 * @param runs Fixed amount of runs, strictly positive.
+                 */
                 void init( std::vector<int> & workers, int runs )
                 {
                     unsigned int nbWorkers = workers.size();
                     freeWorkers = nbWorkers;
-                    attributions.reserve( nbWorkers );
+
+                    busy.clear();
                     busy.resize( nbWorkers, false );
+                    realRank = workers;
+
+                    if( runs <= 0 )
+                    {
+                        return;
+                    }
+
+                    attributions.clear();
+                    attributions.reserve( nbWorkers );
 
                     // Let be the euclidean division of runs by nbWorkers :
                     // runs == q * nbWorkers + r, 0 <= r < nbWorkers
@@ -147,8 +317,6 @@ namespace eo
                     // r requests to workers, in ascending order
                     unsigned int diff = runs - (runs / nbWorkers) * nbWorkers;
                     for (unsigned int i = 0; i < diff; ++attributions[i++]);
-
-                    realRank = workers;
                 }
 
             public:

eo/src/mpi/eoMpiNode.h

@@ -1,3 +1,24 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __MPI_NODE_H__
 # define __MPI_NODE_H__
 
@@ -8,15 +29,34 @@ namespace eo
 {
     namespace mpi
     {
+        /**
+         * @brief Global object used to reach boost::mpi::communicator everywhere.
+         *
+         * boost::mpi::communicator is the main object used to send and receive messages between the different hosts of
+         * a MPI algorithm.
+         *
+         * @ingroup MPI
+         */
         class Node
         {
             public:
 
+                /**
+                 * @brief Initializes the MPI environment with argc and argv.
+                 *
+                 * Should be called at the beginning of every parallel program.
+                 *
+                 * @param argc Main's argc
+                 * @param argv Main's argv
+                 */
                 static void init( int argc, char** argv )
                 {
                     static bmpi::environment env( argc, argv );
                 }
 
+                /**
+                 * @brief Returns the global boost::mpi::communicator
+                 */
                 static bmpi::communicator& comm()
                 {
                     return _comm;

eo/src/mpi/eoMultiParallelApply.h

@@ -1,59 +0,0 @@
-# ifndef __EO_MULTI_PARALLEL_APPLY_H__
-# define __EO_MULTI_PARALLEL_APPLY_H__
-
-# include "eoParallelApply.h"
-
-namespace eo
-{
-    namespace mpi
-    {
-        template< class EOT >
-        class ProcessTaskParallelEval : public ProcessTaskParallelApply<EOT>
-        {
-            public:
-
-            using ProcessTaskParallelApply<EOT>::_wrapped;
-            using ProcessTaskParallelApply<EOT>::d;
-
-            void operator()()
-            {
-                int order = Message::Continue;
-                while( order != Message::Finish )
-                {
-                    _wrapped->operator()();
-                    d->comm.recv( d->masterRank, Channel::Commands, order );
-                }
-            }
-
-            ~ProcessTaskParallelEval()
-            {
-                delete _wrapped;
-            }
-        };
-
-        template< class EOT >
-        struct ParallelEvalStore : public ParallelApplyStore< EOT >
-        {
-            using ParallelApplyStore<EOT>::wrapProcessTask;
-
-            ParallelEvalStore(
-                    eoUF<EOT&, void> & _proc,
-                    int _masterRank,
-                    // long _maxTime = 0,
-                    int _packetSize = 1
-                   ) :
-                ParallelApplyStore< EOT >( _proc, *( new std::vector<EOT> ), _masterRank, _packetSize )
-                // FIXME memory leak because of vector ==> use const correctness
-            {
-                wrapProcessTask( new ProcessTaskParallelEval<EOT> );
-            }
-
-            void data( std::vector<EOT>& _pop )
-            {
-                ParallelApplyStore<EOT>::_data.init( _pop );
-            }
-        };
-    }
-}
-# endif // __EO_PARALLEL_APPLY_H__
-

eo/src/mpi/eoParallelApply.h

@@ -1,40 +1,119 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EO_PARALLEL_APPLY_H__
 # define __EO_PARALLEL_APPLY_H__
 
 # include "eoMpi.h"
 
-# include <eoFunctor.h>
+# include <eoFunctor.h> // eoUF
-# include <vector>
+# include <vector> // std::vector population
+
+/**
+ * @file eoParallelApply.h
+ *
+ * @brief Applies a functor with single parameter to elements of a table, in a parallel fashion.
+ *
+ * This file contains all the required classes to do a parallel apply of a table, in a parallel fashion. This can be
+ * very useful when applying the function can be made without any dependances within the data. In EO, it occurs in
+ * particular during the evaluation: the number of individuals to evaluate can be really high, and the evaluation of one
+ * individual is independant from the evaluation of other individuals.
+ *
+ * Elements in the table are directly replaced, as the table is given by reference. No new table is made during the
+ * process.
+ *
+ * User can tune this job, indicating how many elements of the table should be sent and evaluated by a worker, at a
+ * time: this is called the "packet size", as individuals are groupped into a packet of individuals which are sent to
+ * the worker before evaluation. The problem of choosing the optimal packet size is beyond the purposes of this documentation 
+ * and deserves a theoritical study.
+ *
+ * This job is the parallel equivalent to the function apply<EOT>, defined in apply.h. It just applies the function to
+ * every element of a table. In Python or Javascript, it's the equivalent of the function Map.
+ */
 
 namespace eo
 {
     namespace mpi
     {
+        /**
+         * @brief Structure used to save assignment to a worker, i.e which slice of the table it has to process.
+         *
+         * This slice is defined by the index of the first evaluated argument and the number of processed elements.
+         */
         struct ParallelApplyAssignment
         {
             int index;
             int size;
         };
 
+        /**
+         * @brief Data useful for a parallel apply (map).
+         *
+         * A parallel apply needs at least the functor to apply to every element of the table, and the table itself,
+         * whereas it can be set later with the function init(). Master rank is also needed, to send it informations and
+         * receive informations from it, inside the functors (the job knows these values, but the functors don't). The
+         * size of a packet can be tuned here.
+         *
+         * Internal attributes contain:
+         * - (useful for master) the index of the next element to be evaluated.
+         * - (useful for master) a map containing links between MPI ranks and slices of the table which the worker with
+         *   this rank has evaluated. Without this map, when receiving results from a worker, the master couldn't be
+         *   able to replace the right elements in the table.
+         *
+         * @ingroup MPI
+         */
         template<class EOT>
         struct ParallelApplyData
         {
+            /**
+             * @brief Ctor for Parallel apply (map) data.
+             *
+             * @param _proc The functor to apply on each element in the table
+             * @param _masterRank The MPI rank of the master
+             * @param _packetSize The number of elements on which the function will be applied by the worker, at a time.
+             * @param _pop The table to apply. If this value is NULL, user will have to call init() before launching the
+             * job.
+             */
             ParallelApplyData(
                     eoUF<EOT&, void> & _proc,
-                    std::vector<EOT> & _pop,
                     int _masterRank,
-                    // long _maxTime = 0,
+                    int _packetSize,
-                    int _packetSize
+                    std::vector<EOT> * _pop = 0
                    ) :
-                _data( &_pop ), func( _proc ), index( 0 ), size( _pop.size() ), packetSize( _packetSize ), masterRank( _masterRank ), comm( Node::comm() )
+                _data( _pop ), func( _proc ), index( 0 ), packetSize( _packetSize ), masterRank( _masterRank ), comm( Node::comm() )
             {
                 if ( _packetSize <= 0 )
                 {
                     throw std::runtime_error("Packet size should not be negative.");
                 }
-                tempArray = new EOT[ _packetSize ];
+
+                if( _pop )
+                {
+                    size = _pop->size();
+                }
             }
 
+            /**
+             * @brief Reinitializes the data for a new table to evaluate.
+             */
             void init( std::vector<EOT>& _pop )
             {
                 index = 0;
@@ -43,28 +122,33 @@ namespace eo
                 assignedTasks.clear();
             }
 
-            ~ParallelApplyData()
-            {
-                delete [] tempArray;
-            }
-
             std::vector<EOT>& data()
             {
                 return *_data;
             }
 
+            // All elements are public since functors will often use them.
             std::vector<EOT> * _data;
             eoUF<EOT&, void> & func;
             int index;
             int size;
-            std::map< int /* worker rank */, ParallelApplyAssignment /* min indexes in vector */> assignedTasks;
+            std::map< int /* worker rank */, ParallelApplyAssignment /* last assignment */> assignedTasks;
             int packetSize;
-            EOT* tempArray;
+            std::vector<EOT> tempArray;
 
             int masterRank;
             bmpi::communicator& comm;
         };
 
+        /**
+         * @brief Send task functor implementation for the parallel apply (map) job.
+         *
+         * Master side: Sends a slice of the table to evaluate to the worker.
+         *
+         * Implementation details:
+         * Finds the next slice of data to send to the worker, sends first the size and then the data, and memorizes
+         * that this slice has been distributed to the worker, then updates the next position of element to evaluate.
+         */
         template< class EOT >
         class SendTaskParallelApply : public SendTaskFunction< ParallelApplyData<EOT> >
         {
@@ -101,6 +185,11 @@ namespace eo
             }
         };
 
+        /**
+         * @brief Handle response functor implementation for the parallel apply (map) job.
+         *
+         * Master side: Replaces the slice of data attributed to the worker in the table.
+         */
         template< class EOT >
         class HandleResponseParallelApply : public HandleResponseFunction< ParallelApplyData<EOT> >
         {
@@ -118,6 +207,14 @@ namespace eo
             }
         };
 
+        /**
+         * @brief Process task functor implementation for the parallel apply (map) job.
+         *
+         * Worker side: apply the function to the given slice of data.
+         *
+         * Implementation details: retrieves the number of elements to evaluate, retrieves them, applies the function
+         * and then returns the results.
+         */
         template< class EOT >
         class ProcessTaskParallelApply : public ProcessTaskFunction< ParallelApplyData<EOT> >
         {
@@ -134,17 +231,24 @@ namespace eo
                 int recvSize;
 
                 d->comm.recv( d->masterRank, 1, recvSize );
-                d->comm.recv( d->masterRank, 1, d->tempArray, recvSize );
+                d->tempArray.resize( recvSize );
+                d->comm.recv( d->masterRank, 1, & d->tempArray[0] , recvSize );
                 timerStat.start("worker_processes");
                 for( int i = 0; i < recvSize ; ++i )
                 {
                     d->func( d->tempArray[ i ] );
                 }
                 timerStat.stop("worker_processes");
-                d->comm.send( d->masterRank, 1, d->tempArray, recvSize );
+                d->comm.send( d->masterRank, 1, & d->tempArray[0], recvSize );
             }
         };
 
+        /**
+         * @brief Is finished functor implementation for the parallel apply (map) job.
+         *
+         * Master side: returns true if and only if the whole table has been evaluated. The job is also terminated only
+         * when the whole table has been evaluated.
+         */
         template< class EOT >
         class IsFinishedParallelApply : public IsFinishedFunction< ParallelApplyData<EOT> >
         {
@@ -162,6 +266,14 @@ namespace eo
             }
         };
 
+        /**
+         * @brief Store containing all the datas and the functors for the parallel apply (map) job.
+         *
+         * User can tune functors when constructing the object. For each functor which is not given, a default one is
+         * generated.
+         *
+         * @ingroup MPI
+         */
         template< class EOT >
         struct ParallelApplyStore : public JobStore< ParallelApplyData<EOT> >
         {
@@ -170,20 +282,49 @@ namespace eo
             using JobStore< ParallelApplyData<EOT> >::_ptf;
             using JobStore< ParallelApplyData<EOT> >::_iff;
 
+            /**
+             * @brief Main constructor for the parallel apply (map) job.
+             *
+             * @param _proc The procedure to apply to each element of the table.
+             * @param _masterRank The rank of the master process.
+             * @param _packetSize The number of elements of the table to be evaluated at a time, by the worker.
+             * @param stpa Pointer to Send Task parallel apply functor descendant. If null, a default one is used.
+             * @param hrpa Pointer to Handle Response parallel apply functor descendant. If null, a default one is used.
+             * @param ptpa Pointer to Process Task parallel apply functor descendant. If null, a default one is used.
+             * @param ifpa Pointer to Is Finished parallel apply functor descendant. If null, a default one is used.
+             */
             ParallelApplyStore(
                     eoUF<EOT&, void> & _proc,
-                    std::vector<EOT>& _pop,
                     int _masterRank,
-                    // long _maxTime = 0,
                     int _packetSize = 1,
                     // JobStore functors
-                    SendTaskParallelApply<EOT> * stpa = new SendTaskParallelApply<EOT>,
+                    SendTaskParallelApply<EOT> * stpa = 0,
-                    HandleResponseParallelApply<EOT>* hrpa = new HandleResponseParallelApply<EOT>,
+                    HandleResponseParallelApply<EOT>* hrpa = 0,
-                    ProcessTaskParallelApply<EOT>* ptpa = new ProcessTaskParallelApply<EOT>,
+                    ProcessTaskParallelApply<EOT>* ptpa = 0,
-                    IsFinishedParallelApply<EOT>* ifpa = new IsFinishedParallelApply<EOT>
+                    IsFinishedParallelApply<EOT>* ifpa = 0
                    ) :
-                _data( _proc, _pop, _masterRank, _packetSize )
+                _data( _proc, _masterRank, _packetSize )
             {
+                if( stpa == 0 ) {
+                    stpa = new SendTaskParallelApply<EOT>;
+                    stpa->needDelete( true );
+                }
+
+                if( hrpa == 0 ) {
+                    hrpa = new HandleResponseParallelApply<EOT>;
+                    hrpa->needDelete( true );
+                }
+
+                if( ptpa == 0 ) {
+                    ptpa = new ProcessTaskParallelApply<EOT>;
+                    ptpa->needDelete( true );
+                }
+
+                if( ifpa == 0 ) {
+                    ifpa = new IsFinishedParallelApply<EOT>;
+                    ifpa->needDelete( true );
+                }
+
                 _stf = stpa;
                 _hrf = hrpa;
                 _ptf = ptpa;
@@ -192,22 +333,35 @@ namespace eo
 
             ParallelApplyData<EOT>* data() { return &_data; }
 
-            virtual ~ParallelApplyStore()
+            /**
+             * @brief Reinits the store with a new table to evaluate.
+             *
+             * @param _pop The table of elements to be evaluated.
+             */
+            void data( std::vector<EOT>& _pop )
+            {
+                _data.init( _pop );
+            }
+
+            virtual ~ParallelApplyStore() // for inheritance purposes only
             {
-                delete _stf;
-                delete _hrf;
-                delete _ptf;
-                delete _iff;
             }
 
             protected:
             ParallelApplyData<EOT> _data;
         };
 
-        // TODO commentaire : impossible de faire un typedef sur un template sans passer
+        /**
-        // par un traits => complique la tâche de l'utilisateur pour rien.
+         * @brief Parallel apply job. Present for convenience only.
+         *
+         * A typedef wouldn't have been working, as typedef on templates don't work in C++. Traits would be a
+         * disgraceful overload for the user.
+         *
+         * @ingroup MPI
+         * @see eoParallelApply.h
+         */
         template< typename EOT >
-        class ParallelApply : public Job< ParallelApplyData<EOT> >
+        class ParallelApply : public MultiJob< ParallelApplyData<EOT> >
         {
             public:
 
@@ -216,11 +370,16 @@ namespace eo
                     int _masterRank,
                     ParallelApplyStore<EOT> & store
                     ) :
-                Job< ParallelApplyData<EOT> >( algo, _masterRank, store )
+                MultiJob< ParallelApplyData<EOT> >( algo, _masterRank, store )
             {
                 // empty
             }
         };
+
+        /**
+         * @example t-mpi-parallelApply.cpp
+         * @example t-mpi-multipleRoles.cpp
+         */
     }
 }
 # endif // __EO_PARALLEL_APPLY_H__

eo/src/mpi/eoTerminateJob.h

@@ -1,3 +1,24 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EO_TERMINATE_H__
 # define __EO_TERMINATE_H__
 
@@ -7,6 +28,14 @@ namespace eo
 {
     namespace mpi
     {
+        /**
+         * @ingroup MPI
+         * @{
+         */
+
+        /**
+         * @brief Send task functor which does nothing.
+         */
         struct DummySendTaskFunction : public SendTaskFunction<void>
         {
             void operator()( int _ )
@@ -15,6 +44,9 @@ namespace eo
             }
         };
 
+        /**
+         * @brief Handle response functor which does nothing.
+         */
         struct DummyHandleResponseFunction : public HandleResponseFunction<void>
         {
             void operator()( int _ )
@@ -23,6 +55,9 @@ namespace eo
             }
         };
 
+        /**
+         * @brief Process task functor which does nothing.
+         */
         struct DummyProcessTaskFunction : public ProcessTaskFunction<void>
         {
             void operator()()
@@ -31,6 +66,9 @@ namespace eo
             }
         };
 
+        /**
+         * @brief Is finished functor which returns true everytime.
+         */
         struct DummyIsFinishedFunction : public IsFinishedFunction<void>
         {
             bool operator()()
@@ -39,6 +77,9 @@ namespace eo
             }
         };
 
+        /**
+         * @brief Job store containing all dummy functors and containing no data.
+         */
         struct DummyJobStore : public JobStore<void>
         {
             using JobStore<void>::_stf;
@@ -49,27 +90,32 @@ namespace eo
             DummyJobStore()
             {
                 _stf = new DummySendTaskFunction;
+                _stf->needDelete( true );
                 _hrf = new DummyHandleResponseFunction;
+                _hrf->needDelete( true );
                 _ptf = new DummyProcessTaskFunction;
+                _ptf->needDelete( true );
                 _iff = new DummyIsFinishedFunction;
-            }
+                _iff->needDelete( true );
-
-            ~DummyJobStore()
-            {
-                delete _stf;
-                delete _hrf;
-                delete _ptf;
-                delete _iff;
             }
 
             void* data() { return 0; }
         };
 
-        struct EmptyJob : public Job<void>
+        /**
+         * @brief Job to run after a Multi Job, so as to indicate that every workers should terminate.
+         */
+        struct EmptyJob : public OneShotJob<void>
         {
+            /**
+             * @brief Main EmptyJob ctor
+             *
+             * @param algo Assignment (scheduling) algorithm used.
+             * @param masterRank The rank of the master process.
+             */
             EmptyJob( AssignmentAlgorithm& algo, int masterRank ) :
-                Job<void>( algo, masterRank, *(new DummyJobStore) )
+                OneShotJob<void>( algo, masterRank, *(new DummyJobStore) )
-                // FIXME memory leak => will be corrected by using const correctness
+                // FIXME memory leak, meaningless but present
             {
                 // empty
             }
@@ -84,36 +130,8 @@ namespace eo
             }
         };
 
-        /*
+        /**
-        class TerminateJob : public Job
+         * @}
-        {
-            public:
-                TerminateJob( AssignmentAlgorithm& algo, int _ )
-                    : Job( algo, _ )
-                {
-                    // empty
-                }
-
-                void sendTask( int wrkRank )
-                {
-                    // empty
-                }
-
-                void handleResponse( int wrkRank )
-                {
-                    // empty
-                }
-
-                void processTask( )
-                {
-                    // empty
-                }
-
-                bool isFinished()
-                {
-                    return true;
-                }
-        };
          */
     }
 }

eo/src/serial/Array.cpp

@@ -1,10 +1,30 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # include "Array.h"
 
 namespace eoserial
 {
-
+    std::ostream& Array::print( std::ostream& out ) const
-std::ostream& Array::print( std::ostream& out ) const
+    {
-{
         out << "[";
         bool first = true;
         for (ArrayChildren::const_iterator it = begin(),
@@ -22,10 +42,10 @@ std::ostream& Array::print( std::ostream& out ) const
         }
         out << "]\n";
         return out;
-}
+    }
 
-Array::~Array()
+    Array::~Array()
-{
+    {
         for (ArrayChildren::iterator it = begin(),
                 end = this->end();
                 it != end;
@@ -33,6 +53,6 @@ Array::~Array()
         {
             delete *it;
         }
-}
+    }
 
 } // namespace eoserial

eo/src/serial/Array.h

@@ -1,45 +1,66 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_ARRAY_H__
 # define __EOSERIAL_ARRAY_H__
 
 # include <vector>
-# include <iostream>
 
 # include "Entity.h"
 # include "Serializable.h"
-
 # include "Object.h"
-# include "String.h"
 
 namespace eoserial
 {
 
-// Forward declaration for below declarations.
+    // Forward declaration for below declarations.
-class Array;
+    class Array;
 
-/*
+    /*
      * Declarations of functions present in Utils.h
      * These are put here to avoid instead of including the file Utils.h, which would
      * cause a circular inclusion.
      */
-template< class T >
-void unpack( const Array & array, unsigned int index, T & value );
 
-void unpackObject( const Array & array, unsigned int index, Persistent & value );
+    template< class T >
+        void unpack( const Array & array, unsigned int index, T & value );
 
-template< class Container, template<class> class UnpackAlgorithm >
+    void unpackObject( const Array & array, unsigned int index, Persistent & value );
-void unpackArray( const Array & array, unsigned int index, Container & container );
 
-/**
+    template< class Container, template<class> class UnpackAlgorithm >
+        void unpackArray( const Array & array, unsigned int index, Container & container );
+
+    /**
      * @brief Represents a JSON array.
      *
      * Wrapper for an array, so as to be used as a JSON object.
+     *
+     * @ingroup Serialization
      */
-class Array : public eoserial::Entity, public std::vector< eoserial::Entity* >
+    class Array : public eoserial::Entity, public std::vector< eoserial::Entity* >
-{
+    {
-protected:
+        protected:
             typedef std::vector< eoserial::Entity* > ArrayChildren;
 
-public:
+        public:
             /**
              * @brief Adds the serializable object as a JSON object.
              * @param obj Object which implemnets JsonSerializable.
@@ -141,7 +162,7 @@ public:
                         algo( *this, i, array );
                     }
                 }
-};
+    };
 
 } // namespace eoserial
 

eo/src/serial/Entity.h

@@ -1,9 +1,43 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_ENTITY_H__
 # define __EOSERIAL_ENTITY_H__
 
-# include <iostream>
+# include <iostream> // ostream
-# include <sstream>
 
+
+/**
+ * @brief Contains all the necessary entities to serialize eo objects into JSON objects.
+ *
+ * Allows serialization from user objects into JSON objects, if they implement the interface
+ * eoserial::Serializable or eoserial::Persistent. The following user objects can be serialized:
+ * - primitive types (int, std::string, ...), in particular every type that can be written into a
+ *   std::stringstream.
+ * - objects which implement eoserial::Serializable.
+ * - array of serializable things (primitive or serializable objects).
+ *
+ * @ingroup Utilities
+ * @defgroup Serialization Serialization helpers
+**/
 namespace eoserial
 {
 
@@ -12,6 +46,8 @@ namespace eoserial
  *
  * This class represents a JSON entity, which can be JSON objects,
  * strings or arrays. It is the base class for the JSON hierarchy.
+ *
+ * @ingroup Serialization
  */
 class Entity
 {

eo/src/serial/Object.cpp

@@ -1,12 +1,32 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # include "Object.h"
 
 using namespace eoserial;
 
 namespace eoserial
 {
-
+    std::ostream& Object::print( std::ostream& out ) const
-std::ostream& Object::print( std::ostream& out ) const
+    {
-{
         out << '{';
         bool first = true;
         for(JsonValues::const_iterator it = begin(), end = this->end();
@@ -25,16 +45,16 @@ std::ostream& Object::print( std::ostream& out ) const
         }
         out << "}\n";
         return out;
-}
+    }
 
-Object::~Object()
+    Object::~Object()
-{
+    {
         for(JsonValues::iterator it = begin(), end = this->end();
                 it != end;
                 ++it)
         {
             delete it->second;
         }
-}
+    }
 
 } // namespace eoserial

eo/src/serial/Object.h

@@ -1,31 +1,52 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_OBJECT_H__
 # define __EOSERIAL_OBJECT_H__
 
 # include <map>
 # include <string>
-# include <sstream>
 
 # include "Entity.h"
 # include "Serializable.h"
 
 namespace eoserial
 {
-
+    /**
-/**
      * @brief JSON Object
      *
      * This class represents a JSON object, which is basically a dictionnary
      * of keys (strings) and values (JSON entities).
+     *
+     * @ingroup Serialization
      */
-class Object : public eoserial::Entity, public std::map< std::string, eoserial::Entity* >
+    class Object : public eoserial::Entity, public std::map< std::string, eoserial::Entity* >
-{
+    {
-public:
+        public:
             typedef std::map<std::string, eoserial::Entity*> JsonValues;
 
             /**
              * @brief Adds a pair into the JSON object.
              * @param key The key associated with the eoserial object
-     * @param eoserial The JSON object as created with framework.
+             * @param json The JSON object as created with framework.
              */
             void add( const std::string& key, eoserial::Entity* json )
             {
@@ -60,7 +81,7 @@ public:
              * @brief Prints the content of a JSON object into a stream.
              */
             virtual std::ostream& print( std::ostream& out ) const;
-};
+    };
 
 } // namespace eoserial
 # endif // __EOSERIAL_OBJECT_H__

eo/src/serial/Parser.cpp

@@ -1,7 +1,25 @@
-# include <map>
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # include <string>
-# include <sstream>
-# include <vector>
 
 # include "Parser.h"
 

eo/src/serial/Parser.h

@@ -1,3 +1,24 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_PARSER_H__
 # define __EOSERIAL_PARSER_H__
 
@@ -6,6 +27,8 @@
 # include "Object.h"
 
 /**
+ * @file Parser.h
+ *
  * This file contains a tiny JSON parser used in DAE. This parser just handles 
  * a subset of JSON grammar, with the following restrictions :
  * - all strings must be surrounded by double quotes.
@@ -26,6 +49,8 @@ namespace eoserial
  * This parser does just retrieve values and does NOT check the structure of
  * the input. This implies that if the input is not correct, the result is undefined
  * and can result to a failure on execution.
+ *
+ * @ingroup Serialization
  */
 class Parser
 {
@@ -59,7 +84,7 @@ class Parser
          *
          * @param str The string we're parsing.
          * @param pos The index of the current position in the string.
-         * @param eoserial The current JSON object for which we're adding a key-value pair.
+         * @param json The current JSON object for which we're adding a key-value pair.
          */
         static void parseLeft(const std::string & str, size_t & pos, eoserial::Object* json);
 

eo/src/serial/Serializable.h

@@ -1,35 +1,57 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_SERIALIZABLE_H__
 # define __EOSERIAL_SERIALIZABLE_H__
 
-# include <string>
-
 namespace eoserial
 {
+    class Object; // to avoid recursive inclusion with JsonObject
 
-class Object; // to avoid recursive inclusion with JsonObject
+    /**
-
-/**
      * @brief Interface showing that object can be written to a eoserial type
      * (currently JSON).
+     *
+     * @ingroup Serialization
      */
-class Printable
+    class Printable
-{
+    {
-public:
+        public:
             /**
              * @brief Serializes the object to JSON format.
              * @return A JSON object created with new.
              */
             virtual eoserial::Object* pack() const = 0;
-};
+    };
 
-/**
+    /**
      * @brief Interface showing that object can be eoserialized (written and read
      * from an input).
      *
      * Note : Persistent objects should have a default non-arguments constructor.
+     *
+     * @ingroup Serialization
      */
-class Persistent : public Printable
+    class Persistent : public Printable
-{
+    {
         public:
             /**
              * @brief Loads class fields from a JSON object.
@@ -37,7 +59,7 @@ class Persistent : public Printable
              * is automatically done.
              */
             virtual void unpack(const eoserial::Object* json) = 0;
-};
+    };
 
 } // namespace eoserial
 

eo/src/serial/String.cpp

@@ -1,3 +1,24 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # include "String.h"
 
 namespace eoserial

eo/src/serial/String.h

@@ -1,3 +1,24 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_STRING_H__
 # define __EOSERIAL_STRING_H__
 
@@ -9,14 +30,15 @@
 
 namespace eoserial
 {
-
+    /**
-/**
      * @brief JSON String
      *
      * Wrapper for string, so as to be used as a JSON object.
+     *
+     * @ingroup Serialization
      */
-class String : public eoserial::Entity, public std::string
+    class String : public eoserial::Entity, public std::string
-{
+    {
         public:
 
             /**
@@ -26,7 +48,7 @@ class String : public eoserial::Entity, public std::string
             String( const std::string& str ) : std::string( str ) {}
 
             /**
-         * @brief Ctor used only on parsing.
+             * @brief Ctor used only when parsing.
              */
             String( ) {}
 
@@ -46,9 +68,9 @@ class String : public eoserial::Entity, public std::string
             // Copy and reaffectation are forbidden
             explicit String( const String& _ );
             String& operator=( const String& _ );
-};
+    };
 
-/**
+    /**
      * @brief Casts a eoserial::String into a primitive value, or in a type which at
      * least overload operator>>.
      *
@@ -57,24 +79,24 @@ class String : public eoserial::Entity, public std::string
      * It's not necessary to specify the variable type, which can be infered by compiler when
      * invoking.
      */
-template<class T>
+    template<class T>
-inline void String::deserialize( T & value )
+        inline void String::deserialize( T & value )
-{
+        {
             std::stringstream ss;
             ss.precision(std::numeric_limits<double>::digits10 + 1);
             ss << *this;
             ss >> value;
-}
+        }
 
-/**
+    /**
      * @brief Specialization for strings, which don't need to be converted through
      * a stringstream.
      */
-template<>
+    template<>
-inline void String::deserialize( std::string & value )
+        inline void String::deserialize( std::string & value )
-{
+        {
             value = *this;
-}
+        }
 
 } // namespace eoserial
 

eo/src/serial/Utils.h

@@ -1,3 +1,24 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_UTILS_H__
 # define __EOSERIAL_UTILS_H__
 
@@ -7,7 +28,7 @@
 
 namespace eoserial
 {
-    /*****************************
+    /* ***************************
      * DESERIALIZATION FUNCTIONS *
      *****************************
     These functions are useful for casting eoserial::objects into simple, primitive
@@ -54,7 +75,7 @@ namespace eoserial
         static_cast<Array*>( array[ index ] )->deserialize< Container, UnpackAlgorithm >( container );
     }
 
-    /*******************************
+    /* *****************************
      *** SERIALIZATION FUNCTIONS ***
      *******************************
      These functions are useful for casting classic objects and 

eo/src/serial/eoSerial.h

@@ -1,3 +1,24 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
 # ifndef __EOSERIAL_HEADERS__
 # define __EOSERIAL_HEADERS__
 

eo/src/utils/eoParallel.cpp

@@ -21,7 +21,7 @@
 Contact: http://eodev.sourceforge.net
 
 Authors:
-Caner Candan <caner.candan@thalesgroup.com>
+    Caner Candan <caner.candan@thalesgroup.com>
 
 */
 
@@ -34,11 +34,12 @@ Caner Candan <caner.candan@thalesgroup.com>
 
 eoParallel::eoParallel() :
     _isEnabled( false, "parallelize-loop", "Enable memory shared parallelization into evaluation's loops", '\0' ),
-    _isDynamic( false, "parallelize-dynamic", "Enable dynamic memory shared parallelization", '\0' ),
+    _isDynamic( true, "parallelize-dynamic", "Enable dynamic memory shared parallelization", '\0' ),
     _prefix( "results", "parallelize-prefix", "Here's the prefix filename where the results are going to be stored", '\0' ),
     _nthreads( 0, "parallelize-nthreads", "Define the number of threads you want to use, nthreads = 0 means you want to use all threads available", '\0' ),
     _enableResults( false, "parallelize-enable-results", "Enable the generation of results", '\0' ),
     _doMeasure( false, "parallelize-do-measure", "Do some measures during execution", '\0' ),
+    _packetSize( 1U, "parallelize-packet-size", "Number of elements which should be sent in a single message during a parallel evaluation based on message passing.", '\0'),
     _t_start(0)
 {
 }
@@ -92,6 +93,7 @@ void eoParallel::_createParameters( eoParser& parser )
     parser.processParam( _nthreads, section );
     parser.processParam( _enableResults, section );
     parser.processParam( _doMeasure, section );
+    parser.processParam( _packetSize, section );
 }
 
 void make_parallel(eoParser& parser)

eo/src/utils/eoParallel.h

@@ -20,8 +20,7 @@
 Contact: http://eodev.sourceforge.net
 
 Authors:
-Caner Candan <caner.candan@thalesgroup.com>
+    Caner Candan <caner.candan@thalesgroup.com>
-
 */
 
 /** @defgroup Parallel Parallel
@@ -54,6 +53,7 @@ public:
     std::string prefix() const;
 
     inline unsigned int nthreads() const { return _nthreads.value(); }
+    inline unsigned int packetSize() const { return _packetSize.value(); }
 
     inline bool enableResults() const { return _enableResults.value(); }
     inline bool doMeasure() const { return _doMeasure.value(); }
@@ -68,6 +68,7 @@ private:
     eoValueParam<bool> _isDynamic;
     eoValueParam<std::string> _prefix;
     eoValueParam<unsigned int> _nthreads;
+    eoValueParam<unsigned int> _packetSize;
     eoValueParam<bool> _enableResults;
     eoValueParam<bool> _doMeasure;
     double _t_start;

eo/src/utils/eoTimer.h

@@ -1,86 +1,219 @@
-# ifndef __TIMER_H__
+/*
-# define __TIMER_H__
+(c) Thales group, 2012
 
-# include <sys/time.h>
+    This library is free software; you can redistribute it and/or
-# include <sys/resource.h>
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
 
-# include <vector>
+    This library is distributed in the hope that it will be useful,
-# include <map>
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
+# ifndef __EO_TIMER_H__
+# define __EO_TIMER_H__
+
+# include <sys/time.h> // time()
+# include <sys/resource.h> // rusage()
+
+# include <vector> // std::vector
+# include <map> // std::map
+
+# include "utils/eoParallel.h" // eo::parallel
 
 # ifdef WITH_MPI
+// For serialization purposes
 # include <boost/serialization/access.hpp>
 # include <boost/serialization/vector.hpp>
 # include <boost/serialization/map.hpp>
 # endif
 
-// TODO TODOB commenter
+/**
+ * @brief Timer allowing to measure time between a start point and a stop point.
+ *
+ * This timer allows the user to measure user time, system time and wallclock time
+ * between two points. Basically, user time is time spent in developer code ; system
+ * time is time spent during the IO wait and system calls ; wallclock is the difference
+ * of time we could observe by measuring time with a watch.
+ *
+ * @ingroup Utilities
+ */
 class eoTimer
 {
     public:
 
+        /**
+         * @brief Default ctor. Begins all the timers.
+         */
         eoTimer()
-        {
-            restart();
-        }
-
-        void restart()
         {
             uuremainder = 0;
             usremainder = 0;
+            restart();
+        }
+
+        /**
+         * @brief Restarts all the timers and launch the measure.
+         */
+        void restart()
+        {
             wc_start = time(NULL);
             getrusage( RUSAGE_SELF, &_start );
         }
 
+        /**
+         * @brief Measures the user time spent since the last restart().
+         *
+         * If the number of elapsed seconds is zero, the spent milliseconds are
+         * added to a remainder. If the remainder exceeds one second, it is
+         * added to the number of elapsed seconds.
+         *
+         * @return Number of seconds elapsed in user space.
+         */
         long int usertime()
         {
             struct rusage _now;
             getrusage( RUSAGE_SELF, &_now );
+
             long int result = _now.ru_utime.tv_sec - _start.ru_utime.tv_sec;
-            if( _now.ru_utime.tv_sec == _start.ru_utime.tv_sec )
+            long int remainder = _now.ru_utime.tv_usec - _start.ru_utime.tv_usec;
+            if( remainder >= 0 )
             {
-                uuremainder += _now.ru_utime.tv_usec - _start.ru_utime.tv_usec;
+                uuremainder += remainder;
-                if( uuremainder > 1000000)
+            } else
             {
-                    ++result;
+                uuremainder += ( 1000000 - remainder );
-                    uuremainder = 0;
+                --result;
             }
+
+            if( uuremainder >= 1000000 )
+            {
+                uuremainder -= 1000000;
+                ++result;
             }
             return result;
         }
 
+        /**
+         * @brief Measures the system time spent since the last restart().
+         *
+         * If the number of elapsed seconds is zero, the spent milliseconds are
+         * added to a remainder. If the remainder exceeds one second, it is
+         * added to the number of elapsed seconds.
+         *
+         * @return Number of seconds elapsed in system (kernel) space.
+         */
         long int systime()
         {
             struct rusage _now;
             getrusage( RUSAGE_SELF, &_now );
+
             long int result = _now.ru_stime.tv_sec - _start.ru_stime.tv_sec;
-            if( _now.ru_stime.tv_sec == _start.ru_stime.tv_sec )
+            long int remainder = _now.ru_stime.tv_usec - _start.ru_stime.tv_usec;
+            if( remainder >= 0 )
             {
-                usremainder += _now.ru_stime.tv_usec - _start.ru_stime.tv_usec;
+                usremainder += remainder;
-                if( usremainder > 1000000)
+            } else
             {
-                    ++result;
+                usremainder += ( 1000000 - remainder );
-                    usremainder = 0;
+                --result;
             }
+
+            if( usremainder >= 1000000 )
+            {
+                usremainder -= 1000000;
+                ++result;
             }
             return result;
         }
 
+        /**
+         * @brief Measures the wallclock time spent since the last restart().
+         *
+         * @return Number of seconds elapsed, as a double.
+         */
         double wallclock()
         {
             return std::difftime( std::time(NULL) , wc_start );
         }
 
     protected:
+        // Structure used to measure user and system time.
         struct rusage _start;
+        // Remainder (in milliseconds) for user time.
         long int uuremainder;
+        // Remainder (in milliseconds) for system time.
         long int usremainder;
+        // Structure used to measure wallclock time.
         time_t wc_start;
 };
 
+/**
+ * @brief Registers a group of statistics, each statistic corresponding to user, system and wallclock times distribution.
+ *
+ * This class helps the user to measure time in different parts of an application. A name is associated to a statistic,
+ * on each call to start() and stop() for this name, a new number is added to the statistic, for each of the three
+ * measured times.
+ *
+ * The statistics are only registered if the option "--parallelized-do-measure" is set to true, which can be checked
+ * thanks to global object eo::parallel.
+ *
+ * This shows how the eoTimerStat can be used :
+ * @code
+ * eoTimerStat timerStat;
+ * timerStat.start("first_point");
+ * for( int i = 0; i < 1000; ++i )
+ * {
+ *   timerStat.start("single_computation");
+ *   single_computation( i );
+ *   timerStat.stop("single_computation");
+ * }
+ * // After this loop, timerStat contains a statistic of key "single_computation" which contains 1000 measures for
+ * // each type of time.
+ * timerStat.stop("first_point");
+ * // After this line, timerStat contains another statistic of key "first_point" which counted the duration of the
+ * // whole loop.
+ *
+ * int singleComputationUsertimeMean = 0;
+ * for( int i = 0; i < 1000; ++i )
+ * {
+ *      singleComputationUsertimeMean += timerStat.stats()["single_computation"].utime[i];
+ * }
+ * std::cout << "Mean of user time spent in single computation: " << singleComputationUsertimeMean / 1000. << std::endl;
+ * @endcode
+ *
+ * When using MPI, these statistics can be readily be serialized, so as to be sent over a network, for instance.
+ *
+ * Implementation details: this eoTimerStat is in fact a map of strings (key) / Stat (value). Stat is an internal
+ * structure directly defined in the class, which contains three vectors modeling the distributions of the different
+ * types of elapsed times. Another map of strings (key) / eoTimer (value) allows to effectively retrieve the different
+ * times. The struct Stat will be exposed to client, which will use its members ; however,
+ * the client doesn't have anything to do directly with the timer, that's why the two maps are splitted.
+ *
+ * @ingroup Utilities
+ */
 class eoTimerStat
 {
     public:
 
+        /**
+         * @brief Statistic related to a key (name).
+         *
+         * This structure is the value in the map saved in the eoTimerStat. It contains the statistic bound to a key,
+         * which are the user time distribution, the system time distribution and the wallclock time distribution, as
+         * std::vector s.
+         *
+         * It can readily be serialized with boost when compiling with mpi.
+         */
         struct Stat
         {
             std::vector<long int> utime;
@@ -91,7 +224,8 @@ class eoTimerStat
             friend class boost::serialization::access;
 
             /**
-             * Serializes the statistique in a boost archive (useful for boost::mpi)
+             * Serializes the single statistic in a boost archive (useful for boost::mpi).
+             * Just serializes the 3 vectors.
              */
             template <class Archive>
                 void serialize( Archive & ar, const unsigned int version )
@@ -107,7 +241,8 @@ class eoTimerStat
         friend class boost::serialization::access;
 
         /**
-             * Serializes the map of statistics in a boost archive (useful for boost::mpi)
+         * Serializes the timerStat object in a boost archive (useful for boost::mpi).
+         * Just serializes the map.
          */
         template <class Archive>
             void serialize( Archive & ar, const unsigned int version )
@@ -117,12 +252,35 @@ class eoTimerStat
             }
 # endif
 
+        /**
+         * @brief Starts a new measure for the given key.
+         *
+         * This is only performed if parallel.doMeasure() is true, which is equivalent to the fact that
+         * parser found "--parallel-do-measure=1" in command line args.
+         *
+         * @param key The key of the statistic.
+         */
         void start( const std::string & key )
+        {
+            if( eo::parallel.doMeasure() )
             {
                 _timers[ key ].restart();
             }
+        }
 
+        /**
+         * @brief Stops the mesure for the given key and saves the elapsed times.
+         *
+         * Must follow a call of start with the same key.
+         *
+         * This is only performed if parallel.doMeasure() is true, which is equivalent to the fact that
+         * parser found "--parallel-do-measure=1" in command line args.
+         *
+         * @param key The key of the statistic.
+         */
         void stop( const std::string& key )
+        {
+            if( eo::parallel.doMeasure() )
             {
                 Stat & s = _stats[ key ];
                 eoTimer & t = _timers[ key ];
@@ -130,14 +288,20 @@ class eoTimerStat
                 s.stime.push_back( t.systime() );
                 s.wtime.push_back( t.wallclock() );
             }
+        }
 
-        std::map< std::string, Stat > stats()
+        /**
+         * @brief Getter for the statistics map.
+         */
+        std::map< std::string, Stat >& stats()
         {
             return _stats;
         }
 
     protected:
+        // Statistics map: links a key (string) to a statistic.
         std::map< std::string, Stat > _stats;
+        // Timers map: links a key to its timer.
         std::map< std::string, eoTimer > _timers;
 };
 

eo/test/CMakeLists.txt

@@ -14,12 +14,23 @@ INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/contrib)
 INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/contrib/MGE)
 INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
 
+IF(WITH_MPI)
+    INCLUDE_DIRECTORIES(${BOOST_DIR}/include)
+    INCLUDE_DIRECTORIES(${MPI_DIR}/include)
+ENDIF()
+
 ######################################################################################
 ### 2) Specify where CMake can find the libraries
 ######################################################################################
 
 LINK_DIRECTORIES(${EO_BINARY_DIR}/lib)
 
+IF(WITH_MPI)
+    LINK_DIRECTORIES(${BOOST_DIR}/lib)
+    LINK_DIRECTORIES(${MPI_DIR}/lib)
+    SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx")
+ENDIF()
+
 ######################################################################################
 ### 3) Define your targets and link the librairies
 ######################################################################################
@@ -65,8 +76,8 @@ SET (TEST_LIST
   t-eoExtendedVelocity
   t-eoLogger
   t-eoIQRStat
-  t-eoParallel
+  #t-eoParallel
-  #t-openmp # does not work anymore since functions used in this test were removed from EO
+  #t-openmp
   #t-eoDualFitness
   t-eoParser
   )

eo/test/mpi/CMakeLists.txt

@@ -0,0 +1,55 @@
+###############################################################################
+##
+## CMakeLists file for eo/test/mpi
+##
+###############################################################################
+
+######################################################################################
+### 1) Include the sources
+######################################################################################
+
+MESSAGE("EO SOURCE DIR: ${EO_SOURCE_DIR}")
+MESSAGE("OMPI: ${MPI_DIR}")
+MESSAGE("BOOST: ${BOOST_DIR}")
+
+INCLUDE_DIRECTORIES(${MPI_DIR}/include)
+INCLUDE_DIRECTORIES(${BOOST_DIR}/include)
+INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/src)
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
+
+######################################################################################
+### 2) Specify where CMake can find the libraries
+######################################################################################
+
+LINK_DIRECTORIES(${EO_BINARY_DIR}/lib)
+LINK_DIRECTORIES(${MPI_DIR}/lib)
+LINK_DIRECTORIES(${BOOST_DIR}/lib)
+
+######################################################################################
+### 3) Define your targets and link the librairies
+######################################################################################
+
+SET (TEST_LIST
+    t-mpi-parallelApply
+    t-mpi-wrapper
+    t-mpi-multipleRoles
+    t-mpi-eval
+    )
+
+FOREACH (test ${TEST_LIST})
+  SET ("T_${test}_SOURCES" "${test}.cpp")
+ENDFOREACH (test)
+
+SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx")
+ADD_DEFINITIONS(-DWITH_MPI)
+
+IF(ENABLE_CMAKE_TESTING)
+    FOREACH (test ${TEST_LIST})
+    ADD_EXECUTABLE(${test} ${T_${test}_SOURCES})
+    ADD_TEST(${test} ${test})
+    TARGET_LINK_LIBRARIES(${test} boost_mpi boost_serialization eoutils eompi eoserial eo)
+    INSTALL(TARGETS ${test} RUNTIME DESTINATION share/eo/test COMPONENT test)
+    ENDFOREACH (test)
+ENDIF()
+
+######################################################################################

eo/test/mpi/multipleRoles.cpp

@@ -1,108 +0,0 @@
-# include <mpi/eoMpi.h>
-# include <mpi/eoParallelApply.h>
-
-# include <boost/serialization/vector.hpp>
-
-# include <iostream>
-
-# include <vector>
-using namespace std;
-
-using namespace eo::mpi;
-
-// Role map
-// 0 : general master
-// 1, 2 : worker of general job, master of subjob
-// 3 to 7 : workers of subjob
-
-struct SubWork: public eoUF< int&, void >
-{
-    void operator() ( int & x )
-    {
-        cout << "Subwork phase." << endl;
-        ++x;
-    }
-};
-
-void subtask( vector<int>& v, int rank )
-{
-    vector<int> workers;
-    workers.push_back( rank + 2 );
-    workers.push_back( rank + 4 );
-    DynamicAssignmentAlgorithm algo( workers );
-    SubWork sw;
-
-    ParallelApplyStore<int> store( sw, v, rank );
-    ParallelApply<int> job( algo, rank, store );
-    job.run();
-}
-
-struct Work: public eoUF< vector<int>&, void >
-{
-    void operator() ( vector<int>& v )
-    {
-        cout << "Work phase..." << endl;
-        subtask( v, Node::comm().rank() );
-        for( int i = 0; i < v.size(); ++i )
-        {
-            v[i] *= 2;
-        }
-    }
-};
-
-int main(int argc, char** argv)
-{
-    // eo::log << eo::setlevel( eo::debug );
-    Node::init( argc, argv );
-    vector<int> v;
-
-    v.push_back(1);
-    v.push_back(3);
-    v.push_back(3);
-    v.push_back(7);
-    v.push_back(42);
-
-    vector< vector<int> > metaV;
-    metaV.push_back( v );
-    metaV.push_back( v );
-
-    switch( Node::comm().rank() )
-    {
-        case 0:
-        case 1:
-        case 2:
-            {
-                Work w;
-                DynamicAssignmentAlgorithm algo( 1, 2 );
-                ParallelApplyStore< vector<int> > store( w, metaV, 0 );
-                ParallelApply< vector<int> > job( algo, 0, store );
-                job.run();
-                if( job.isMaster() )
-                {
-                    v = metaV[0];
-                    cout << "Results : " << endl;
-                    for(int i = 0; i < v.size(); ++i)
-                    {
-                        cout << v[i] << ' ';
-                    }
-                    cout << endl;
-                }
-            }
-            break;
-
-        default:
-            {
-                // all the other nodes are sub workers
-                int rank = Node::comm().rank();
-                if ( rank == 3 or rank == 5 )
-                {
-                    subtask( v, 1 );
-                } else {
-                    subtask( v, 2 );
-                }
-            }
-            break;
-    }
-
-    return 0;
-}

eo/test/mpi/t-mpi-eval.cpp

@@ -1,18 +1,39 @@
-//-----------------------------------------------------------------------------
+/*
-// t-eoMpiParallel.cpp
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
+
+/*
+ * This file shows an example of parallel evaluation of a population, when using an eoEasyEA algorithm.
+ * Moreover, we add a basic wrapper on the parallel evaluation, so as to show how to retrieve the best solutions.
+ */
 //-----------------------------------------------------------------------------
 
 #include <eo>
 #include <eoPopEvalFunc.h>
 
 #include <es/make_real.h>
-// #include <apply.h>
 #include "../real_value.h"
 
 #include <mpi/eoMpi.h>
 
-#include <mpi/eoTerminateJob.h>
-
 #include <boost/mpi.hpp>
 
 #include <vector>
@@ -82,6 +103,16 @@ class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial
 
 typedef eoRealSerializable EOT;
 
+/*
+ * Wrapper for HandleResponse: shows the best answer, as it is found.
+ *
+ * Finding the best solution is an associative operation (as it is based on a "min" function, which is associative too)
+ * and that's why we can perform it here. Indeed, the min element of 5 elements is the min element of the 3 first
+ * elements and the min element of the 2 last elements:
+ * min(1, 2, 3, 4, 5) = min( min(1, 2, 3), min(4, 5) )
+ *
+ * This is a reduction. See MapReduce example to have another examples of reduction.
+ */
 struct CatBestAnswers : public eo::mpi::HandleResponseParallelApply<EOT>
 {
     CatBestAnswers()
@@ -89,14 +120,15 @@ struct CatBestAnswers : public eo::mpi::HandleResponseParallelApply<EOT>
         best.fitness( 1000000000. );
     }
 
-    using eo::mpi::HandleResponseParallelApply<EOT>::_wrapped;
+    // if EOT were a template, we would have to do: (thank you C++ :)
-    using eo::mpi::HandleResponseParallelApply<EOT>::d;
+    // using eo::mpi::HandleResponseParallelApply<EOT>::_wrapped;
+    // using eo::mpi::HandleResponseParallelApply<EOT>::d;
 
     void operator()(int wrkRank)
     {
         int index = d->assignedTasks[wrkRank].index;
         int size = d->assignedTasks[wrkRank].size;
-        (*_wrapped)( wrkRank );
+        (*_wrapped)( wrkRank ); // call to the wrapped function HERE
         for(int i = index; i < index+size; ++i)
         {
             if( best.fitness() < d->data()[ i ].fitness() )
@@ -136,6 +168,9 @@ int main(int ac, char** av)
     eoEvalFuncPtr< EOT, double, const std::vector< double >& > mainEval( real_value );
     eoEvalFuncCounter< EOT > eval( mainEval );
 
+    // until this point, everything (but eo::mpi::Node::init) is exactly as in an sequential version.
+    // We then instanciate the parallel algorithm. The store is directly used by the eoParallelPopLoopEval, which
+    // internally uses parallel apply.
     int rank = eo::mpi::Node::comm().rank();
     eo::mpi::DynamicAssignmentAlgorithm assign;
     if( rank == eo::mpi::DEFAULT_MASTER )
@@ -144,10 +179,10 @@ int main(int ac, char** av)
 
         eo::log << "Size of population : " << popSize << std::endl;
 
-        eo::mpi::ParallelEvalStore< EOT > store( eval, eo::mpi::DEFAULT_MASTER );
+        eo::mpi::ParallelApplyStore< EOT > store( eval, eo::mpi::DEFAULT_MASTER );
         store.wrapHandleResponse( new CatBestAnswers );
 
-        eoParallelPopLoopEval< EOT > popEval( eval, assign, &store, eo::mpi::DEFAULT_MASTER, 3 );
+        eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, &store );
         eo::log << eo::quiet << "Before first evaluation." << std::endl;
         popEval( pop, pop );
         eo::log << eo::quiet << "After first evaluation." << std::endl;
@@ -159,8 +194,8 @@ int main(int ac, char** av)
         eo::log << eo::quiet << "DONE!" << std::endl;
     } else
     {
-        eoPop< EOT > pop( popSize, init );
+        eoPop< EOT > pop; // the population doesn't have to be initialized, as it is not used by workers.
-        eoParallelPopLoopEval< EOT > popEval( eval, assign, eo::mpi::DEFAULT_MASTER, 3 );
+        eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, eval );
         popEval( pop, pop );
     }
 

eo/test/mpi/t-mpi-multipleRoles.cpp

@@ -0,0 +1,169 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
+
+/*
+ * This file shows an example of how to make a hierarchy between nodes, when using a parallel apply. In this basic
+ * test, the master delegates the charge of finding workers to 2 "sub" masters, which then send part of the table to
+ * their workers.
+ *
+ * It's convenient to establish a role map, so as to clearly identify every role:
+ * - The node 0 is the general master, that delegates the job. It sends the table to the 2 submasters, and waits for the
+ *   results.
+ * - Nodes 1 and 2 are the worker of the first job: the delegates. They receive the elements of the table and
+ *   retransmit them to the subworkers. They play the roles of worker in the delegating job, and master in the plus one
+ *   job.
+ * - Following nodes (3 to 6) are workers of the plus one job. They do the real job. Nodes 3 and 5 are attached to
+ *   submaster 1, 4 and 6 to submaster 2.
+ *
+ * This test requires exactly 7 hosts. If the size is bigger, an exception will be thrown at the beginning.
+ **/
+
+# include <mpi/eoMpi.h>
+# include <mpi/eoParallelApply.h>
+# include <mpi/eoTerminateJob.h>
+
+# include <boost/serialization/vector.hpp>
+
+# include <iostream>
+
+# include <vector>
+using namespace std;
+
+using namespace eo::mpi;
+
+// The real job to execute, for the subworkers: add one to each element of a table.
+struct SubWork: public eoUF< int&, void >
+{
+    void operator() ( int & x )
+    {
+        cout << "Subwork phase." << endl;
+        ++x;
+    }
+};
+
+// Function called by both subworkers and delegates.
+// v is the vector to process, rank is the MPI rank of the sub master
+void subtask( vector<int>& v, int rank )
+{
+    // Attach workers according to nodes.
+    // Submaster with rank 1 will have ranks 3 and 5 as subworkers.
+    // Submaster with rank 2 will have ranks 4 and 6 as subworkers.
+    vector<int> workers;
+    workers.push_back( rank + 2 );
+    workers.push_back( rank + 4 );
+    DynamicAssignmentAlgorithm algo( workers );
+    SubWork sw;
+
+    // Launch the job!
+    ParallelApplyStore<int> store( sw, rank );
+    store.data( v );
+    ParallelApply<int> job( algo, rank, store );
+    job.run();
+    EmptyJob stop( algo, rank );
+}
+
+// Functor applied by submasters. Wait for the subworkers responses and then add some random processing (here, multiply
+// each result by two).
+// Note that this work receives a vector of integers as an entry, while subworkers task's operator receives a simple
+// integer.
+struct Work: public eoUF< vector<int>&, void >
+{
+    void operator() ( vector<int>& v )
+    {
+        cout << "Work phase..." << endl;
+        subtask( v, Node::comm().rank() );
+        for( int i = 0; i < v.size(); ++i )
+        {
+            v[i] *= 2;
+        }
+    }
+};
+
+int main(int argc, char** argv)
+{
+    // eo::log << eo::setlevel( eo::debug );
+    Node::init( argc, argv );
+    if( Node::comm().size() != 7 ) {
+        throw std::runtime_error("World size should be 7.");
+    }
+
+    vector<int> v;
+
+    v.push_back(1);
+    v.push_back(3);
+    v.push_back(3);
+    v.push_back(7);
+    v.push_back(42);
+
+    // As submasters' operator receives a vector<int> as an input, and ParallelApply takes a vector of
+    // operator's input as an input, we have to deal with a vector of vector of integers for the master task.
+    vector< vector<int> > metaV;
+    // Here, we send twice the same vector. We could also have splitted the first vector into two vectors, one
+    // containing the beginning and another one containing the end.
+    metaV.push_back( v );
+    metaV.push_back( v );
+
+    // Assigning roles is done by comparing MPI ranks.
+    switch( Node::comm().rank() )
+    {
+        // Nodes from 0 to 2 are implicated into the delegating task.
+        case 0:
+        case 1:
+        case 2:
+            {
+                Work w;
+                DynamicAssignmentAlgorithm algo( 1, 2 );
+                ParallelApplyStore< vector<int> > store( w, 0 );
+                store.data( metaV );
+                ParallelApply< vector<int> > job( algo, 0, store );
+                job.run();
+                if( job.isMaster() )
+                {
+                    EmptyJob stop( algo, 0 );
+                    v = metaV[0];
+                    cout << "Results : " << endl;
+                    for(int i = 0; i < v.size(); ++i)
+                    {
+                        cout << v[i] << ' ';
+                    }
+                    cout << endl;
+                }
+            }
+            break;
+
+        // Other nodes are implicated into the subwork task.
+        default:
+            {
+                // all the other nodes are sub workers
+                int rank = Node::comm().rank();
+                if ( rank == 3 or rank == 5 )
+                {
+                    subtask( v, 1 );
+                } else {
+                    subtask( v, 2 );
+                }
+            }
+            break;
+    }
+
+    return 0;
+}

eo/test/mpi/t-mpi-parallelApply.cpp

@@ -1,5 +1,44 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
+
+/*
+ * This file shows an example of use of parallel apply, in the following context: each element of a table is
+ * incremented... in a parallel fashion. While this operation is very easy to perform even on a single host, it's just
+ * an example for parallel apply use.
+ *
+ * Besides, this is also a test for assignment (scheduling) algorithms, in different cases. The test succeeds if and
+ * only if the program terminates without any segfault ; otherwise, there could be a deadlock which prevents the end or
+ * a segfault at any time.
+ *
+ * One important thing is to instanciate an EmptyJob after having launched a ParallelApplyJob, so as the workers to be
+ * aware that the job is done (as it's a MultiJob).
+ *
+ * This test needs at least 3 processes to be launched. Under this size, it will directly throw an exception, at the
+ * beginning;
+ */
+
 # include <mpi/eoMpi.h>
 # include <mpi/eoParallelApply.h>
+# include <mpi/eoTerminateJob.h>
 
 # include <iostream>
 
@@ -8,6 +47,9 @@ using namespace std;
 
 using namespace eo::mpi;
 
+/*
+ * The function to be called on each element of the table: just increment the value.
+ */
 struct plusOne : public eoUF< int&, void >
 {
     void operator() ( int & x )
@@ -16,22 +58,26 @@ struct plusOne : public eoUF< int&, void >
     }
 };
 
+/*
+ * Internal structure representating a test.
+ */
 struct Test
 {
-    AssignmentAlgorithm * assign;
+    AssignmentAlgorithm * assign;   // used assignment algorithm for this test.
-    string description;
+    string description;             // textual description of the test
-    int requiredNodesNumber; // nb : chosen nodes ranks must be sequential
+    int requiredNodesNumber;        // number of required nodes. NB : chosen nodes ranks must be sequential
 };
 
-// These tests require at least 3 processes to be launched.
 int main(int argc, char** argv)
 {
-    // eo::log << eo::setlevel( eo::debug );
+    // eo::log << eo::setlevel( eo::debug ); // if you like tty full of rainbows, decomment this line and comment the following one.
     eo::log << eo::setlevel( eo::quiet );
+
     bool launchOnlyOne = false ; // Set this to true if you wanna launch only the first test.
 
     Node::init( argc, argv );
 
+    // Initializes a vector with random values.
     srand( time(0) );
     vector<int> v;
     for( int i = 0; i < 1000; ++i )
@@ -39,18 +85,27 @@ int main(int argc, char** argv)
         v.push_back( rand() );
     }
 
+    // We need to be sure the values are correctly incremented between each test. So as to check this, we save the
+    // original vector into a variable originalV, and put an offset variable to 0. After each test, the offset is
+    // incremented and we can compare the returned value of each element to the value of each element in originalV +
+    // offset. If the two values are different, there has been a problem.
     int offset = 0;
     vector<int> originalV = v;
 
+    // Instanciates the functor to apply on each element
     plusOne plusOneInstance;
 
     vector< Test > tests;
 
     const int ALL = Node::comm().size();
+    if( ALL < 3 ) {
+        throw std::runtime_error("Needs at least 3 processes to be launched!");
+    }
 
+    // Tests are auto described thanks to member "description"
     Test tIntervalStatic;
     tIntervalStatic.assign = new StaticAssignmentAlgorithm( 1, REST_OF_THE_WORLD, v.size() );
-    tIntervalStatic.description = "Correct static assignment with interval.";
+    tIntervalStatic.description = "Correct static assignment with interval."; // workers have ranks from 1 to size - 1
     tIntervalStatic.requiredNodesNumber = ALL;
     tests.push_back( tIntervalStatic );
 
@@ -110,22 +165,32 @@ int main(int argc, char** argv)
 
     for( unsigned int i = 0; i < tests.size(); ++i )
     {
-        ParallelApplyStore< int > store( plusOneInstance, v, eo::mpi::DEFAULT_MASTER, 3 );
+        // Instanciates a store with the functor, the master rank and size of packet (see ParallelApplyStore doc).
-        // Job< JobData<int> > job( *(tests[i].assign), eo::mpi::DEFAULT_MASTER, store );
+        ParallelApplyStore< int > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 3 );
+        // Updates the contained data
+        store.data( v );
+        // Creates the job with the assignment algorithm, the master rank and the store
         ParallelApply< int > job( *(tests[i].assign), eo::mpi::DEFAULT_MASTER, store );
 
+        // Only master writes information
         if( job.isMaster() )
         {
             cout << "Test : " << tests[i].description << endl;
         }
 
+        // Workers whose rank is inferior to required nodes number have to run the test, the other haven't anything to
+        // do.
         if( Node::comm().rank() < tests[i].requiredNodesNumber )
         {
             job.run();
         }
 
+        // After the job run, the master checks the result with offset and originalV
         if( job.isMaster() )
         {
+            // This job has to be instanciated, not launched, so as to tell the workers they're done with the parallel
+            // job.
+            EmptyJob stop( *(tests[i].assign), eo::mpi::DEFAULT_MASTER ); 
             ++offset;
             for(int i = 0; i < v.size(); ++i)
             {
@@ -139,6 +204,7 @@ int main(int argc, char** argv)
             cout << endl;
         }
 
+        // MPI synchronization (all the processes wait to be here).
         Node::comm().barrier();
 
         delete tests[i].assign;

eo/test/mpi/t-mpi-wrapper.cpp

@@ -0,0 +1,132 @@
+/*
+(c) Thales group, 2012
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation;
+    version 2 of the License.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+Contact: http://eodev.sourceforge.net
+
+Authors:
+    Benjamin Bouvier <benjamin.bouvier@gmail.com>
+*/
+
+/*
+ * This file shows an example of how to wrap a handler of a job store. Here, the wrapped handler is the "IsFinished"
+ * one. The only function that has been added is that the wrapper prints a message on standard output, indicating what
+ * the wrapped function returns as a result.
+ *
+ * This test is performed on a parallel apply job, the same as in parallelApply. The main difference is when
+ * instanciating the store.
+ */
+
+# include <mpi/eoMpi.h>
+# include <mpi/eoParallelApply.h>
+# include <mpi/eoTerminateJob.h>
+
+# include <iostream>
+
+# include <vector>
+using namespace std;
+
+using namespace eo::mpi;
+
+// Job functor.
+struct plusOne : public eoUF< int&, void >
+{
+    void operator() ( int & x )
+    {
+        ++x;
+    }
+};
+
+/*
+ * Shows the wrapped result of IsFinished, prints a message and returns the wrapped value.
+ * times is an integer counting how many time the wrapper (hence the wrapped too) has been called.
+ */
+template< class EOT >
+struct ShowWrappedResult : public IsFinishedParallelApply<EOT>
+{
+    using IsFinishedParallelApply<EOT>::_wrapped;
+
+    ShowWrappedResult ( IsFinishedParallelApply<EOT> * w = 0 ) : IsFinishedParallelApply<EOT>( w ), times( 0 )
+    {
+        // empty
+    }
+
+    bool operator()()
+    {
+        bool wrappedValue = _wrapped->operator()(); // (*_wrapped)();
+        cout << times << ") Wrapped function would say that it is " << ( wrappedValue ? "":"not ") << "finished" << std::endl;
+        ++times;
+        return wrappedValue;
+    }
+
+    private:
+    int times;
+};
+
+int main(int argc, char** argv)
+{
+    // eo::log << eo::setlevel( eo::debug );
+    eo::log << eo::setlevel( eo::quiet );
+
+    Node::init( argc, argv );
+
+    srand( time(0) );
+    vector<int> v;
+    for( int i = 0; i < 1000; ++i )
+    {
+        v.push_back( rand() );
+    }
+
+    int offset = 0;
+    vector<int> originalV = v;
+
+    plusOne plusOneInstance;
+
+    StaticAssignmentAlgorithm assign( v.size() );
+
+    ParallelApplyStore< int > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 1 );
+    store.data( v );
+    // This is the only thing which changes: we wrap the IsFinished function.
+    // According to RAII, we'll delete the invokated wrapper at the end of the main ; the store won't delete it
+    // automatically.
+    IsFinishedParallelApply* wrapper = new ShowWrappedResult<int>;
+    store.wrapIsFinished( wrapper );
+
+    ParallelApply<int> job( assign, eo::mpi::DEFAULT_MASTER, store );
+    // Equivalent to:
+    // Job< ParallelApplyData<int> > job( assign, 0, store );
+    job.run();
+    EmptyJob stop( assign, eo::mpi::DEFAULT_MASTER );
+
+    if( job.isMaster() )
+    {
+        ++offset;
+        for(int i = 0; i < v.size(); ++i)
+        {
+            cout << v[i] << ' ';
+            if( originalV[i] + offset != v[i] )
+            {
+                cout << " <-- ERROR at this point." << endl;
+                exit( EXIT_FAILURE );
+            }
+        }
+        cout << endl;
+    }
+
+    delete wrapper;
+
+    return 0;
+}
+

eo/test/mpi/wrapper.cpp

@@ -1,88 +0,0 @@
-# include <mpi/eoMpi.h>
-# include <mpi/eoParallelApply.h>
-
-# include <iostream>
-
-# include <vector>
-using namespace std;
-
-using namespace eo::mpi;
-
-struct plusOne : public eoUF< int&, void >
-{
-    void operator() ( int & x )
-    {
-        ++x;
-    }
-};
-
-template< class EOT >
-struct ShowWrappedResult : public IsFinishedParallelApply<EOT>
-{
-    using IsFinishedParallelApply<EOT>::_wrapped;
-
-    ShowWrappedResult ( IsFinishedParallelApply<EOT> * w = 0 ) : IsFinishedParallelApply<EOT>( w ), times( 0 )
-    {
-        // empty
-    }
-
-    bool operator()()
-    {
-        bool wrappedValue = _wrapped->operator()(); // (*_wrapped)();
-        cout << times << ") Wrapped function would say that it is " << ( wrappedValue ? "":"not ") << "finished" << std::endl;
-        ++times;
-        return wrappedValue;
-    }
-
-    private:
-    int times;
-};
-
-// These tests require at least 3 processes to be launched.
-int main(int argc, char** argv)
-{
-    // eo::log << eo::setlevel( eo::debug );
-    eo::log << eo::setlevel( eo::quiet );
-
-    Node::init( argc, argv );
-
-    srand( time(0) );
-    vector<int> v;
-    for( int i = 0; i < 1000; ++i )
-    {
-        v.push_back( rand() );
-    }
-
-    int offset = 0;
-    vector<int> originalV = v;
-
-    plusOne plusOneInstance;
-
-    StaticAssignmentAlgorithm assign( v.size() );
-
-    ParallelApplyStore< int > store( plusOneInstance, v, eo::mpi::DEFAULT_MASTER, 1 );
-    store.wrapIsFinished( new ShowWrappedResult<int> );
-
-    ParallelApply<int> job( assign, eo::mpi::DEFAULT_MASTER, store );
-    // Equivalent to:
-    // Job< ParallelApplyData<int> > job( assign, 0, store );
-    job.run();
-
-    if( job.isMaster() )
-    {
-        ++offset;
-        for(int i = 0; i < v.size(); ++i)
-        {
-            cout << v[i] << ' ';
-            if( originalV[i] + offset != v[i] )
-            {
-                cout << " <-- ERROR at this point." << endl;
-                exit( EXIT_FAILURE );
-            }
-        }
-        cout << endl;
-    }
-
-    return 0;
-}
-