diff --git a/eo/.gitignore b/eo/.gitignore new file mode 100644 index 000000000..2bfc9fc77 --- /dev/null +++ b/eo/.gitignore @@ -0,0 +1,5 @@ +*.swp +debug/ +release/ +*CMakeFiles* +*Makefile diff --git a/eo/CMakeLists.txt b/eo/CMakeLists.txt index 63de5b229..31cd2cf44 100644 --- a/eo/CMakeLists.txt +++ b/eo/CMakeLists.txt @@ -45,10 +45,12 @@ ENABLE_LANGUAGE(C) ### 2) Include required modules / configuration files ##################################################################################### -FIND_PACKAGE(OpenMP) -IF(OPENMP_FOUND) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") +IF(WITH_OMP) + FIND_PACKAGE(OpenMP) + IF(OPENMP_FOUND) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + ENDIF() ENDIF() INCLUDE(CMakeBackwardCompatibilityCXX) @@ -119,6 +121,11 @@ ENDIF (ENABLE_CMAKE_TESTING) ADD_SUBDIRECTORY(doc) ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(test) + +IF(WITH_MPI) + ADD_SUBDIRECTORY(test/mpi) +ENDIF() + ADD_SUBDIRECTORY(tutorial) ###################################################################################### diff --git a/eo/eo-conf.cmake b/eo/eo-conf.cmake index 9d3d5c3a8..7b1ab6419 100644 --- a/eo/eo-conf.cmake +++ b/eo/eo-conf.cmake @@ -6,3 +6,13 @@ SET(PROJECT_VERSION_PATCH 0) SET(PROJECT_VERSION_MISC "-edge") # ADD_DEFINITIONS(-DDEPRECATED_MESSAGES) # disable warning deprecated function messages +# If you plan to use OpenMP, put the following boolean to true : +SET(WITH_OMP FALSE CACHE BOOL "Use OpenMP ?" FORCE) + +# If you plan to use MPI, precise here where are the static libraries from +# openmpi and boost::mpi. + +SET(WITH_MPI FALSE CACHE BOOL "Use mpi ?" FORCE) +SET(MPI_DIR "/mpi/directory" CACHE PATH "OpenMPI directory" FORCE) +SET(BOOST_DIR "/boost/directory" CACHE PATH "Boost directory" FORCE) + diff --git a/eo/src/CMakeLists.txt b/eo/src/CMakeLists.txt index c03bc509a..caeb673ad 100644 --- a/eo/src/CMakeLists.txt +++ b/eo/src/CMakeLists.txt @@ -5,7 +5,32 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) ###################################################################################### -### 2) Define the eo target +### 2) Optional: add MPI and Boost MPI dependencies. +###################################################################################### + +IF(WITH_MPI) + MESSAGE("[EO] Compilation with MPI and BoostMPI.") + + SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") + + # headers location + INCLUDE_DIRECTORIES(${MPI_DIR}/include) + INCLUDE_DIRECTORIES(${BOOST_DIR}/include) + + # lib location + LINK_DIRECTORIES(${MPI_DIR}/lib) + LINK_DIRECTORIES(${BOOST_DIR}/lib) + + # for conditional compilation in code + ADD_DEFINITIONS(-DWITH_MPI) + + LINK_LIBRARIES(boost_mpi boost_serialization) + + ADD_SUBDIRECTORY(mpi) +ENDIF() + +###################################################################################### +### 3) Define the eo target ###################################################################################### SET(EO_LIB_OUTPUT_PATH ${EO_BINARY_DIR}/lib) @@ -27,14 +52,14 @@ FILE(GLOB HDRS *.h eo) INSTALL(FILES ${HDRS} DESTINATION include/eo COMPONENT headers) ###################################################################################### -### 3) Optionnal: define your target(s)'s version: no effect for windows +### 4) Optionnal: define your target(s)'s version: no effect for windows ###################################################################################### SET(EO_VERSION ${GLOBAL_VERSION}) SET_TARGET_PROPERTIES(eo PROPERTIES VERSION "${EO_VERSION}") ###################################################################################### -### 4) Where must cmake go now ? +### 5) Where must cmake go now ? ###################################################################################### ADD_SUBDIRECTORY(do) @@ -43,6 +68,7 @@ ADD_SUBDIRECTORY(ga) ADD_SUBDIRECTORY(gp) ADD_SUBDIRECTORY(other) ADD_SUBDIRECTORY(utils) +ADD_SUBDIRECTORY(serial) IF(ENABLE_PYEO) ADD_SUBDIRECTORY(pyeo) diff --git a/eo/src/apply.h b/eo/src/apply.h index bbd30aa3a..45c4cfc2d 100644 --- a/eo/src/apply.h +++ b/eo/src/apply.h @@ -19,7 +19,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Contact: todos@geneura.ugr.es, http://geneura.ugr.es - mak@dhi.dk + mak@dhi.dk */ //----------------------------------------------------------------------------- @@ -51,41 +51,41 @@ void apply(eoUF& _proc, std::vector& _pop) double t1 = 0; if ( eo::parallel.enableResults() ) - { - t1 = omp_get_wtime(); - } + { + t1 = omp_get_wtime(); + } if (!eo::parallel.isDynamic()) - { + { #pragma omp parallel for if(eo::parallel.isEnabled()) //default(none) shared(_proc, _pop, size) #ifdef _MSC_VER - //Visual Studio supports only OpenMP version 2.0 in which - //an index variable must be of a signed integral type - for (long long i = 0; i < size; ++i) { _proc(_pop[i]); } + //Visual Studio supports only OpenMP version 2.0 in which + //an index variable must be of a signed integral type + for (long long i = 0; i < size; ++i) { _proc(_pop[i]); } #else // _MSC_VER - for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } + for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } #endif - } + } else - { + { #pragma omp parallel for schedule(dynamic) if(eo::parallel.isEnabled()) #ifdef _MSC_VER - //Visual Studio supports only OpenMP version 2.0 in which - //an index variable must be of a signed integral type - for (long long i = 0; i < size; ++i) { _proc(_pop[i]); } + //Visual Studio supports only OpenMP version 2.0 in which + //an index variable must be of a signed integral type + for (long long i = 0; i < size; ++i) { _proc(_pop[i]); } #else // _MSC_VER - //doesnot work with gcc 4.1.2 - //default(none) shared(_proc, _pop, size) - for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } + //doesnot work with gcc 4.1.2 + //default(none) shared(_proc, _pop, size) + for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } #endif - } + } if ( eo::parallel.enableResults() ) - { - double t2 = omp_get_wtime(); - eoLogger log; - log << eo::file(eo::parallel.prefix()) << t2 - t1 << ' '; - } + { + double t2 = omp_get_wtime(); + eoLogger log; + log << eo::file(eo::parallel.prefix()) << t2 - t1 << ' '; + } #else // _OPENMP @@ -109,7 +109,7 @@ void apply(eoUF& _proc, std::vector& _pop) // //default(none) shared(_proc, _pop, size) // for (size_t i = 0; i < size; ++i) // { -// _proc(_pop[i]); +// _proc(_pop[i]); // } // } @@ -127,7 +127,7 @@ void apply(eoUF& _proc, std::vector& _pop) // //default(none) shared(_proc, _pop, size) // for (size_t i = 0; i < size; ++i) // { -// _proc(_pop[i]); +// _proc(_pop[i]); // } // } diff --git a/eo/src/eo b/eo/src/eo index 3a31daea4..fd660b7b9 100644 --- a/eo/src/eo +++ b/eo/src/eo @@ -142,6 +142,9 @@ #include // includes eoRealBounds.h #include // no eoIntVectorBounds +// Serialization stuff +#include + // aliens #include #include diff --git a/eo/src/eoEasyEA.h b/eo/src/eoEasyEA.h index 2c7c5474c..4d9c7b6da 100644 --- a/eo/src/eoEasyEA.h +++ b/eo/src/eoEasyEA.h @@ -37,8 +37,6 @@ #include #include - - template class eoIslandsEasyEA ; template class eoDistEvalEasyEA ; @@ -102,6 +100,33 @@ template class eoEasyEA: public eoAlgo offspring.reserve(_offspringSize); // This line avoids an incremental resize of offsprings. } + /** + * @brief Ctor allowing to specify which pop eval function we're going to use. + * + * Ctor taking a breed and merge, an overload of ctor to define an offspring size, and + * the pop eval function used. This allows to precise if we would like to use the + * parallel evaluation, for instance. + */ + eoEasyEA( + eoContinue& _continuator, + eoEvalFunc& _eval, + eoPopEvalFunc& _pop_eval, + eoBreed& _breed, + eoReplacement& _replace, + unsigned _offspringSize + ) : continuator(_continuator), + eval (_eval), + loopEval(_eval), + popEval(_pop_eval), + selectTransform(dummySelect, dummyTransform), + breed(_breed), + mergeReduce(dummyMerge, dummyReduce), + replace(_replace), + isFirstCall(true) + { + offspring.reserve(_offspringSize); // This line avoids an incremental resize of offsprings. + } + /* eoEasyEA(eoContinue & _continuator, eoPopEvalFunc & _pop_eval, @@ -219,45 +244,44 @@ template class eoEasyEA: public eoAlgo /// Apply a few generation of evolution to the population. virtual void operator()(eoPop& _pop) { - if (isFirstCall) - { - size_t total_capacity = _pop.capacity() + offspring.capacity(); - _pop.reserve(total_capacity); - offspring.reserve(total_capacity); - isFirstCall = false; - } - eoPop empty_pop; - - popEval(empty_pop, _pop); // A first eval of pop. - - do + if (isFirstCall) { - try + size_t total_capacity = _pop.capacity() + offspring.capacity(); + _pop.reserve(total_capacity); + offspring.reserve(total_capacity); + isFirstCall = false; + } + + eoPop empty_pop; + + do + { + try { - unsigned pSize = _pop.size(); - offspring.clear(); // new offspring + unsigned pSize = _pop.size(); - breed(_pop, offspring); + offspring.clear(); // new offspring - popEval(_pop, offspring); // eval of parents + offspring if necessary + breed(_pop, offspring); - replace(_pop, offspring); // after replace, the new pop. is in _pop + popEval(_pop, offspring); // eval of parents + offspring if necessary - if (pSize > _pop.size()) - throw std::runtime_error("Population shrinking!"); - else if (pSize < _pop.size()) - throw std::runtime_error("Population growing!"); + replace(_pop, offspring); // after replace, the new pop. is in _pop + if (pSize > _pop.size()) + throw std::runtime_error("Population shrinking!"); + else if (pSize < _pop.size()) + throw std::runtime_error("Population growing!"); } - catch (std::exception& e) + catch (std::exception& e) { - std::string s = e.what(); - s.append( " in eoEasyEA"); - throw std::runtime_error( s ); + std::string s = e.what(); + s.append( " in eoEasyEA"); + throw std::runtime_error( s ); } } - while ( continuator( _pop ) ); + while ( continuator( _pop ) ); } protected : diff --git a/eo/src/eoPopEvalFunc.h b/eo/src/eoPopEvalFunc.h index aba74d9ab..7fcad1467 100644 --- a/eo/src/eoPopEvalFunc.h +++ b/eo/src/eoPopEvalFunc.h @@ -30,6 +30,16 @@ #include #include +# ifdef WITH_MPI +#include +#include +#include +#include +#include + +#include // ceil +# endif // WITH_MPI + /** eoPopEvalFunc: This abstract class is for GLOBAL evaluators * of a population after variation. * It takes 2 populations (typically the parents and the offspring) @@ -77,6 +87,142 @@ private: eoEvalFunc & eval; }; +#ifdef WITH_MPI +/** + * @brief Evaluator of a population of EOT which uses parallelization to evaluate individuals. + * + * This class implements an instance of eoPopEvalFunc that applies a private eoEvalFunc to + * all offspring, but in a parallel way. The original process becomes the central host from a network ("master"), and + * other machines disponible in the MPI network ("slaves") are used as evaluators. Population to evaluate is splitted in + * little packets of individuals, which are sent to the evaluators, that process the effective call to eval. Once all + * the individuals have been evaluated, they are returned to the master. The whole process is entirely invisible to the + * eyes of the user, who just has to launch a certain number of processes in MPI so as to have a result. + * + * The eoEvalFunc is no more directly given, but it is stored in the eo::mpi::ParallelApplyStore, which can be + * instanciated if no one is given at construction. + * + * The use of this class requires the user to have called the eo::mpi::Node::init function, at the beginning of its + * program. + * + * @ingroup Evaluation Parallel + * + * @author Benjamin Bouvier + */ +template +class eoParallelPopLoopEval : public eoPopEvalFunc +{ + public: + /** + * @brief Constructor which creates the job store for the user. + * + * This constructor is the simplest to use, as it creates the store used by the parallel job, for the user. + * The user just precises the scheduling algorithm, the rank of the master and then gives its eval function and + * the size of a packet (how many individuals should be in a single message to evaluator). + * + * @param _assignAlgo The scheduling algorithm used to give orders to evaluators. + * @param _masterRank The MPI rank of the master. + * @param _eval The evaluation functor used to evaluate each individual in the population. + * @param _packetSize The number of individuals to send in one message to evaluator, and which are evaluated at + * a time. + */ + eoParallelPopLoopEval( + // Job parameters + eo::mpi::AssignmentAlgorithm& _assignAlgo, + int _masterRank, + // Default parameters for store + eoEvalFunc & _eval, + int _packetSize = 1 + ) : + assignAlgo( _assignAlgo ), + masterRank( _masterRank ), + needToDeleteStore( true ) // we used new, we'll have to use delete (RAII) + { + store = new eo::mpi::ParallelApplyStore( _eval, _masterRank, _packetSize ); + } + + /** + * @brief Constructor which allows the user to customize its job store. + * + * This constructor allows the user to customize the store, for instance by adding wrappers and other + * functionnalities, before using it for the parallelized evaluation. + * + * @param _assignAlgo The scheduling algorithm used to give orders to evaluators. + * @param _masterRank The MPI rank of the master. + * @param _store Pointer to a parallel eval store given by the user. + */ + eoParallelPopLoopEval( + // Job parameters + eo::mpi::AssignmentAlgorithm& _assignAlgo, + int _masterRank, + eo::mpi::ParallelApplyStore* _store + ) : + assignAlgo( _assignAlgo ), + masterRank( _masterRank ), + store( _store ), + needToDeleteStore( false ) // we haven't used new for creating store, we don't care if we have to delete it (RAII). + { + // empty + } + + /** + * @brief Default destructor. Sends a message to all evaluators indicating that the global loop (eoEasyEA, for + * instance) is over. + */ + ~eoParallelPopLoopEval() + { + // Only the master has to send the termination message + if( eo::mpi::Node::comm().rank() == masterRank ) + { + eo::mpi::EmptyJob job( assignAlgo, masterRank ); + job.run(); + } + + // RAII + if( needToDeleteStore ) + { + delete store; + } + } + + /** + * @brief Parallel implementation of the operator(). + * + * @param _parents Population of parents (ignored). + * @param _offspring Population of children, which will be evaluated. + */ + void operator()( eoPop & _parents, eoPop & _offspring ) + { + (void)_parents; + // Reinits the store and the scheduling algorithm + store->data( _offspring ); + // For static scheduling, it's mandatory to reinit attributions + int nbWorkers = assignAlgo.availableWorkers(); + assignAlgo.reinit( nbWorkers ); + if( ! eo::parallel.isDynamic() ) { + store->data()->packetSize = ceil( static_cast( _offspring.size() ) / nbWorkers ); + } + // Effectively launches the job. + eo::mpi::ParallelApply job( assignAlgo, masterRank, *store ); + job.run(); + } + + private: + // Scheduling algorithm + eo::mpi::AssignmentAlgorithm & assignAlgo; + // Master MPI rank + int masterRank; + + // Store + eo::mpi::ParallelApplyStore* store; + // Do we have to delete the store by ourselves ? + bool needToDeleteStore; +}; + +/** + * @example t-mpi-eval.cpp + */ +#endif + ///////////////////////////////////////////////////////////// // eoTimeVaryingLoopEval ///////////////////////////////////////////////////////////// diff --git a/eo/src/mpi/CMakeLists.txt b/eo/src/mpi/CMakeLists.txt new file mode 100644 index 000000000..0e22e47e0 --- /dev/null +++ b/eo/src/mpi/CMakeLists.txt @@ -0,0 +1,32 @@ +###################################################################################### +### 1) Include the sources +###################################################################################### + +INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/src) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +###################################################################################### +### 2) Define the eompi target +###################################################################################### + +SET(EOMPI_LIB_OUTPUT_PATH ${EO_BINARY_DIR}/lib) +SET(LIBRARY_OUTPUT_PATH ${EOMPI_LIB_OUTPUT_PATH}) + +SET(EOMPI_SOURCES + eoMpi.cpp + ) + +ADD_LIBRARY(eompi STATIC ${EOMPI_SOURCES}) +INSTALL(TARGETS eompi ARCHIVE DESTINATION lib COMPONENT libraries) + +FILE(GLOB HDRS *.h) +INSTALL(FILES ${HDRS} DESTINATION include/eo/mpi COMPONENT headers) + +###################################################################################### +### 3) Optionnal +###################################################################################### + +SET(EOMPI_VERSION ${GLOBAL_VERSION}) +SET_TARGET_PROPERTIES(eompi PROPERTIES VERSION "${EOMPI_VERSION}") + +###################################################################################### diff --git a/eo/src/mpi/eoMpi.cpp b/eo/src/mpi/eoMpi.cpp new file mode 100644 index 000000000..ab7543284 --- /dev/null +++ b/eo/src/mpi/eoMpi.cpp @@ -0,0 +1,11 @@ +# include "eoMpi.h" + +namespace eo +{ + namespace mpi + { + bmpi::communicator Node::_comm; + eoTimerStat timerStat; + } +} + diff --git a/eo/src/mpi/eoMpi.h b/eo/src/mpi/eoMpi.h new file mode 100644 index 000000000..ee92e0878 --- /dev/null +++ b/eo/src/mpi/eoMpi.h @@ -0,0 +1,836 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EO_MPI_H__ +# define __EO_MPI_H__ + +# include // std::vector + +# include +# include +# include +# include + +# include "eoMpiNode.h" +# include "eoMpiAssignmentAlgorithm.h" + +namespace eo +{ + /** + * @ingroup Parallel + * @defgroup MPI Message Passing Interface + * @brief See namespace eo::mpi to have all explanations about this module. + * @{ + */ + + /** + * @brief MPI parallelization helpers for EO. + * + * This namespace contains parallelization functions which help to parallelize computations in EO. It is based on a + * generic algorithm, which is then customized with functors, corresponding to the algorithm main steps. These + * computations are centralized, i.e there is one central host whose role is to handle the steps of the algorithm ; + * we call it the "master". The other hosts just have to perform a "dummy" computation, which may be any kind of + * processing ; we call them, the "slaves", or less pejoratively, the "workers". Workers can communicate to each + * other, but they receive their orders from the Master and send him back some results. A worker can also be the + * master of a different parallelization process, as soon as it is a part of its work. Machines of the network, also + * called hosts, are identified by an unique number: their rank. At any time during the execution of the program, + * all the hosts know the total number of hosts. + * + * A parallelized Job is a set of tasks which are independant (i.e can be executed in random order without + * modifiying the result) and take a data input and compute a data output to be sent to the Master. The data can be + * of any type, however they have to be serialized to be sent over a network. It is sufficient that they can be + * serialized through boost. + * + * @todo For serialization purposes, don't depend upon boost. It would be easy to use only eoserial and send strings + * via mpi. + * + * The main steps of the algorithm are the following: + * - For the master: + * - Have we done with the treatment we are doing ? + * - If this is the case, we can quit. + * - Otherwise, send an input data to some available worker. + * - If there's no available worker, wait for a worker to be free. + * - When receiving the response, handle it (eventually compute something on the output data, store it...). + * - Go back to the first step. + * - For the worker, it is even easier: + * - Wait for an order. + * - If there's nothing to do, just quit. + * - Otherwise, eventually retrieve data and do the work. + * - Go back to the first step. + * + * There is of course some network adjustements to do and precisions to give there, but the main ideas are present. As the + * job is fully centralized, this is the master who tells the workers when to quit and when to work. + * + * The idea behind these MPI helpers is to be the most generic possible. If we look back at the steps of the + * algorithm, we found that the steps can be splitted into 2 parts: the first consists in the steps of any + * parallelization algorithm and the other consists in the specific parts of the algorithm. Ideally, the user should + * just have to implement the specific parts of the algorithm. We identified these parts to be: + * - For the master: + * - What does mean to have terminated ? There are only two alternatives, in our binary world: either it is + * terminated, or it is not. Hence we only need a function returning a boolean to know if we're done with the + * computation : we'll call it IsFinished. + * - What do we have to do when we send a task ? We don't have any a priori on the form of the sent data, or + * the number of sent data. Moreover, as the tasks are all independant, we don't care of who will do the + * computation, as soon as it's done. Knowing the rank of the worker will be sufficient to send him data. We + * have identified another function, taking a single argument which is the rank of the worker: we'll call it + * SendTask. + * - What do we have to do when we receive a response from a worker ? One more time, we don't know which form + * or structure can have the receive data, only the user can know. Also we let the user the charge to retrieve + * the data ; he just has to know from who the master will retrieve the data. Here is another function, taking + * a rank (the sender's one) as a function argument : this will be HandleResponse. + * - For the worker: + * - What is the processing ? It can have any nature. We just need to be sure that a data is sent back to the + * master, but it seems difficult to check that: it will be the role of the user to assert that data is sent by + * the worker at the end of an execution. We've got identified our last function: ProcessTask. + * + * In term of implementation, it would be annoying to have only abstract classes with these 4 methods to implement. It + * would mean that if you want to alter just one of these 4 functions, you have to implement a new sub class, with a + * new constructor which could have the same signature. Besides, this fashion doesn't allow you to add dynamic + * functionalities, using the design pattern Decorator for instance, without implement a class for each type of + * decoration you want to add. For these reasons, we decided to transform function into functors ; the user can then + * wrap the existing, basic comportments into more sophisticated computations, whenever he wants, and without the + * notion of order. We retrieve here the power of extension given by the design pattern Decorator. + * + * Our 4 functors could have a big amount of data in common (see eoParallelApply to have an idea). + * So as to make it easy for the user to implement these 4 functors, we consider that these functors + * have to share a common data structure. This data structure is referenced (as a pointer) in the 4 functors, so the + * user doesn't need to pass a lot of parameters to each functor constructor. + * + * There are two kinds of jobs: + * - The job which are launched a fixed and well known amount of times, i.e both master and workers know how many + * times they will be launched. They are "one shot jobs". + * - The job which are launched an unknown amount of times, for instance embedded in a while loop for which we don't + * know the amount of repetitions (typically, eoEasyEA loop is a good example, as we don't know the continuator + * condition). They are called "multi job". + * As the master tells the workers to quit, we have to differentiate these two kinds of jobs. When the job is of the + * kind "multi job", the workers would have to perform a while(true) loop so as to receive the orders ; but even if + * the master tells them to quit, they would begin another job and wait for another order, while the master would + * have quit: this would cause a deadlock and workers processes would be blocked, waiting for an order. + */ + namespace mpi + { + /** + * @brief A timer which allows user to generate statistics about computation times. + */ + extern eoTimerStat timerStat; + + /** + * @brief Tags used in MPI messages for framework communication + * + * These tags are used for framework communication and fits "channels", so as to differentiate when we're + * sending an order to a worker (Commands) or data (Messages). They are not reserved by the framework and can be + * used by the user, but he is not bound to. + * + * @ingroup MPI + */ + namespace Channel + { + const int Commands = 0; + const int Messages = 1; + } + + /** + * @brief Simple orders used by the framework. + * + * These orders are sent by the master to the workers, to indicate to them if they should receive another task + * to do (Continue), if an one shot job is done (Finish) or if a multi job is done (Kill). + * + * @ingroup MPI + */ + namespace Message + { + const int Continue = 0; + const int Finish = 1; + const int Kill = 2; + } + + /** + * @brief If the job only has one master, the user can use this constant, so as not to worry with integer ids. + * + * @ingroup MPI + */ + const int DEFAULT_MASTER = 0; + + /** + * @brief Base class for the 4 algorithm functors. + * + * This class can embed a data (JobData) and a wrapper, so as to make all the 4 functors wrappable. + * We can add a wrapper at initialization or at any time when executing the program. + * + * According to RAII, the boolean needDelete helps to know if we have to use the operator delete on the wrapper + * or not. Hence, if any functor is wrapped, user has just to put this boolean to true, to indicate to wrapper + * that it should call delete. This allows to mix wrapper initialized in the heap (with new) or in the stack. + * + * @param JobData a Data type, which can have any form. It can a struct, a single int, anything. + * + * @param Wrapped the type of the functor, which will be stored as a pointer under the name _wrapped. + * This allows to wrap directly the functor in functors of the same type + * here, instead of dealing with SharedDataFunction* that we would have to cast all the time. + * Doing also allows to handle the wrapped functor as the functor we're writing, when coding the wrappers, + * instead of doing some static_cast. For instance, if there are 2 functors subclasses, fA and fB, fA + * implementing doFa() and fB implementing doFb(), we could have the following code: + * @code + * struct fA_wrapper + * { + * // some code + * void doFa() + * { + * _wrapped->doFa(); + * std::cout << "I'm a fA wrapper!" << std::endl; + * // if we didn't have the second template parameter, but a SharedDataFunction, we would have to do this: + * static_cast(_wrapped)->doFa(); + * // do other things (it's a wrapper) + * } + * }; + * + * struct fB_wrapper + * { + * // some code + * void doFb() + * { + * _wrapped->doFb(); // and not: static_cast(_wrapped)->doFb(); + * } + * }; + * @endcode + * This makes the code easier to write for the user. + * + * @ingroup MPI + */ + template< typename JobData, typename Wrapped > + struct SharedDataFunction + { + /** + * @brief Default constructor. + * + * The user is not bound to give a wrapped functor. + */ + SharedDataFunction( Wrapped * w = 0 ) : _data( 0 ), _wrapped( w ), _needDelete( false ) + { + // empty + } + + /** + * @brief Destructor. + * + * Calls delete on the wrapped function, only if necessary. + */ + virtual ~SharedDataFunction() + { + if( _wrapped && _wrapped->needDelete() ) + { + delete _wrapped; + } + } + + /** + * @brief Setter for the wrapped function. + * + * It doesn't do anything on the current wrapped function, like deleting it. + */ + void wrapped( Wrapped * w ) + { + _wrapped = w; + } + + /** + * @brief Setter for the data present in the functor. + * + * Calls the setter on the functor and on the wrapped functors, in a Composite pattern fashion. + */ + void data( JobData* d ) + { + _data = d; + if( _wrapped ) + { + _wrapped->data( d ); + } + } + + /** + * @brief Returns true if we need to use operator delete on this wrapper, false otherwise. + * + * Allows the user to reject delete responsability to the framework, by setting this value to true. + **/ + bool needDelete() { return _needDelete; } + void needDelete( bool b ) { _needDelete = b; } + + protected: + JobData* _data; + Wrapped* _wrapped; // Pointer and not a reference so as to be set at any time and to avoid affectation + bool _needDelete; + }; + + /** + * @brief Functor (master side) used to send a task to the worker. + * + * The user doesn't have to know which worker will receive a task, so we just indicate to master the rank of the + * worker. The data used for computation have to be explicitly sent by the master to the worker, with indicated + * rank. Once this functor has been called, the worker is considered busy until it sends a return message to the + * master. + * + * This is a functor implementing void operator()(int), and also a shared data function, containing wrapper on its + * own type. + * + * @ingroup MPI + */ + template< typename JobData > + struct SendTaskFunction : public eoUF, public SharedDataFunction< JobData, SendTaskFunction > + { + public: + + SendTaskFunction( SendTaskFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~SendTaskFunction() {} // for inherited classes + }; + + /** + * @brief Functor (master side) used to indicate what to do when receiving a response. + * + * The master calls this function as soon as it receives some data, in some channel. Thanks to MPI, we retrieve + * the rank of the data's sender. This functor is then called with this rank. There is no memoization of a link + * between sent data and rank, so the user has to implement it, if he needs it. + * + * This is a functor implementing void operator()(int), and also a shared data function, containing wrapper on + * its own type. + * + * @ingroup MPI + */ + template< typename JobData > + struct HandleResponseFunction : public eoUF, public SharedDataFunction< JobData, HandleResponseFunction > + { + public: + + HandleResponseFunction( HandleResponseFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~HandleResponseFunction() {} // for inherited classes + }; + + /** + * @brief Functor (worker side) implementing the processing to do. + * + * This is where the real computation happen. + * Whenever the master sends the command "Continue" to workers, which indicates the worker will receive a task, + * the worker calls this functor. The user has to explicitly retrieve the data, handle it and transmit it, + * processed, back to the master. If the worker does not send any data back to the master, the latter will + * consider the worker isn't done and a deadlock could occur. + * + * This is a functor implementing void operator()(), and also a shared data function, containing wrapper on its + * own type. + * + * @ingroup MPI + */ + template< typename JobData > + struct ProcessTaskFunction : public eoF, public SharedDataFunction< JobData, ProcessTaskFunction > + { + public: + + ProcessTaskFunction( ProcessTaskFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~ProcessTaskFunction() {} // for inherited classes + }; + + /** + * @brief Functor (master side) indicating whether the job is done or not. + * + * The master loops on this functor to know when to stop. When this functor returns true, the master will wait + * for the last responses and properly stops the job. Whenever this functor returns false, the master will send + * tasks, until this functor returns true. + * + * This is a functor implementing bool operator()(), and also a shared function, containing wrapper on its own + * type. + * + * @ingroup MPI + */ + template< typename JobData > + struct IsFinishedFunction : public eoF, public SharedDataFunction< JobData, IsFinishedFunction > + { + public: + + IsFinishedFunction( IsFinishedFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~IsFinishedFunction() {} // for inherited classes + }; + + /** + * @brief Contains all the required data and the functors to launch a job. + * + * Splitting the functors and data from the job in itself allows to use the same functors and data for multiples + * instances of the same job. You define your store once and can use it a lot of times during your program. If + * the store was included in the job, you'd have to give again all the functors and all the datas to each + * invokation of the job. + * + * Job store contains the 4 functors (pointers, so as to be able to wrap them ; references couldn't have + * permitted that) described above and the JobData used by all these functors. It contains + * also helpers to easily wrap the functors, getters and setters on all of them. + * + * The user has to implement data(), which is the getter for retrieving JobData. We don't have any idea of who + * owns the data, moreover it is impossible to initialize it in this generic JobStore, as we don't know its + * form. As a matter of fact, the user has to define this in the JobStore subclasses. + * + * @ingroup MPI + */ + template< typename JobData > + struct JobStore + { + /** + * @brief Default ctor with the 4 functors. + */ + JobStore( + SendTaskFunction* stf, + HandleResponseFunction* hrf, + ProcessTaskFunction* ptf, + IsFinishedFunction* iff + ) : + _stf( stf ), _hrf( hrf ), _ptf( ptf ), _iff( iff ) + { + // empty + } + + /** + * @brief Empty ctor, useful for not forcing users to call the other constructor. + * + * When using this constructor, the user have to care about the 4 functors pointers, otherwise null pointer + * segfaults have to be expected. + */ + JobStore() + { + // empty + } + + /** + * @brief Default destructor. + * + * JobStore is the highest layer which calls needDelete on its functors. + */ + ~JobStore() + { + if( _stf->needDelete() ) delete _stf; + if( _hrf->needDelete() ) delete _hrf; + if( _ptf->needDelete() ) delete _ptf; + if( _iff->needDelete() ) delete _iff; + } + + // Getters + SendTaskFunction & sendTask() { return *_stf; } + HandleResponseFunction & handleResponse() { return *_hrf; } + ProcessTaskFunction & processTask() { return *_ptf; } + IsFinishedFunction & isFinished() { return *_iff; } + + // Setters + void sendTask( SendTaskFunction* stf ) { _stf = stf; } + void handleResponse( HandleResponseFunction* hrf ) { _hrf = hrf; } + void processTask( ProcessTaskFunction* ptf ) { _ptf = ptf; } + void isFinished( IsFinishedFunction* iff ) { _iff = iff; } + + /** + * @brief Helpers for wrapping send task functor. + */ + void wrapSendTask( SendTaskFunction* stf ) + { + if( stf ) + { + stf->wrapped( _stf ); + _stf = stf; + } + } + + /** + * @brief Helpers for wrapping handle response functor. + */ + void wrapHandleResponse( HandleResponseFunction* hrf ) + { + if( hrf ) + { + hrf->wrapped( _hrf ); + _hrf = hrf; + } + } + + /** + * @brief Helpers for wrapping process task functor. + */ + void wrapProcessTask( ProcessTaskFunction* ptf ) + { + if( ptf ) + { + ptf->wrapped( _ptf ); + _ptf = ptf; + } + } + + /** + * @brief Helpers for wrapping is finished functor. + */ + void wrapIsFinished( IsFinishedFunction* iff ) + { + if( iff ) + { + iff->wrapped( _iff ); + _iff = iff; + } + } + + virtual JobData* data() = 0; + + protected: + + SendTaskFunction< JobData >* _stf; + HandleResponseFunction< JobData >* _hrf; + ProcessTaskFunction< JobData >* _ptf; + IsFinishedFunction< JobData >* _iff; + }; + + /** + * @example t-mpi-wrapper.cpp + */ + + /** + * @brief Class implementing the centralized job algorithm. + * + * This class handles all the job algorithm. With its store and its assignment (scheduling) algorithm, it + * executes the general algorithm described above, adding some networking, so as to make the global process + * work. It initializes all the functors with the data, then launches the main loop, indicating to workers when + * they will have to work and when they will finish, by sending them a termination message (integer that can be + * customized). As the algorithm is centralized, it is also mandatory to indicate what is the MPI rank of the + * master process, hence the workers will know from who they should receive their commands. + * + * Any of the 3 master functors can launch exception, it will be catched and rethrown as a std::runtime_exception + * to the higher layers. + * + * @ingroup MPI + */ + template< class JobData > + class Job + { + public: + /** + * @brief Main constructor for Job. + * + * @param _algo The used assignment (scheduling) algorithm. It has to be initialized, with its maximum + * possible number of workers (some workers referenced in this algorithm shouldn't be busy). See + * AssignmentAlgorithm for more details. + * + * @param _masterRank The MPI rank of the master. + * + * @param _workerStopCondition Number of the message which will cause the workers to terminate. It could + * be one of the constants defined in eo::mpi::Commands, or any other integer. The user has to be sure + * that a message containing this integer will be sent to each worker on the Commands channel, otherwise + * deadlock will happen. Master sends Finish messages at the end of a simple job, but as a job can + * happen multiples times (multi job), workers don't have to really finish on these messages but on + * another message. This is here where you can configurate it. See also OneShotJob and MultiJob. + * + * @param store The JobStore containing functors and data for this job. + */ + Job( AssignmentAlgorithm& _algo, + int _masterRank, + int _workerStopCondition, + JobStore & _store + ) : + assignmentAlgo( _algo ), + masterRank( _masterRank ), + workerStopCondition( _workerStopCondition ), + comm( Node::comm() ), + // Functors + store( _store ), + sendTask( _store.sendTask() ), + handleResponse( _store.handleResponse() ), + processTask( _store.processTask() ), + isFinished( _store.isFinished() ) + { + _isMaster = Node::comm().rank() == _masterRank; + + sendTask.data( _store.data() ); + handleResponse.data( _store.data() ); + processTask.data( _store.data() ); + isFinished.data( _store.data() ); + } + + protected: + + /** + * @brief Finally block of the main algorithm + * + * Herb Sutter's trick for having a finally block, in a try/catch section: invoke a class at the + * beginning of the try, its destructor will be called in every cases. + * + * This implements the end of the master algorithm: + * - sends to all available workers that they are free, + * - waits for last responses, handles them and sends termination messages to last workers. + */ + struct FinallyBlock + { + FinallyBlock( + int _totalWorkers, + AssignmentAlgorithm& _algo, + Job< JobData > & _that + ) : + totalWorkers( _totalWorkers ), + assignmentAlgo( _algo ), + that( _that ), + // global field + comm( Node::comm() ) + { + // empty + } + + ~FinallyBlock() + { +# ifndef NDEBUG + eo::log << eo::debug; + eo::log << "[M" << comm.rank() << "] Frees all the idle." << std::endl; +# endif + // frees all the idle workers + timerStat.start("master_wait_for_idles"); + std::vector idles = assignmentAlgo.idles(); + for(unsigned int i = 0; i < idles.size(); ++i) + { + comm.send( idles[i], Channel::Commands, Message::Finish ); + } + timerStat.stop("master_wait_for_idles"); + +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Waits for all responses." << std::endl; +# endif + // wait for all responses + timerStat.start("master_wait_for_all_responses"); + while( assignmentAlgo.availableWorkers() != totalWorkers ) + { + bmpi::status status = comm.probe( bmpi::any_source, bmpi::any_tag ); + int wrkRank = status.source(); + that.handleResponse( wrkRank ); + comm.send( wrkRank, Channel::Commands, Message::Finish ); + assignmentAlgo.confirm( wrkRank ); + } + timerStat.stop("master_wait_for_all_responses"); +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Leaving master task." << std::endl; +# endif + } + + protected: + + int totalWorkers; + AssignmentAlgorithm& assignmentAlgo; + Job< JobData > & that; + + bmpi::communicator & comm; + }; + + /** + * @brief Master part of the job. + * + * Launches the parallelized job algorithm : while there is something to do (! IsFinished ), get a + * worker who will be the assignee ; if no worker is available, wait for a response, handle it and reask + * for an assignee. Then send the command and the task. + * Once there is no more to do (IsFinished), indicate to all available workers that they're free, wait + * for all the responses and send termination messages (see also FinallyBlock). + */ + void master( ) + { + int totalWorkers = assignmentAlgo.availableWorkers(); +# ifndef NDEBUG + eo::log << eo::debug; + eo::log << "[M" << comm.rank() << "] Have " << totalWorkers << " workers." << std::endl; +# endif + try { + FinallyBlock finally( totalWorkers, assignmentAlgo, *this ); + while( ! isFinished() ) + { + timerStat.start("master_wait_for_assignee"); + int assignee = assignmentAlgo.get( ); + while( assignee <= 0 ) + { +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Waitin' for node..." << std::endl; +# endif + bmpi::status status = comm.probe( bmpi::any_source, bmpi::any_tag ); + int wrkRank = status.source(); +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Node " << wrkRank << " just terminated." << std::endl; +# endif + handleResponse( wrkRank ); + assignmentAlgo.confirm( wrkRank ); + assignee = assignmentAlgo.get( ); + } + timerStat.stop("master_wait_for_assignee"); +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Assignee : " << assignee << std::endl; +# endif + + timerStat.start("master_wait_for_send"); + comm.send( assignee, Channel::Commands, Message::Continue ); + sendTask( assignee ); + timerStat.stop("master_wait_for_send"); + } + } catch( const std::exception & e ) + { + std::string s = e.what(); + s.append( " in eoMpi loop"); + throw std::runtime_error( s ); + } + } + + /** + * @brief Worker part of the algorithm. + * + * The algorithm is more much simpler: wait for an order; if it's termination message, leave. Otherwise, + * prepare to work. + */ + void worker( ) + { + int order; +# ifndef NDEBUG + eo::log << eo::debug; +# endif + timerStat.start("worker_wait_for_order"); + comm.recv( masterRank, Channel::Commands, order ); + timerStat.stop("worker_wait_for_order"); + + while( true ) + { +# ifndef NDEBUG + eo::log << "[W" << comm.rank() << "] Waiting for an order..." << std::endl; +# endif + if ( order == workerStopCondition ) + { +# ifndef NDEBUG + eo::log << "[W" << comm.rank() << "] Leaving worker task." << std::endl; +# endif + return; + } else if( order == Message::Continue ) + { +# ifndef NDEBUG + eo::log << "[W" << comm.rank() << "] Processing task..." << std::endl; +# endif + processTask( ); + } + + timerStat.start("worker_wait_for_order"); + comm.recv( masterRank, Channel::Commands, order ); + timerStat.stop("worker_wait_for_order"); + } + } + + public: + + /** + * @brief Launches the job algorithm, according to the role of the host (roles are deduced from the + * master rank indicated in the constructor). + */ + void run( ) + { + ( _isMaster ) ? master( ) : worker( ); + } + + /** + * @brief Returns true if the current host is the master, false otherwise. + */ + bool isMaster( ) + { + return _isMaster; + } + + protected: + + AssignmentAlgorithm& assignmentAlgo; + int masterRank; + const int workerStopCondition; + bmpi::communicator& comm; + + JobStore& store; + SendTaskFunction & sendTask; + HandleResponseFunction & handleResponse; + ProcessTaskFunction & processTask; + IsFinishedFunction & isFinished; + + bool _isMaster; + }; + + /** + * @brief Job that will be launched only once. + * + * As explained in eo::mpi documentation, jobs can happen either a well known amount of times or an unknown + * amount of times. This class implements the general case when the job is launched a well known amount of + * times. The job will be terminated on both sides (master and worker) once the master would have said it. + * + * It uses the message Message::Finish as the termination message. + * + * @ingroup MPI + */ + template< class JobData > + class OneShotJob : public Job< JobData > + { + public: + OneShotJob( AssignmentAlgorithm& algo, + int masterRank, + JobStore & store ) + : Job( algo, masterRank, Message::Finish, store ) + { + // empty + } + }; + + /** + * @brief Job that will be launched an unknown amount of times, in worker side. + * + * As explained in eo::mpi documentation, jobs can happen either a well known amount of times or an unknown + * amount of times. This class implements the general case when the job is launched an unknown amount of times, for + * instance in a while loop. The master will run many jobs (or the same job many times), but the workers will + * launch it only once. + * + * It uses the message Message::Kill as the termination message. This message can be launched with an EmptyJob, + * launched only by the master. If no Message::Kill is sent on the Channels::Commands, the worker will wait + * forever, which will cause a deadlock. + * + * @ingroup MPI + */ + template< class JobData > + class MultiJob : public Job< JobData > + { + public: + MultiJob ( AssignmentAlgorithm& algo, + int masterRank, + JobStore & store ) + : Job( algo, masterRank, Message::Kill, store ) + { + // empty + } + }; + } + + /** + * @} + */ +} +# endif // __EO_MPI_H__ + diff --git a/eo/src/mpi/eoMpiAssignmentAlgorithm.h b/eo/src/mpi/eoMpiAssignmentAlgorithm.h new file mode 100644 index 000000000..07f674157 --- /dev/null +++ b/eo/src/mpi/eoMpiAssignmentAlgorithm.h @@ -0,0 +1,387 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __MPI_ASSIGNMENT_ALGORITHM_H__ +# define __MPI_ASSIGNMENT_ALGORITHM_H__ + +# include // std::vector +# include "eoMpiNode.h" + +namespace eo +{ + namespace mpi + { + /** + * @brief Constant indicating to use all the resting available workers, in assignment algorithms constructor + * using an interval. + * + * @ingroup MPI + */ + const int REST_OF_THE_WORLD = -1; + + /** + * @brief Contains informations on the available workers and allows to find assignees for jobs. + * + * Available workers are workers who aren't processing anything. When they've received an order, workers switch + * from the state "available" to the state "busy", and the master has to wait for their response for considering + * them available again. + * + * @ingroup MPI + */ + struct AssignmentAlgorithm + { + /** + * @brief Gets the rank of an available worker, so as to send it a task. + * + * @return The MPI rank of an available worker, or -1 if there is no available worker. + */ + virtual int get( ) = 0; + + /** + * @brief Gets the number of total available workers. + * + * Before the first call, it is equal to the total number of present workers, as specified in the + * specific assignment algorithm constructor. It allows the Job class to know when all the responses have + * been received, by comparing this number to the total number of workers. + * + * @return Integer indicating how many workers are available. + */ + virtual int availableWorkers( ) = 0; + + /** + * @brief Reinject the worker of indicated rank in the available state. + * + * @param wrkRank The MPI rank of the worker who has finished its job. + */ + virtual void confirm( int wrkRank ) = 0; + + /** + * @brief Indicates who are the workers which do nothing. + * + * At the end of the algorithm, the master has to warn all the workers that it's done. All the workers mean, + * the workers which are currently processing data, and the other ones who could be waiting : the idles. + * This function indicates to the master which worker aren't doing anything. + * + * @return A std::vector containing all the MPI ranks of the idles workers. + */ + virtual std::vector idles( ) = 0; + + /** + * @brief Reinitializes the assignment algorithm with the right number of runs. + * + * In fact, this is only useful for static assignment algorithm, which has to be reinitialized every time + * it's used, in the case of a Multi Job. It's the user's responsability to call this function. + * + * @todo Not really clean. Find a better way to do it. + */ + virtual void reinit( int runs ) = 0; + }; + + /** + * @brief Assignment (scheduling) algorithm which handles workers in a queue. + * + * With this assignment algorithm, workers are put in a queue and may be called an unlimited number of times. + * Whenever a worker returns, it is added to the queue, and it becomes available for the next call to get(). + * The available workers are all located in the queue at any time, so the number of available workers is + * directly equal to the size of the queue. + * + * This kind of assignment is adapted for tasks whose execution time is stochastic or unknown, but without any + * warranty to be faster than other assignments. + * + * @ingroup MPI + */ + struct DynamicAssignmentAlgorithm : public AssignmentAlgorithm + { + public: + + /** + * @brief Uses all the hosts whose rank is higher to 1, inclusive, as workers. + */ + DynamicAssignmentAlgorithm( ) + { + for(int i = 1; i < Node::comm().size(); ++i) + { + availableWrk.push_back( i ); + } + } + + /** + * @brief Uses the unique host with given rank as a worker. + * + * @param unique MPI rank of the unique worker. + */ + DynamicAssignmentAlgorithm( int unique ) + { + availableWrk.push_back( unique ); + } + + /** + * @brief Uses the workers whose ranks are present in the argument as workers. + * + * @param workers std::vector containing MPI ranks of workers. + */ + DynamicAssignmentAlgorithm( const std::vector & workers ) + { + availableWrk = workers; + } + + /** + * @brief Uses a range of ranks as workers. + * + * @param first The first worker to be included (inclusive) + * @param last The last worker to be included (inclusive). If last == eo::mpi::REST_OF_THE_WORLD, all + * hosts whose rank is higher than first are taken. + */ + DynamicAssignmentAlgorithm( int first, int last ) + { + if( last == REST_OF_THE_WORLD ) + { + last = Node::comm().size() - 1; + } + + for( int i = first; i <= last; ++i) + { + availableWrk.push_back( i ); + } + } + + virtual int get( ) + { + int assignee = -1; + if (! availableWrk.empty() ) + { + assignee = availableWrk.back(); + availableWrk.pop_back(); + } + return assignee; + } + + int availableWorkers() + { + return availableWrk.size(); + } + + void confirm( int rank ) + { + availableWrk.push_back( rank ); + } + + std::vector idles( ) + { + return availableWrk; + } + + void reinit( int _ ) + { + ++_; + // nothing to do + } + + protected: + std::vector< int > availableWrk; + }; + + /** + * @brief Assignment algorithm which gives to each worker a precise number of tasks to do, in a round robin + * fashion. + * + * This scheduling algorithm attributes, at initialization or when calling reinit(), a fixed amount of runs to + * distribute to the workers. The amount of runs is then equally distributed between all workers ; if total + * number of runs is not a direct multiple of workers number, then remainding unaffected runs are distributed to + * workers from the first to the last, in a round-robin fashion. + * + * This scheduling should be used when the amount of runs can be computed or is fixed and when we guess that the + * duration of processing task will be the same for each run. There is no warranty that this algorithm is more + * or less efficient that another one. When having a doubt, use DynamicAssignmentAlgorithm. + * + * @ingroup MPI + */ + struct StaticAssignmentAlgorithm : public AssignmentAlgorithm + { + public: + /** + * @brief Uses a given precise set of workers. + * + * @param workers std::vector of MPI ranks of workers which will be used. + * @param runs Fixed amount of runs, strictly positive. + */ + StaticAssignmentAlgorithm( std::vector& workers, int runs ) + { + init( workers, runs ); + } + + /** + * @brief Uses a range of workers. + * + * @param first The first MPI rank of worker to use + * @param last The last MPI rank of worker to use. If it's equal to REST_OF_THE_WORLD, then all the + * workers from the first one are taken as workers. + * @param runs Fixed amount of runs, strictly positive. + */ + StaticAssignmentAlgorithm( int first, int last, int runs ) + { + std::vector workers; + + if( last == REST_OF_THE_WORLD ) + { + last = Node::comm().size() - 1; + } + + for(int i = first; i <= last; ++i) + { + workers.push_back( i ); + } + init( workers, runs ); + } + + /** + * @brief Uses all the hosts whose rank is higher than 1 (inclusive) as workers. + * + * @param runs Fixed amount of runs, strictly positive. If it's not set, you'll have to call reinit() + * later. + */ + StaticAssignmentAlgorithm( int runs = 0 ) + { + std::vector workers; + for(int i = 1; i < Node::comm().size(); ++i) + { + workers.push_back( i ); + } + + init( workers, runs ); + } + + /** + * @brief Uses an unique host as worker. + * + * @param unique The MPI rank of the host which will be the worker. + * @param runs Fixed amount of runs, strictly positive. + */ + StaticAssignmentAlgorithm( int unique, int runs ) + { + std::vector workers; + workers.push_back( unique ); + init( workers, runs ); + } + + private: + /** + * @brief Initializes the static scheduling. + * + * Gives to each worker an equal attribution number, equal to runs / workers.size(), eventually plus one + * if number of workers is not a divisor of runs. + * + * @param workers Vector of hosts' ranks + * @param runs Fixed amount of runs, strictly positive. + */ + void init( std::vector & workers, int runs ) + { + unsigned int nbWorkers = workers.size(); + freeWorkers = nbWorkers; + + busy.clear(); + busy.resize( nbWorkers, false ); + realRank = workers; + + if( runs <= 0 ) + { + return; + } + + attributions.clear(); + attributions.reserve( nbWorkers ); + + // Let be the euclidean division of runs by nbWorkers : + // runs == q * nbWorkers + r, 0 <= r < nbWorkers + // This one liner affects q requests to each worker + for (unsigned int i = 0; i < nbWorkers; attributions[i++] = runs / nbWorkers) ; + // The first line computes r and the one liner affects the remaining + // r requests to workers, in ascending order + unsigned int diff = runs - (runs / nbWorkers) * nbWorkers; + for (unsigned int i = 0; i < diff; ++attributions[i++]); + } + + public: + int get( ) + { + int assignee = -1; + for( unsigned i = 0; i < busy.size(); ++i ) + { + if( !busy[i] && attributions[i] > 0 ) + { + busy[i] = true; + --freeWorkers; + assignee = realRank[ i ]; + break; + } + } + return assignee; + } + + int availableWorkers( ) + { + return freeWorkers; + } + + std::vector idles() + { + std::vector ret; + for(unsigned int i = 0; i < busy.size(); ++i) + { + if( !busy[i] ) + { + ret.push_back( realRank[i] ); + } + } + return ret; + } + + void confirm( int rank ) + { + int i = -1; // i is the real index in table + for( unsigned int j = 0; j < realRank.size(); ++j ) + { + if( realRank[j] == rank ) + { + i = j; + break; + } + } + + --attributions[ i ]; + busy[ i ] = false; + ++freeWorkers; + } + + void reinit( int runs ) + { + init( realRank, runs ); + } + + private: + std::vector attributions; + std::vector realRank; + std::vector busy; + unsigned int freeWorkers; + }; + } +} +# endif // __MPI_ASSIGNMENT_ALGORITHM_H__ diff --git a/eo/src/mpi/eoMpiNode.h b/eo/src/mpi/eoMpiNode.h new file mode 100644 index 000000000..27c03312c --- /dev/null +++ b/eo/src/mpi/eoMpiNode.h @@ -0,0 +1,71 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __MPI_NODE_H__ +# define __MPI_NODE_H__ + +# include +namespace bmpi = boost::mpi; + +namespace eo +{ + namespace mpi + { + /** + * @brief Global object used to reach boost::mpi::communicator everywhere. + * + * boost::mpi::communicator is the main object used to send and receive messages between the different hosts of + * a MPI algorithm. + * + * @ingroup MPI + */ + class Node + { + public: + + /** + * @brief Initializes the MPI environment with argc and argv. + * + * Should be called at the beginning of every parallel program. + * + * @param argc Main's argc + * @param argv Main's argv + */ + static void init( int argc, char** argv ) + { + static bmpi::environment env( argc, argv ); + } + + /** + * @brief Returns the global boost::mpi::communicator + */ + static bmpi::communicator& comm() + { + return _comm; + } + + protected: + static bmpi::communicator _comm; + }; + } +} +# endif // __MPI_NODE_H__ + diff --git a/eo/src/mpi/eoParallelApply.h b/eo/src/mpi/eoParallelApply.h new file mode 100644 index 000000000..6b58aaf1b --- /dev/null +++ b/eo/src/mpi/eoParallelApply.h @@ -0,0 +1,387 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EO_PARALLEL_APPLY_H__ +# define __EO_PARALLEL_APPLY_H__ + +# include "eoMpi.h" + +# include // eoUF +# include // std::vector population + +/** + * @file eoParallelApply.h + * + * @brief Applies a functor with single parameter to elements of a table, in a parallel fashion. + * + * This file contains all the required classes to do a parallel apply of a table, in a parallel fashion. This can be + * very useful when applying the function can be made without any dependances within the data. In EO, it occurs in + * particular during the evaluation: the number of individuals to evaluate can be really high, and the evaluation of one + * individual is independant from the evaluation of other individuals. + * + * Elements in the table are directly replaced, as the table is given by reference. No new table is made during the + * process. + * + * User can tune this job, indicating how many elements of the table should be sent and evaluated by a worker, at a + * time: this is called the "packet size", as individuals are groupped into a packet of individuals which are sent to + * the worker before evaluation. The problem of choosing the optimal packet size is beyond the purposes of this documentation + * and deserves a theoritical study. + * + * This job is the parallel equivalent to the function apply, defined in apply.h. It just applies the function to + * every element of a table. In Python or Javascript, it's the equivalent of the function Map. + */ + +namespace eo +{ + namespace mpi + { + /** + * @brief Structure used to save assignment to a worker, i.e which slice of the table it has to process. + * + * This slice is defined by the index of the first evaluated argument and the number of processed elements. + */ + struct ParallelApplyAssignment + { + int index; + int size; + }; + + /** + * @brief Data useful for a parallel apply (map). + * + * A parallel apply needs at least the functor to apply to every element of the table, and the table itself, + * whereas it can be set later with the function init(). Master rank is also needed, to send it informations and + * receive informations from it, inside the functors (the job knows these values, but the functors don't). The + * size of a packet can be tuned here. + * + * Internal attributes contain: + * - (useful for master) the index of the next element to be evaluated. + * - (useful for master) a map containing links between MPI ranks and slices of the table which the worker with + * this rank has evaluated. Without this map, when receiving results from a worker, the master couldn't be + * able to replace the right elements in the table. + * + * @ingroup MPI + */ + template + struct ParallelApplyData + { + /** + * @brief Ctor for Parallel apply (map) data. + * + * @param _proc The functor to apply on each element in the table + * @param _masterRank The MPI rank of the master + * @param _packetSize The number of elements on which the function will be applied by the worker, at a time. + * @param table The table to apply. If this value is NULL, user will have to call init() before launching the + * job. + */ + ParallelApplyData( + eoUF & _proc, + int _masterRank, + int _packetSize, + std::vector * table = 0 + ) : + _table( table ), func( _proc ), index( 0 ), packetSize( _packetSize ), masterRank( _masterRank ), comm( Node::comm() ) + { + if ( _packetSize <= 0 ) + { + throw std::runtime_error("Packet size should not be negative."); + } + + if( table ) + { + size = table->size(); + } + } + + /** + * @brief Reinitializes the data for a new table to evaluate. + */ + void init( std::vector& table ) + { + index = 0; + size = table.size(); + _table = &table; + assignedTasks.clear(); + } + + std::vector& table() + { + return *_table; + } + + // All elements are public since functors will often use them. + std::vector * _table; + eoUF & func; + int index; + int size; + std::map< int /* worker rank */, ParallelApplyAssignment /* last assignment */> assignedTasks; + int packetSize; + std::vector tempArray; + + int masterRank; + bmpi::communicator& comm; + }; + + /** + * @brief Send task functor implementation for the parallel apply (map) job. + * + * Master side: Sends a slice of the table to evaluate to the worker. + * + * Implementation details: + * Finds the next slice of data to send to the worker, sends first the size and then the data, and memorizes + * that this slice has been distributed to the worker, then updates the next position of element to evaluate. + */ + template< class EOT > + class SendTaskParallelApply : public SendTaskFunction< ParallelApplyData > + { + public: + using SendTaskFunction< ParallelApplyData >::_data; + + SendTaskParallelApply( SendTaskParallelApply * w = 0 ) : SendTaskFunction< ParallelApplyData >( w ) + { + // empty + } + + void operator()(int wrkRank) + { + int futureIndex; + + if( _data->index + _data->packetSize < _data->size ) + { + futureIndex = _data->index + _data->packetSize; + } else { + futureIndex = _data->size; + } + + int sentSize = futureIndex - _data->index ; + + _data->comm.send( wrkRank, 1, sentSize ); + + eo::log << eo::progress << "Evaluating individual " << _data->index << std::endl; + + _data->assignedTasks[ wrkRank ].index = _data->index; + _data->assignedTasks[ wrkRank ].size = sentSize; + + _data->comm.send( wrkRank, 1, & ( (_data->table())[ _data->index ] ) , sentSize ); + _data->index = futureIndex; + } + }; + + /** + * @brief Handle response functor implementation for the parallel apply (map) job. + * + * Master side: Replaces the slice of data attributed to the worker in the table. + */ + template< class EOT > + class HandleResponseParallelApply : public HandleResponseFunction< ParallelApplyData > + { + public: + using HandleResponseFunction< ParallelApplyData >::_data; + + HandleResponseParallelApply( HandleResponseParallelApply * w = 0 ) : HandleResponseFunction< ParallelApplyData >( w ) + { + // empty + } + + void operator()(int wrkRank) + { + _data->comm.recv( wrkRank, 1, & (_data->table()[ _data->assignedTasks[wrkRank].index ] ), _data->assignedTasks[wrkRank].size ); + } + }; + + /** + * @brief Process task functor implementation for the parallel apply (map) job. + * + * Worker side: apply the function to the given slice of data. + * + * Implementation details: retrieves the number of elements to evaluate, retrieves them, applies the function + * and then returns the results. + */ + template< class EOT > + class ProcessTaskParallelApply : public ProcessTaskFunction< ParallelApplyData > + { + public: + using ProcessTaskFunction< ParallelApplyData >::_data; + + ProcessTaskParallelApply( ProcessTaskParallelApply * w = 0 ) : ProcessTaskFunction< ParallelApplyData >( w ) + { + // empty + } + + void operator()() + { + int recvSize; + + _data->comm.recv( _data->masterRank, 1, recvSize ); + _data->tempArray.resize( recvSize ); + _data->comm.recv( _data->masterRank, 1, & _data->tempArray[0] , recvSize ); + timerStat.start("worker_processes"); + for( int i = 0; i < recvSize ; ++i ) + { + _data->func( _data->tempArray[ i ] ); + } + timerStat.stop("worker_processes"); + _data->comm.send( _data->masterRank, 1, & _data->tempArray[0], recvSize ); + } + }; + + /** + * @brief Is finished functor implementation for the parallel apply (map) job. + * + * Master side: returns true if and only if the whole table has been evaluated. The job is also terminated only + * when the whole table has been evaluated. + */ + template< class EOT > + class IsFinishedParallelApply : public IsFinishedFunction< ParallelApplyData > + { + public: + using IsFinishedFunction< ParallelApplyData >::_data; + + IsFinishedParallelApply( IsFinishedParallelApply * w = 0 ) : IsFinishedFunction< ParallelApplyData >( w ) + { + // empty + } + + bool operator()() + { + return _data->index == _data->size; + } + }; + + /** + * @brief Store containing all the datas and the functors for the parallel apply (map) job. + * + * User can tune functors when constructing the object. For each functor which is not given, a default one is + * generated. + * + * @ingroup MPI + */ + template< class EOT > + struct ParallelApplyStore : public JobStore< ParallelApplyData > + { + using JobStore< ParallelApplyData >::_stf; + using JobStore< ParallelApplyData >::_hrf; + using JobStore< ParallelApplyData >::_ptf; + using JobStore< ParallelApplyData >::_iff; + + /** + * @brief Main constructor for the parallel apply (map) job. + * + * @param _proc The procedure to apply to each element of the table. + * @param _masterRank The rank of the master process. + * @param _packetSize The number of elements of the table to be evaluated at a time, by the worker. + * @param stpa Pointer to Send Task parallel apply functor descendant. If null, a default one is used. + * @param hrpa Pointer to Handle Response parallel apply functor descendant. If null, a default one is used. + * @param ptpa Pointer to Process Task parallel apply functor descendant. If null, a default one is used. + * @param ifpa Pointer to Is Finished parallel apply functor descendant. If null, a default one is used. + */ + ParallelApplyStore( + eoUF & _proc, + int _masterRank, + int _packetSize = 1, + // JobStore functors + SendTaskParallelApply * stpa = 0, + HandleResponseParallelApply* hrpa = 0, + ProcessTaskParallelApply* ptpa = 0, + IsFinishedParallelApply* ifpa = 0 + ) : + _data( _proc, _masterRank, _packetSize ) + { + if( stpa == 0 ) { + stpa = new SendTaskParallelApply; + stpa->needDelete( true ); + } + + if( hrpa == 0 ) { + hrpa = new HandleResponseParallelApply; + hrpa->needDelete( true ); + } + + if( ptpa == 0 ) { + ptpa = new ProcessTaskParallelApply; + ptpa->needDelete( true ); + } + + if( ifpa == 0 ) { + ifpa = new IsFinishedParallelApply; + ifpa->needDelete( true ); + } + + _stf = stpa; + _hrf = hrpa; + _ptf = ptpa; + _iff = ifpa; + } + + ParallelApplyData* data() { return &_data; } + + /** + * @brief Reinits the store with a new table to evaluate. + * + * @param _pop The table of elements to be evaluated. + */ + void data( std::vector& _pop ) + { + _data.init( _pop ); + } + + virtual ~ParallelApplyStore() // for inheritance purposes only + { + } + + protected: + ParallelApplyData _data; + }; + + /** + * @brief Parallel apply job. Present for convenience only. + * + * A typedef wouldn't have been working, as typedef on templates don't work in C++. Traits would be a + * disgraceful overload for the user. + * + * @ingroup MPI + * @see eoParallelApply.h + */ + template< typename EOT > + class ParallelApply : public MultiJob< ParallelApplyData > + { + public: + + ParallelApply( + AssignmentAlgorithm & algo, + int _masterRank, + ParallelApplyStore & store + ) : + MultiJob< ParallelApplyData >( algo, _masterRank, store ) + { + // empty + } + }; + + /** + * @example t-mpi-parallelApply.cpp + * @example t-mpi-multipleRoles.cpp + */ + } +} +# endif // __EO_PARALLEL_APPLY_H__ + + diff --git a/eo/src/mpi/eoTerminateJob.h b/eo/src/mpi/eoTerminateJob.h new file mode 100644 index 000000000..fe231f0e3 --- /dev/null +++ b/eo/src/mpi/eoTerminateJob.h @@ -0,0 +1,140 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EO_TERMINATE_H__ +# define __EO_TERMINATE_H__ + +# include "eoMpi.h" + +namespace eo +{ + namespace mpi + { + /** + * @ingroup MPI + * @{ + */ + + /** + * @brief Send task functor which does nothing. + */ + struct DummySendTaskFunction : public SendTaskFunction + { + void operator()( int _ ) + { + ++_; + } + }; + + /** + * @brief Handle response functor which does nothing. + */ + struct DummyHandleResponseFunction : public HandleResponseFunction + { + void operator()( int _ ) + { + ++_; + } + }; + + /** + * @brief Process task functor which does nothing. + */ + struct DummyProcessTaskFunction : public ProcessTaskFunction + { + void operator()() + { + // nothing! + } + }; + + /** + * @brief Is finished functor which returns true everytime. + */ + struct DummyIsFinishedFunction : public IsFinishedFunction + { + bool operator()() + { + return true; + } + }; + + /** + * @brief Job store containing all dummy functors and containing no data. + */ + struct DummyJobStore : public JobStore + { + using JobStore::_stf; + using JobStore::_hrf; + using JobStore::_ptf; + using JobStore::_iff; + + DummyJobStore() + { + _stf = new DummySendTaskFunction; + _stf->needDelete( true ); + _hrf = new DummyHandleResponseFunction; + _hrf->needDelete( true ); + _ptf = new DummyProcessTaskFunction; + _ptf->needDelete( true ); + _iff = new DummyIsFinishedFunction; + _iff->needDelete( true ); + } + + void* data() { return 0; } + }; + + /** + * @brief Job to run after a Multi Job, so as to indicate that every workers should terminate. + */ + struct EmptyJob : public OneShotJob + { + /** + * @brief Main EmptyJob ctor + * + * @param algo Assignment (scheduling) algorithm used. + * @param masterRank The rank of the master process. + */ + EmptyJob( AssignmentAlgorithm& algo, int masterRank ) : + OneShotJob( algo, masterRank, *(new DummyJobStore) ) + // the job store is deleted on destructor + { + // empty + } + + ~EmptyJob() + { + std::vector< int > idles = assignmentAlgo.idles(); + for(unsigned i = 0, size = idles.size(); i < size; ++i) + { + comm.send( idles[i], Channel::Commands, Message::Kill ); + } + delete & this->store; + } + }; + + /** + * @} + */ + } +} + +# endif // __EO_TERMINATE_H__ diff --git a/eo/src/serial/Array.cpp b/eo/src/serial/Array.cpp new file mode 100644 index 000000000..180aad16b --- /dev/null +++ b/eo/src/serial/Array.cpp @@ -0,0 +1,58 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# include "Array.h" + +namespace eoserial +{ + std::ostream& Array::print( std::ostream& out ) const + { + out << "["; + bool first = true; + for (ArrayChildren::const_iterator it = begin(), + end = this->end(); + it != end; + ++it) + { + if ( first ) + { + first = false; + } else { + out << ", "; + } + (*it)->print( out ); + } + out << "]\n"; + return out; + } + + Array::~Array() + { + for (ArrayChildren::iterator it = begin(), + end = this->end(); + it != end; + ++it) + { + delete *it; + } + } + +} // namespace eoserial diff --git a/eo/src/serial/Array.h b/eo/src/serial/Array.h new file mode 100644 index 000000000..d453add99 --- /dev/null +++ b/eo/src/serial/Array.h @@ -0,0 +1,170 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_ARRAY_H__ +# define __EOSERIAL_ARRAY_H__ + +# include + +# include "Entity.h" +# include "Serializable.h" +# include "Object.h" + +namespace eoserial +{ + + // Forward declaration for below declarations. + class Array; + + /* + * Declarations of functions present in Utils.h + * These are put here to avoid instead of including the file Utils.h, which would + * cause a circular inclusion. + */ + + template< class T > + void unpack( const Array & array, unsigned int index, T & value ); + + void unpackObject( const Array & array, unsigned int index, Persistent & value ); + + template< class Container, template class UnpackAlgorithm > + void unpackArray( const Array & array, unsigned int index, Container & container ); + + /** + * @brief Represents a JSON array. + * + * Wrapper for an array, so as to be used as a JSON object. + * + * @ingroup Serialization + */ + class Array : public eoserial::Entity, public std::vector< eoserial::Entity* > + { + protected: + typedef std::vector< eoserial::Entity* > ArrayChildren; + + public: + /** + * @brief Adds the serializable object as a JSON object. + * @param obj Object which implemnets JsonSerializable. + */ + void push_back( const eoserial::Printable* obj ) + { + ArrayChildren::push_back( obj->pack() ); + } + + /** + * @brief Proxy for vector::push_back. + */ + void push_back( eoserial::Entity* json ) + { + ArrayChildren::push_back( json ); + } + + /** + * @brief Prints the JSON array into the given stream. + * @param out The stream + */ + virtual std::ostream& print( std::ostream& out ) const; + + /** + * @brief Dtor + */ + ~Array(); + + /* + * The following parts allows the user to automatically deserialize an eoserial::Array into a + * standard container, by giving the algorithm which will be used to deserialize contained entities. + */ + + /** + * @brief Functor which determines how to retrieve the real value contained in a eoserial::Entity at + * a given place. + * + * It will be applied for each contained variable in the array. + */ + template + struct BaseAlgorithm + { + /** + * @brief Main operator. + * + * @param array The eoserial::Array from which we're reading. + * @param i The index of the contained value. + * @param container The standard (STL) container in which we'll push back the read value. + */ + virtual void operator()( const eoserial::Array& array, unsigned int i, Container & container ) const = 0; + }; + + /** + * @brief BaseAlgorithm for retrieving primitive variables. + * + * This one should be used to retrieve primitive (and types which implement operator>>) variables, for instance + * int, double, std::string, etc... + */ + template + struct UnpackAlgorithm : public BaseAlgorithm + { + void operator()( const eoserial::Array& array, unsigned int i, C & container ) const + { + typename C::value_type t; + unpack( array, i, t ); + container.push_back( t ); + } + }; + + /** + * @brief BaseAlgorithm for retrieving eoserial::Persistent objects. + * + * This one should be used to retrieve objects which implement eoserial::Persistent. + */ + template + struct UnpackObjectAlgorithm : public BaseAlgorithm + { + void operator()( const eoserial::Array& array, unsigned int i, C & container ) const + { + typename C::value_type t; + unpackObject( array, i, t ); + container.push_back( t ); + } + }; + + /** + * @brief General algorithm for array deserialization. + * + * Applies the BaseAlgorithm to each contained variable in the eoserial::Array. + */ + template class UnpackAlgorithm> + inline void deserialize( Container & array ) + { + UnpackAlgorithm< Container > algo; + for( unsigned int i = 0, size = this->size(); + i < size; + ++i) + { + algo( *this, i, array ); + } + } + }; + +} // namespace eoserial + +# endif // __EOSERIAL_ARRAY_H__ + diff --git a/eo/src/serial/CMakeLists.txt b/eo/src/serial/CMakeLists.txt new file mode 100644 index 000000000..5358b30a4 --- /dev/null +++ b/eo/src/serial/CMakeLists.txt @@ -0,0 +1,35 @@ +###################################################################################### +### 1) Include the sources +###################################################################################### + +INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/src) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +###################################################################################### +### 2) Define the eoserial target +###################################################################################### + +SET(EOSERIAL_LIB_OUTPUT_PATH ${EO_BINARY_DIR}/lib) +SET(LIBRARY_OUTPUT_PATH ${EOSERIAL_LIB_OUTPUT_PATH}) + +SET(EOSERIAL_SOURCES + Array.cpp + Object.cpp + Parser.cpp + String.cpp + ) + +ADD_LIBRARY(eoserial STATIC ${EOSERIAL_SOURCES}) +INSTALL(TARGETS eoserial ARCHIVE DESTINATION lib COMPONENT libraries) + +FILE(GLOB HDRS *.h) +INSTALL(FILES ${HDRS} DESTINATION include/eo/serial COMPONENT headers) + +###################################################################################### +### 3) Optionnal +###################################################################################### + +SET(EOSERIAL_VERSION ${GLOBAL_VERSION}) +SET_TARGET_PROPERTIES(eoserial PROPERTIES VERSION "${EOSERIAL_VERSION}") + +###################################################################################### diff --git a/eo/src/serial/Entity.h b/eo/src/serial/Entity.h new file mode 100644 index 000000000..50155e13a --- /dev/null +++ b/eo/src/serial/Entity.h @@ -0,0 +1,70 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_ENTITY_H__ +# define __EOSERIAL_ENTITY_H__ + +# include // ostream + + +/** + * @brief Contains all the necessary entities to serialize eo objects into JSON objects. + * + * Allows serialization from user objects into JSON objects, if they implement the interface + * eoserial::Serializable or eoserial::Persistent. The following user objects can be serialized: + * - primitive types (int, std::string, ...), in particular every type that can be written into a + * std::stringstream. + * - objects which implement eoserial::Serializable. + * - array of serializable things (primitive or serializable objects). + * + * @ingroup Utilities + * @defgroup Serialization Serialization helpers +**/ +namespace eoserial +{ + +/** + * @brief JSON entity + * + * This class represents a JSON entity, which can be JSON objects, + * strings or arrays. It is the base class for the JSON hierarchy. + * + * @ingroup Serialization + */ +class Entity +{ +public: + + /** + * Virtual dtor (base class). + */ + virtual ~Entity() { /* empty */ } + + /** + * @brief Prints the content of a JSON object into a stream. + * @param out The stream in which we're printing. + */ + virtual std::ostream& print( std::ostream& out ) const = 0; +}; + +} // namespace eoserial + +# endif // __ENTITY_H__ diff --git a/eo/src/serial/Object.cpp b/eo/src/serial/Object.cpp new file mode 100644 index 000000000..dd859052e --- /dev/null +++ b/eo/src/serial/Object.cpp @@ -0,0 +1,60 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# include "Object.h" + +using namespace eoserial; + +namespace eoserial +{ + std::ostream& Object::print( std::ostream& out ) const + { + out << '{'; + bool first = true; + for(JsonValues::const_iterator it = begin(), end = this->end(); + it != end; + ++it) + { + if ( first ) + { + first = false; + } else { + out << ", "; + } + + out << '"' << it->first << "\":"; // key + it->second->print( out ); // value + } + out << "}\n"; + return out; + } + + Object::~Object() + { + for(JsonValues::iterator it = begin(), end = this->end(); + it != end; + ++it) + { + delete it->second; + } + } + +} // namespace eoserial diff --git a/eo/src/serial/Object.h b/eo/src/serial/Object.h new file mode 100644 index 000000000..9424f1c4c --- /dev/null +++ b/eo/src/serial/Object.h @@ -0,0 +1,88 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_OBJECT_H__ +# define __EOSERIAL_OBJECT_H__ + +# include +# include + +# include "Entity.h" +# include "Serializable.h" + +namespace eoserial +{ + /** + * @brief JSON Object + * + * This class represents a JSON object, which is basically a dictionnary + * of keys (strings) and values (JSON entities). + * + * @ingroup Serialization + */ + class Object : public eoserial::Entity, public std::map< std::string, eoserial::Entity* > + { + public: + typedef std::map JsonValues; + + /** + * @brief Adds a pair into the JSON object. + * @param key The key associated with the eoserial object + * @param json The JSON object as created with framework. + */ + void add( const std::string& key, eoserial::Entity* json ) + { + (*this)[ key ] = json; + } + + /** + * @brief Adds a pair into the JSON object. + * @param key The key associated with the eoserial object + * @param obj A JSON-serializable object + */ + void add( const std::string& key, const eoserial::Printable* obj ) + { + (*this)[ key ] = obj->pack(); + } + + /** + * @brief Deserializes a Serializable class instance from this JSON object. + * @param obj The object we want to rebuild. + */ + void deserialize( eoserial::Persistent & obj ) + { + obj.unpack( this ); + } + + /** + * @brief Dtor + */ + ~Object(); + + /** + * @brief Prints the content of a JSON object into a stream. + */ + virtual std::ostream& print( std::ostream& out ) const; + }; + +} // namespace eoserial +# endif // __EOSERIAL_OBJECT_H__ + diff --git a/eo/src/serial/Parser.cpp b/eo/src/serial/Parser.cpp new file mode 100644 index 000000000..258174c86 --- /dev/null +++ b/eo/src/serial/Parser.cpp @@ -0,0 +1,171 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# include + +# include "Parser.h" + +# include "Array.h" +# include "Object.h" +# include "String.h" + +// in debug mode only +// # define DEBUG(x) std::cout << x << std::endl; +# define DEBUG(x) + +using namespace eoserial; + +namespace eoserial +{ + +/** + * @brief Parses a string contained between double quotes. + * + * Strings can contain escaped double quotes. + * @param str The string we're parsing. + * @param pos The index of current position in parsed string. + * This index will be updated so as to allow the parser to + * continue. + */ +static std::string parseString(const std::string& str, size_t & pos) +{ + // example : "hello" + // example 2 : "\"world\"" + // for hello: + // firstQuote == 0, secondQuote == 6 + // sub string should be from firstQuote+1 to secondQuote-1 + // so its size should be (secondQuote-1 -(firstQuote+1) + 1) + std::string value; + size_t firstQuote = str.find( '"', pos ); + size_t secondQuote; + + /* instead of just seeking the second quote, we need to ensure + // that there is no escaped quote before this one. + // actually this is harder than that. Using backslashes + // to escape double quotes mean that backslashes have to be + // escaped to. + // example : "text\\" to symbolize : text\ + // example : "text\\\" to symbolize : text\" + // In fact, we should find if number of backslashes is odd; in this case, + // the double quotes are escaped and we should find the next one. + */ + int backslashesCount; + do { + ++pos; + secondQuote = str.find( '"', pos ); + size_t i = secondQuote - 1; + + // Find the backslashes + backslashesCount = 0; + while ( str[ i ] == '\\' ) + { + --i; + ++backslashesCount; + } + pos = secondQuote; + } while( backslashesCount % 2 == 1 ); + + value = str.substr( firstQuote+1, secondQuote-firstQuote-1 ); + pos = secondQuote + 1; + return value; +} + +/** + * @brief Moves the given index pos to the next character which is + * neither a coma, a space nor a new line. + * + * @param str The string in which we want to ignores those characters. + * @param pos The index of current position in parsed string. + */ +static void ignoreChars(const std::string& str, size_t & pos) +{ + // ignore white spaces and comas + for (char current = str[ pos ]; + current == ',' || current == ' ' || current == '\n'; + current = str[ ++pos ]); +} + +String* Parser::parseJsonString(const std::string & str, size_t & pos) +{ + return new String( parseString( str, pos ) ); +} + +Object* Parser::parse(const std::string & str) +{ + size_t initial(0); // we begin at position 0 + return static_cast( parseRight(str, initial) ); +} + +Entity* Parser::parseRight(const std::string & str, size_t & pos) +{ + Entity* value = 0; + + if ( str[ pos ] == '{' ) + { + // next one is an object + DEBUG("We read an object.") + Object* obj = new Object; + pos += 1; + while( pos < str.size() && str[ pos ] != '}' ) + { + parseLeft( str, pos, obj ); + ignoreChars( str, pos ); + } + DEBUG("We just finished to read an object ! ") + pos += 1; // we're on the }, go to the next char + value = obj; + } + else if ( str[ pos ] == '"' ) + { + // next one is a string + DEBUG("We read a string") + value = parseJsonString( str, pos ); + } + else if ( str[ pos ] == '[' ) + { + // next one is an array + DEBUG("We read an array") + Array* array = new Array; + pos += 1; + while( pos < str.size() && str[ pos ] != ']' ) + { + Entity* child = parseRight( str, pos ); + if ( child ) + array->push_back( child ); + } + DEBUG("We've finished to read our array.") + pos += 1; // we're on the ], go to the next char + value = array; + } + ignoreChars( str, pos ); + return value; +} + +void Parser::parseLeft(const std::string & str, size_t & pos, Object* eoserial) +{ + std::string key = parseString(str, pos); + ++pos; // the colon + DEBUG("We've read the key ") + (*eoserial)[ key ] = parseRight( str, pos ); +} + +} // namespace eoserial + diff --git a/eo/src/serial/Parser.h b/eo/src/serial/Parser.h new file mode 100644 index 000000000..20f6a1bb2 --- /dev/null +++ b/eo/src/serial/Parser.h @@ -0,0 +1,103 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_PARSER_H__ +# define __EOSERIAL_PARSER_H__ + +# include "Entity.h" +# include "String.h" +# include "Object.h" + +/** + * @file Parser.h + * + * This file contains a tiny JSON parser used in DAE. This parser just handles + * a subset of JSON grammar, with the following restrictions : + * - all strings must be surrounded by double quotes. + * - everything which is not an object or an array is considered to be a string + * (even integers, booleans,...). + * - no syntax check is done. We trust the programmer and he has to ensure that + * every JSON string he produces is valid. + * + * @author Benjamin BOUVIER + */ + +namespace eoserial +{ + +/** + * @brief Parser from a JSON source. + * + * This parser does just retrieve values and does NOT check the structure of + * the input. This implies that if the input is not correct, the result is undefined + * and can result to a failure on execution. + * + * @ingroup Serialization + */ +class Parser +{ + public: + + /** + * @brief Parses the given string and returns the JSON object read. + */ + static eoserial::Object* parse(const std::string & str); + + protected: + + /** + * @brief Parses the right part of a JSON object as a string. + * + * The right part of an object can be a string (for instance : + * "key":"value"), a JSON array (for instance: "key":["1"]) or + * another JSON object (for instance: "key":{"another_key":"value"}). + * + * The right parts are found after keys (which are parsed by parseLeft) + * and in arrays. + * + * @param str The string we're parsing. + * @param pos The index of the current position in the string. + * @return The JSON object matching the right part. + */ + static eoserial::Entity* parseRight(const std::string & str, size_t & pos); + + /** + * @brief Parses the left value of a key-value pair, which is the key. + * + * @param str The string we're parsing. + * @param pos The index of the current position in the string. + * @param json The current JSON object for which we're adding a key-value pair. + */ + static void parseLeft(const std::string & str, size_t & pos, eoserial::Object* json); + + /** + * @brief Retrieves a string in a JSON content. + * + * @param str The string we're parsing. + * @param pos The index of the current position of parsing, + * which will be updated. + */ + static eoserial::String* parseJsonString(const std::string & str, size_t & pos); +}; + +} // namespace eoserial + +# endif // __EOSERIAL_PARSER_H__ diff --git a/eo/src/serial/Serializable.h b/eo/src/serial/Serializable.h new file mode 100644 index 000000000..715e9c973 --- /dev/null +++ b/eo/src/serial/Serializable.h @@ -0,0 +1,66 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_SERIALIZABLE_H__ +# define __EOSERIAL_SERIALIZABLE_H__ + +namespace eoserial +{ + class Object; // to avoid recursive inclusion with JsonObject + + /** + * @brief Interface showing that object can be written to a eoserial type + * (currently JSON). + * + * @ingroup Serialization + */ + class Printable + { + public: + /** + * @brief Serializes the object to JSON format. + * @return A JSON object created with new. + */ + virtual eoserial::Object* pack() const = 0; + }; + + /** + * @brief Interface showing that object can be eoserialized (written and read + * from an input). + * + * Note : Persistent objects should have a default non-arguments constructor. + * + * @ingroup Serialization + */ + class Persistent : public Printable + { + public: + /** + * @brief Loads class fields from a JSON object. + * @param json A JSON object. Programmer doesn't have to delete it, it + * is automatically done. + */ + virtual void unpack(const eoserial::Object* json) = 0; + }; + +} // namespace eoserial + +# endif // __EOSERIAL_SERIALIZABLE_H__ diff --git a/eo/src/serial/String.cpp b/eo/src/serial/String.cpp new file mode 100644 index 000000000..c50882786 --- /dev/null +++ b/eo/src/serial/String.cpp @@ -0,0 +1,32 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# include "String.h" + +namespace eoserial +{ + std::ostream& String::print( std::ostream& out ) const + { + out << '"' << *this << '"'; + return out; + } +} // namespace eoserial + diff --git a/eo/src/serial/String.h b/eo/src/serial/String.h new file mode 100644 index 000000000..526cab365 --- /dev/null +++ b/eo/src/serial/String.h @@ -0,0 +1,103 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_STRING_H__ +# define __EOSERIAL_STRING_H__ + +# include +# include +# include + +# include "Entity.h" + +namespace eoserial +{ + /** + * @brief JSON String + * + * Wrapper for string, so as to be used as a JSON object. + * + * @ingroup Serialization + */ + class String : public eoserial::Entity, public std::string + { + public: + + /** + * @brief Default ctor. + * @param str The string we want to wrap. + */ + String( const std::string& str ) : std::string( str ) {} + + /** + * @brief Ctor used only when parsing. + */ + String( ) {} + + /** + * @brief Prints out the string. + */ + virtual std::ostream& print( std::ostream& out ) const; + + /** + * @brief Deserializes the current String into a given primitive type value. + * @param value The value in which we're writing. + */ + template + inline void deserialize( T & value ); + + protected: + // Copy and reaffectation are forbidden + explicit String( const String& _ ); + String& operator=( const String& _ ); + }; + + /** + * @brief Casts a eoserial::String into a primitive value, or in a type which at + * least overload operator>>. + * + * @param value A reference to the variable we're writing into. + * + * It's not necessary to specify the variable type, which can be infered by compiler when + * invoking. + */ + template + inline void String::deserialize( T & value ) + { + std::stringstream ss; + ss.precision(std::numeric_limits::digits10 + 1); + ss << *this; + ss >> value; + } + + /** + * @brief Specialization for strings, which don't need to be converted through + * a stringstream. + */ + template<> + inline void String::deserialize( std::string & value ) + { + value = *this; + } + +} // namespace eoserial + +# endif // __EOSERIAL_STRING_H__ diff --git a/eo/src/serial/Utils.h b/eo/src/serial/Utils.h new file mode 100644 index 000000000..33172a747 --- /dev/null +++ b/eo/src/serial/Utils.h @@ -0,0 +1,189 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_UTILS_H__ +# define __EOSERIAL_UTILS_H__ + +# include "Array.h" +# include "Object.h" +# include "String.h" + +namespace eoserial +{ + /* *************************** + * DESERIALIZATION FUNCTIONS * + ***************************** + These functions are useful for casting eoserial::objects into simple, primitive + variables or into class instance which implement eoserial::Persistent. + + The model is always quite the same : + - the first argument is the containing object (which is a eoserial::Entity, + an object or an array) + - the second argument is the key or index, + - the last argument is the value in which we're writing. + */ + + template< class T > + inline void unpack( const Object & obj, const std::string & key, T & value ) + { + static_cast( obj.find( key )->second )->deserialize( value ); + } + + inline void unpackObject( const Object & obj, const std::string & key, Persistent & value ) + { + static_cast( obj.find( key )->second )->deserialize( value ); + } + + template< class Container, template class UnpackAlgorithm > + inline void unpackArray( const Object & obj, const std::string & key, Container & array ) + { + static_cast( obj.find( key )->second )->deserialize< Container, UnpackAlgorithm >( array ); + } + + template< class T > + inline void unpack( const Array & array, unsigned int index, T & value ) + { + static_cast( array[ index ] )->deserialize( value ); + } + + inline void unpackObject( const Array & array, unsigned int index, Persistent & value ) + { + static_cast( array[ index ] )->deserialize( value ); + } + + template< class Container, template class UnpackAlgorithm > + inline void unpackArray( const Array & array, unsigned int index, Container & container ) + { + static_cast( array[ index ] )->deserialize< Container, UnpackAlgorithm >( container ); + } + + /* ***************************** + *** SERIALIZATION FUNCTIONS *** + ******************************* + These functions are useful for casting classic objects and + eoserial::Persistent objects into eoserial entities which + can be manipulated by the framework. + */ + + /** + * @brief Casts a value of a stream-serializable type (i.e, which implements + * operator <<) into a JsonString. + * + * This is used when serializing the objects : all primitives types should be + * converted into strings to get more easily manipulated. + * + * @param value The value we're converting. + * @return JsonString wrapper for the value. + */ + template + String* make( const T & value ) + { + std::stringstream ss; + ss.precision(std::numeric_limits::digits10 + 1); + ss << value; + return new String( ss.str() ); + } + + /** + * @brief Specialization for strings : no need to convert as they're still + * usable as strings. + */ + template<> + inline String* make( const std::string & value ) + { + return new String( value ); + } + + /* + * These functions are useful for automatically serializing STL containers into + * eoserial arrays which could be used by the framework. + **/ + + /** + * @brief Functor which explains how to push the value into the eoserial::Array. + */ + template< class T > + struct PushAlgorithm + { + /** + * @brief Main operator. + * + * @param array The eoserial::array in which we're writing. + * @param value The variable we are writing. + */ + virtual void operator()( Array & array, const T & value ) = 0; + }; + + /** + * @brief Push algorithm for primitive variables. + * + * This one should be used when inserting primitive (and types which implement + * operator<<) variables. + */ + template< class T > + struct MakeAlgorithm : public PushAlgorithm + { + void operator()( Array & array, const T & value ) + { + array.push_back( make( value ) ); + } + }; + + /** + * @brief Push algorithm for eoserial::Persistent variables. + */ + template< class T > + struct SerializablePushAlgorithm : public PushAlgorithm + { + void operator()( Array & array, const T & obj ) + { + // obj address is not saved into array.push_back. + array.push_back( &obj ); + } + }; + + /** + * @brief Casts a STL container (vector or list, for instance) + * into a eoserial::Array. + * + * @þaram PushAlgorithm The algorithm used for inserting new element in the eoserial::Array. + * This algorithm is directly called, so it is its own charge to invoke push_back on the + * eoserial::Array. + */ + template< class Container, template class PushAlgorithm > + Array* makeArray( const Container & array ) + { + Array* returned_array = new Array; + typedef typename Container::const_iterator iterator; + typedef typename Container::value_type Type; + PushAlgorithm< Type > algo; + for ( + iterator it = array.begin(), end = array.end(); + it != end; + ++it) + { + algo( *returned_array, *it ); + } + return returned_array; + } +} // namespace eoserial + +# endif //__EOSERIAL_UTILS_H__ diff --git a/eo/src/serial/eoSerial.h b/eo/src/serial/eoSerial.h new file mode 100644 index 000000000..55a116f0c --- /dev/null +++ b/eo/src/serial/eoSerial.h @@ -0,0 +1,33 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EOSERIAL_HEADERS__ +# define __EOSERIAL_HEADERS__ + +# include "Object.h" +# include "Serializable.h" +# include "Array.h" +# include "Object.h" +# include "String.h" +# include "Parser.h" +# include "Utils.h" + +# endif // __EOSERIAL_HEADERS__ diff --git a/eo/src/serial/json_example b/eo/src/serial/json_example new file mode 100644 index 000000000..7ecb3edda --- /dev/null +++ b/eo/src/serial/json_example @@ -0,0 +1,8 @@ +{"a":"b", +"obj": + {"obj_a":"obj_}b","subobj_a": + {"subk":"subv"} + }, +"c":"d", +"array":["1","2",{"\"array\"_obj\"":"array_ov]"}, ["3"], "4"] +} diff --git a/eo/src/utils/eoParallel.cpp b/eo/src/utils/eoParallel.cpp index f8e3c745c..cae5e8c41 100644 --- a/eo/src/utils/eoParallel.cpp +++ b/eo/src/utils/eoParallel.cpp @@ -21,7 +21,7 @@ Contact: http://eodev.sourceforge.net Authors: -Caner Candan + Caner Candan */ @@ -34,11 +34,12 @@ Caner Candan eoParallel::eoParallel() : _isEnabled( false, "parallelize-loop", "Enable memory shared parallelization into evaluation's loops", '\0' ), - _isDynamic( false, "parallelize-dynamic", "Enable dynamic memory shared parallelization", '\0' ), + _isDynamic( true, "parallelize-dynamic", "Enable dynamic memory shared parallelization", '\0' ), _prefix( "results", "parallelize-prefix", "Here's the prefix filename where the results are going to be stored", '\0' ), _nthreads( 0, "parallelize-nthreads", "Define the number of threads you want to use, nthreads = 0 means you want to use all threads available", '\0' ), _enableResults( false, "parallelize-enable-results", "Enable the generation of results", '\0' ), _doMeasure( false, "parallelize-do-measure", "Do some measures during execution", '\0' ), + _packetSize( 1U, "parallelize-packet-size", "Number of elements which should be sent in a single message during a parallel evaluation based on message passing.", '\0'), _t_start(0) { } @@ -92,6 +93,7 @@ void eoParallel::_createParameters( eoParser& parser ) parser.processParam( _nthreads, section ); parser.processParam( _enableResults, section ); parser.processParam( _doMeasure, section ); + parser.processParam( _packetSize, section ); } void make_parallel(eoParser& parser) diff --git a/eo/src/utils/eoParallel.h b/eo/src/utils/eoParallel.h index 3f22f6c43..b812fecc9 100644 --- a/eo/src/utils/eoParallel.h +++ b/eo/src/utils/eoParallel.h @@ -20,8 +20,7 @@ Contact: http://eodev.sourceforge.net Authors: -Caner Candan - + Caner Candan */ /** @defgroup Parallel Parallel @@ -54,6 +53,7 @@ public: std::string prefix() const; inline unsigned int nthreads() const { return _nthreads.value(); } + inline unsigned int packetSize() const { return _packetSize.value(); } inline bool enableResults() const { return _enableResults.value(); } inline bool doMeasure() const { return _doMeasure.value(); } @@ -68,6 +68,7 @@ private: eoValueParam _isDynamic; eoValueParam _prefix; eoValueParam _nthreads; + eoValueParam _packetSize; eoValueParam _enableResults; eoValueParam _doMeasure; double _t_start; diff --git a/eo/src/utils/eoTimer.h b/eo/src/utils/eoTimer.h new file mode 100644 index 000000000..211313778 --- /dev/null +++ b/eo/src/utils/eoTimer.h @@ -0,0 +1,309 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EO_TIMER_H__ +# define __EO_TIMER_H__ + +# include // time() +# include // rusage() + +# include // std::vector +# include // std::map + +# include "utils/eoParallel.h" // eo::parallel + +# ifdef WITH_MPI +// For serialization purposes +# include +# include +# include +# endif + +/** + * @brief Timer allowing to measure time between a start point and a stop point. + * + * This timer allows the user to measure user time, system time and wallclock time + * between two points. Basically, user time is time spent in developer code ; system + * time is time spent during the IO wait and system calls ; wallclock is the difference + * of time we could observe by measuring time with a watch. + * + * @ingroup Utilities + */ +class eoTimer +{ + public: + + /** + * @brief Default ctor. Begins all the timers. + */ + eoTimer() + { + uuremainder = 0; + usremainder = 0; + restart(); + } + + /** + * @brief Restarts all the timers and launch the measure. + */ + void restart() + { + wc_start = time(NULL); + getrusage( RUSAGE_SELF, &_start ); + } + + /** + * @brief Measures the user time spent since the last restart(). + * + * If the number of elapsed seconds is zero, the spent milliseconds are + * added to a remainder. If the remainder exceeds one second, it is + * added to the number of elapsed seconds. + * + * @return Number of seconds elapsed in user space. + */ + long int usertime() + { + struct rusage _now; + getrusage( RUSAGE_SELF, &_now ); + + long int result = _now.ru_utime.tv_sec - _start.ru_utime.tv_sec; + long int remainder = _now.ru_utime.tv_usec - _start.ru_utime.tv_usec; + if( remainder >= 0 ) + { + uuremainder += remainder; + } else + { + uuremainder += ( 1000000 - remainder ); + --result; + } + + if( uuremainder >= 1000000 ) + { + uuremainder -= 1000000; + ++result; + } + return result; + } + + /** + * @brief Measures the system time spent since the last restart(). + * + * If the number of elapsed seconds is zero, the spent milliseconds are + * added to a remainder. If the remainder exceeds one second, it is + * added to the number of elapsed seconds. + * + * @return Number of seconds elapsed in system (kernel) space. + */ + long int systime() + { + struct rusage _now; + getrusage( RUSAGE_SELF, &_now ); + + long int result = _now.ru_stime.tv_sec - _start.ru_stime.tv_sec; + long int remainder = _now.ru_stime.tv_usec - _start.ru_stime.tv_usec; + if( remainder >= 0 ) + { + usremainder += remainder; + } else + { + usremainder += ( 1000000 - remainder ); + --result; + } + + if( usremainder >= 1000000 ) + { + usremainder -= 1000000; + ++result; + } + return result; + } + + /** + * @brief Measures the wallclock time spent since the last restart(). + * + * @return Number of seconds elapsed, as a double. + */ + double wallclock() + { + return std::difftime( std::time(NULL) , wc_start ); + } + + protected: + // Structure used to measure user and system time. + struct rusage _start; + // Remainder (in milliseconds) for user time. + long int uuremainder; + // Remainder (in milliseconds) for system time. + long int usremainder; + // Structure used to measure wallclock time. + time_t wc_start; +}; + +/** + * @brief Registers a group of statistics, each statistic corresponding to user, system and wallclock times distribution. + * + * This class helps the user to measure time in different parts of an application. A name is associated to a statistic, + * on each call to start() and stop() for this name, a new number is added to the statistic, for each of the three + * measured times. + * + * The statistics are only registered if the option "--parallelized-do-measure" is set to true, which can be checked + * thanks to global object eo::parallel. + * + * This shows how the eoTimerStat can be used : + * @code + * eoTimerStat timerStat; + * timerStat.start("first_point"); + * for( int i = 0; i < 1000; ++i ) + * { + * timerStat.start("single_computation"); + * single_computation( i ); + * timerStat.stop("single_computation"); + * } + * // After this loop, timerStat contains a statistic of key "single_computation" which contains 1000 measures for + * // each type of time. + * timerStat.stop("first_point"); + * // After this line, timerStat contains another statistic of key "first_point" which counted the duration of the + * // whole loop. + * + * int singleComputationUsertimeMean = 0; + * for( int i = 0; i < 1000; ++i ) + * { + * singleComputationUsertimeMean += timerStat.stats()["single_computation"].utime[i]; + * } + * std::cout << "Mean of user time spent in single computation: " << singleComputationUsertimeMean / 1000. << std::endl; + * @endcode + * + * When using MPI, these statistics can be readily be serialized, so as to be sent over a network, for instance. + * + * Implementation details: this eoTimerStat is in fact a map of strings (key) / Stat (value). Stat is an internal + * structure directly defined in the class, which contains three vectors modeling the distributions of the different + * types of elapsed times. Another map of strings (key) / eoTimer (value) allows to effectively retrieve the different + * times. The struct Stat will be exposed to client, which will use its members ; however, + * the client doesn't have anything to do directly with the timer, that's why the two maps are splitted. + * + * @ingroup Utilities + */ +class eoTimerStat +{ + public: + + /** + * @brief Statistic related to a key (name). + * + * This structure is the value in the map saved in the eoTimerStat. It contains the statistic bound to a key, + * which are the user time distribution, the system time distribution and the wallclock time distribution, as + * std::vector s. + * + * It can readily be serialized with boost when compiling with mpi. + */ + struct Stat + { + std::vector utime; + std::vector stime; + std::vector wtime; +#ifdef WITH_MPI + // Gives access to boost serialization + friend class boost::serialization::access; + + /** + * Serializes the single statistic in a boost archive (useful for boost::mpi). + * Just serializes the 3 vectors. + */ + template + void serialize( Archive & ar, const unsigned int version ) + { + ar & utime & stime & wtime; + (void) version; // avoid compilation warning + } +# endif + }; + +#ifdef WITH_MPI + // Gives access to boost serialization + friend class boost::serialization::access; + + /** + * Serializes the timerStat object in a boost archive (useful for boost::mpi). + * Just serializes the map. + */ + template + void serialize( Archive & ar, const unsigned int version ) + { + ar & _stats; + (void) version; // avoid compilation warning + } +# endif + + /** + * @brief Starts a new measure for the given key. + * + * This is only performed if parallel.doMeasure() is true, which is equivalent to the fact that + * parser found "--parallel-do-measure=1" in command line args. + * + * @param key The key of the statistic. + */ + void start( const std::string & key ) + { + if( eo::parallel.doMeasure() ) + { + _timers[ key ].restart(); + } + } + + /** + * @brief Stops the mesure for the given key and saves the elapsed times. + * + * Must follow a call of start with the same key. + * + * This is only performed if parallel.doMeasure() is true, which is equivalent to the fact that + * parser found "--parallel-do-measure=1" in command line args. + * + * @param key The key of the statistic. + */ + void stop( const std::string& key ) + { + if( eo::parallel.doMeasure() ) + { + Stat & s = _stats[ key ]; + eoTimer & t = _timers[ key ]; + s.utime.push_back( t.usertime() ); + s.stime.push_back( t.systime() ); + s.wtime.push_back( t.wallclock() ); + } + } + + /** + * @brief Getter for the statistics map. + */ + std::map< std::string, Stat >& stats() + { + return _stats; + } + + protected: + // Statistics map: links a key (string) to a statistic. + std::map< std::string, Stat > _stats; + // Timers map: links a key to its timer. + std::map< std::string, eoTimer > _timers; +}; + +# endif // __TIMER_H__ + diff --git a/eo/test/CMakeLists.txt b/eo/test/CMakeLists.txt index c4fcc8db5..8bf8b500e 100644 --- a/eo/test/CMakeLists.txt +++ b/eo/test/CMakeLists.txt @@ -14,12 +14,23 @@ INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/contrib) INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/contrib/MGE) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +IF(WITH_MPI) + INCLUDE_DIRECTORIES(${BOOST_DIR}/include) + INCLUDE_DIRECTORIES(${MPI_DIR}/include) +ENDIF() + ###################################################################################### ### 2) Specify where CMake can find the libraries ###################################################################################### LINK_DIRECTORIES(${EO_BINARY_DIR}/lib) +IF(WITH_MPI) + LINK_DIRECTORIES(${BOOST_DIR}/lib) + LINK_DIRECTORIES(${MPI_DIR}/lib) + SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") +ENDIF() + ###################################################################################### ### 3) Define your targets and link the librairies ###################################################################################### @@ -65,8 +76,8 @@ SET (TEST_LIST t-eoExtendedVelocity t-eoLogger t-eoIQRStat - t-eoParallel - #t-openmp # does not work anymore since functions used in this test were removed from EO + #t-eoParallel + #t-openmp #t-eoDualFitness t-eoParser ) diff --git a/eo/test/mpi/CMakeLists.txt b/eo/test/mpi/CMakeLists.txt new file mode 100644 index 000000000..37b1227ff --- /dev/null +++ b/eo/test/mpi/CMakeLists.txt @@ -0,0 +1,55 @@ +############################################################################### +## +## CMakeLists file for eo/test/mpi +## +############################################################################### + +###################################################################################### +### 1) Include the sources +###################################################################################### + +MESSAGE("EO SOURCE DIR: ${EO_SOURCE_DIR}") +MESSAGE("OMPI: ${MPI_DIR}") +MESSAGE("BOOST: ${BOOST_DIR}") + +INCLUDE_DIRECTORIES(${MPI_DIR}/include) +INCLUDE_DIRECTORIES(${BOOST_DIR}/include) +INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/src) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +###################################################################################### +### 2) Specify where CMake can find the libraries +###################################################################################### + +LINK_DIRECTORIES(${EO_BINARY_DIR}/lib) +LINK_DIRECTORIES(${MPI_DIR}/lib) +LINK_DIRECTORIES(${BOOST_DIR}/lib) + +###################################################################################### +### 3) Define your targets and link the librairies +###################################################################################### + +SET (TEST_LIST + t-mpi-parallelApply + t-mpi-wrapper + t-mpi-multipleRoles + t-mpi-eval + ) + +FOREACH (test ${TEST_LIST}) + SET ("T_${test}_SOURCES" "${test}.cpp") +ENDFOREACH (test) + +SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") +ADD_DEFINITIONS(-DWITH_MPI) + +IF(ENABLE_CMAKE_TESTING) + FOREACH (test ${TEST_LIST}) + ADD_EXECUTABLE(${test} ${T_${test}_SOURCES}) + ADD_TEST(${test} ${test}) + TARGET_LINK_LIBRARIES(${test} boost_mpi boost_serialization eoutils eompi eoserial eo) + INSTALL(TARGETS ${test} RUNTIME DESTINATION share/eo/test COMPONENT test) + ENDFOREACH (test) +ENDIF() + +###################################################################################### diff --git a/eo/test/mpi/t-mpi-eval.cpp b/eo/test/mpi/t-mpi-eval.cpp new file mode 100644 index 000000000..5216123ac --- /dev/null +++ b/eo/test/mpi/t-mpi-eval.cpp @@ -0,0 +1,227 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ + +/* + * This file shows an example of parallel evaluation of a population, when using an eoEasyEA algorithm. + * Moreover, we add a basic wrapper on the parallel evaluation, so as to show how to retrieve the best solutions. + */ +//----------------------------------------------------------------------------- + +#include +#include + +#include +#include "../real_value.h" + +#include + +#include + +#include +using namespace std; + +//----------------------------------------------------------------------------- + +class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial::Persistent +{ + public: + + eoRealSerializable(unsigned size = 0, double value = 0.0): + eoReal(size, value) {} + + eoserial::Object* pack() const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "array", + eoserial::makeArray< vector, eoserial::MakeAlgorithm > + ( *this ) + ); + return obj; + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpackArray< vector, eoserial::Array::UnpackAlgorithm > + ( *obj, "array", *this ); + } + + // Gives access to boost serialization + friend class boost::serialization::access; + + /** + * Serializes the decomposition in a boost archive (useful for boost::mpi) + */ + template + void save( Archive & ar, const unsigned int version ) const + { + std::stringstream ss; + printOn( ss ); + std::string asStr = ss.str(); + ar & asStr; + + (void) version; // avoid compilation warning + } + + /** + * Deserializes the decomposition from a boost archive (useful for boost:mpi) + */ + template + void load( Archive & ar, const unsigned int version ) + { + std::string asStr; + ar & asStr; + std::stringstream ss; + ss << asStr; + readFrom( ss ); + + (void) version; // avoid compilation warning + } + + // Indicates that boost save and load operations are not the same. + BOOST_SERIALIZATION_SPLIT_MEMBER() + +}; + +typedef eoRealSerializable EOT; + +/* + * Wrapper for HandleResponse: shows the best answer, as it is found. + * + * Finding the best solution is an associative operation (as it is based on a "min" function, which is associative too) + * and that's why we can perform it here. Indeed, the min element of 5 elements is the min element of the 3 first + * elements and the min element of the 2 last elements: + * min(1, 2, 3, 4, 5) = min( min(1, 2, 3), min(4, 5) ) + * + * This is a reduction. See MapReduce example to have another examples of reduction. + */ +struct CatBestAnswers : public eo::mpi::HandleResponseParallelApply +{ + CatBestAnswers() + { + best.fitness( 1000000000. ); + } + + /* + our structure inherits the member _wrapped from HandleResponseFunction, + which is a HandleResponseFunction pointer; + + it inherits also the member _d (like Data), which is a pointer to the + ParallelApplyData used in the HandleResponseParallelApply<EOT>. Details + of this data are contained in the file eoParallelApply. We need just to know that + it contains a member assignedTasks which maps a worker rank and the sent slice + to be processed by the worker, and a reference to the processed table via the + call of the data() function. + */ + + // if EOT were a template, we would have to do: (thank you C++ :) + // using eo::mpi::HandleResponseParallelApply::_wrapped; + // using eo::mpi::HandleResponseParallelApply::d; + + void operator()(int wrkRank) + { + eo::mpi::ParallelApplyData * d = _data; + // Retrieve informations about the slice processed by the worker + int index = d->assignedTasks[wrkRank].index; + int size = d->assignedTasks[wrkRank].size; + // call to the wrapped function HERE + (*_wrapped)( wrkRank ); + // Compare fitnesses of evaluated individuals with the best saved + for(int i = index; i < index+size; ++i) + { + if( best.fitness() < d->table()[ i ].fitness() ) + { + eo::log << eo::quiet << "Better solution found:" << d->table()[i].fitness() << std::endl; + best = d->table()[ i ]; + } + } + } + + protected: + + EOT best; +}; + +int main(int ac, char** av) +{ + eo::mpi::Node::init( ac, av ); + // eo::log << eo::setlevel( eo::debug ); + eo::log << eo::setlevel( eo::quiet ); + + eoParser parser(ac, av); + + unsigned int popSize = parser.getORcreateParam((unsigned int)100, "popSize", "Population Size", 'P', "Evolution Engine").value(); + unsigned int dimSize = parser.getORcreateParam((unsigned int)10, "dimSize", "Dimension Size", 'd', "Evolution Engine").value(); + + uint32_t seedParam = parser.getORcreateParam((uint32_t)0, "seed", "Random number seed", 0).value(); + if (seedParam == 0) { seedParam = time(0); } + + make_parallel(parser); + make_help(parser); + + rng.reseed( seedParam ); + + eoUniformGenerator< double > gen(-5, 5); + eoInitFixedLength< EOT > init( dimSize, gen ); + + eoEvalFuncPtr< EOT, double, const std::vector< double >& > mainEval( real_value ); + eoEvalFuncCounter< EOT > eval( mainEval ); + + // until this point, everything (but eo::mpi::Node::init) is exactly as in an sequential version. + // We then instanciate the parallel algorithm. The store is directly used by the eoParallelPopLoopEval, which + // internally uses parallel apply. + int rank = eo::mpi::Node::comm().rank(); + eo::mpi::DynamicAssignmentAlgorithm assign; + if( rank == eo::mpi::DEFAULT_MASTER ) + { + eoPop< EOT > pop( popSize, init ); + + eo::log << "Size of population : " << popSize << std::endl; + + /* + eo::mpi::ParallelApplyStore< EOT > store( eval, eo::mpi::DEFAULT_MASTER ); + store.wrapHandleResponse( new CatBestAnswers ); + + eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, &store ); + */ + + eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, eval, 5 ); + + eo::log << eo::quiet << "Before first evaluation." << std::endl; + popEval( pop, pop ); + eo::log << eo::quiet << "After first evaluation." << std::endl; + + pop = eoPop< EOT >( popSize, init ); + popEval( pop, pop ); + eo::log << eo::quiet << "After second evaluation." << std::endl; + + eo::log << eo::quiet << "DONE!" << std::endl; + } else + { + eoPop< EOT > pop; // the population doesn't have to be initialized, as it is not used by workers. + eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, eval ); + popEval( pop, pop ); + } + + return 0; +} + +//----------------------------------------------------------------------------- diff --git a/eo/test/mpi/t-mpi-multipleRoles.cpp b/eo/test/mpi/t-mpi-multipleRoles.cpp new file mode 100644 index 000000000..36d588b82 --- /dev/null +++ b/eo/test/mpi/t-mpi-multipleRoles.cpp @@ -0,0 +1,169 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ + +/* + * This file shows an example of how to make a hierarchy between nodes, when using a parallel apply. In this basic + * test, the master delegates the charge of finding workers to 2 "sub" masters, which then send part of the table to + * their workers. + * + * It's convenient to establish a role map, so as to clearly identify every role: + * - The node 0 is the general master, that delegates the job. It sends the table to the 2 submasters, and waits for the + * results. + * - Nodes 1 and 2 are the worker of the first job: the delegates. They receive the elements of the table and + * retransmit them to the subworkers. They play the roles of worker in the delegating job, and master in the plus one + * job. + * - Following nodes (3 to 6) are workers of the plus one job. They do the real job. Nodes 3 and 5 are attached to + * submaster 1, 4 and 6 to submaster 2. + * + * This test requires exactly 7 hosts. If the size is bigger, an exception will be thrown at the beginning. + **/ + +# include +# include +# include + +# include + +# include + +# include +using namespace std; + +using namespace eo::mpi; + +// The real job to execute, for the subworkers: add one to each element of a table. +struct SubWork: public eoUF< int&, void > +{ + void operator() ( int & x ) + { + cout << "Subwork phase." << endl; + ++x; + } +}; + +// Function called by both subworkers and delegates. +// v is the vector to process, rank is the MPI rank of the sub master +void subtask( vector& v, int rank ) +{ + // Attach workers according to nodes. + // Submaster with rank 1 will have ranks 3 and 5 as subworkers. + // Submaster with rank 2 will have ranks 4 and 6 as subworkers. + vector workers; + workers.push_back( rank + 2 ); + workers.push_back( rank + 4 ); + DynamicAssignmentAlgorithm algo( workers ); + SubWork sw; + + // Launch the job! + ParallelApplyStore store( sw, rank ); + store.data( v ); + ParallelApply job( algo, rank, store ); + job.run(); + EmptyJob stop( algo, rank ); +} + +// Functor applied by submasters. Wait for the subworkers responses and then add some random processing (here, multiply +// each result by two). +// Note that this work receives a vector of integers as an entry, while subworkers task's operator receives a simple +// integer. +struct Work: public eoUF< vector&, void > +{ + void operator() ( vector& v ) + { + cout << "Work phase..." << endl; + subtask( v, Node::comm().rank() ); + for( int i = 0; i < v.size(); ++i ) + { + v[i] *= 2; + } + } +}; + +int main(int argc, char** argv) +{ + // eo::log << eo::setlevel( eo::debug ); + Node::init( argc, argv ); + if( Node::comm().size() != 7 ) { + throw std::runtime_error("World size should be 7."); + } + + vector v; + + v.push_back(1); + v.push_back(3); + v.push_back(3); + v.push_back(7); + v.push_back(42); + + // As submasters' operator receives a vector as an input, and ParallelApply takes a vector of + // operator's input as an input, we have to deal with a vector of vector of integers for the master task. + vector< vector > metaV; + // Here, we send twice the same vector. We could also have splitted the first vector into two vectors, one + // containing the beginning and another one containing the end. + metaV.push_back( v ); + metaV.push_back( v ); + + // Assigning roles is done by comparing MPI ranks. + switch( Node::comm().rank() ) + { + // Nodes from 0 to 2 are implicated into the delegating task. + case 0: + case 1: + case 2: + { + Work w; + DynamicAssignmentAlgorithm algo( 1, 2 ); + ParallelApplyStore< vector > store( w, 0 ); + store.data( metaV ); + ParallelApply< vector > job( algo, 0, store ); + job.run(); + if( job.isMaster() ) + { + EmptyJob stop( algo, 0 ); + v = metaV[0]; + cout << "Results : " << endl; + for(int i = 0; i < v.size(); ++i) + { + cout << v[i] << ' '; + } + cout << endl; + } + } + break; + + // Other nodes are implicated into the subwork task. + default: + { + // all the other nodes are sub workers + int rank = Node::comm().rank(); + if ( rank == 3 or rank == 5 ) + { + subtask( v, 1 ); + } else { + subtask( v, 2 ); + } + } + break; + } + + return 0; +} diff --git a/eo/test/mpi/t-mpi-parallelApply.cpp b/eo/test/mpi/t-mpi-parallelApply.cpp new file mode 100644 index 000000000..7cefa203a --- /dev/null +++ b/eo/test/mpi/t-mpi-parallelApply.cpp @@ -0,0 +1,214 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ + +/* + * This file shows an example of use of parallel apply, in the following context: each element of a table is + * incremented... in a parallel fashion. While this operation is very easy to perform even on a single host, it's just + * an example for parallel apply use. + * + * Besides, this is also a test for assignment (scheduling) algorithms, in different cases. The test succeeds if and + * only if the program terminates without any segfault ; otherwise, there could be a deadlock which prevents the end or + * a segfault at any time. + * + * One important thing is to instanciate an EmptyJob after having launched a ParallelApplyJob, so as the workers to be + * aware that the job is done (as it's a MultiJob). + * + * This test needs at least 3 processes to be launched. Under this size, it will directly throw an exception, at the + * beginning; + */ + +# include +# include +# include + +# include + +# include +using namespace std; + +using namespace eo::mpi; + +/* + * The function to be called on each element of the table: just increment the value. + */ +struct plusOne : public eoUF< int&, void > +{ + void operator() ( int & x ) + { + ++x; + } +}; + +/* + * Internal structure representating a test. + */ +struct Test +{ + AssignmentAlgorithm * assign; // used assignment algorithm for this test. + string description; // textual description of the test + int requiredNodesNumber; // number of required nodes. NB : chosen nodes ranks must be sequential +}; + +int main(int argc, char** argv) +{ + // eo::log << eo::setlevel( eo::debug ); // if you like tty full of rainbows, decomment this line and comment the following one. + eo::log << eo::setlevel( eo::quiet ); + + bool launchOnlyOne = false ; // Set this to true if you wanna launch only the first test. + + Node::init( argc, argv ); + + // Initializes a vector with random values. + srand( time(0) ); + vector v; + for( int i = 0; i < 1000; ++i ) + { + v.push_back( rand() ); + } + + // We need to be sure the values are correctly incremented between each test. So as to check this, we save the + // original vector into a variable originalV, and put an offset variable to 0. After each test, the offset is + // incremented and we can compare the returned value of each element to the value of each element in originalV + + // offset. If the two values are different, there has been a problem. + int offset = 0; + vector originalV = v; + + // Instanciates the functor to apply on each element + plusOne plusOneInstance; + + vector< Test > tests; + + const int ALL = Node::comm().size(); + if( ALL < 3 ) { + throw std::runtime_error("Needs at least 3 processes to be launched!"); + } + + // Tests are auto described thanks to member "description" + Test tIntervalStatic; + tIntervalStatic.assign = new StaticAssignmentAlgorithm( 1, REST_OF_THE_WORLD, v.size() ); + tIntervalStatic.description = "Correct static assignment with interval."; // workers have ranks from 1 to size - 1 + tIntervalStatic.requiredNodesNumber = ALL; + tests.push_back( tIntervalStatic ); + + if( !launchOnlyOne ) + { + Test tWorldStatic; + tWorldStatic.assign = new StaticAssignmentAlgorithm( v.size() ); + tWorldStatic.description = "Correct static assignment with whole world as workers."; + tWorldStatic.requiredNodesNumber = ALL; + tests.push_back( tWorldStatic ); + + Test tStaticOverload; + tStaticOverload.assign = new StaticAssignmentAlgorithm( v.size()+100 ); + tStaticOverload.description = "Static assignment with too many runs."; + tStaticOverload.requiredNodesNumber = ALL; + tests.push_back( tStaticOverload ); + + Test tUniqueStatic; + tUniqueStatic.assign = new StaticAssignmentAlgorithm( 1, v.size() ); + tUniqueStatic.description = "Correct static assignment with unique worker."; + tUniqueStatic.requiredNodesNumber = 2; + tests.push_back( tUniqueStatic ); + + Test tVectorStatic; + vector workers; + workers.push_back( 1 ); + workers.push_back( 2 ); + tVectorStatic.assign = new StaticAssignmentAlgorithm( workers, v.size() ); + tVectorStatic.description = "Correct static assignment with precise workers specified."; + tVectorStatic.requiredNodesNumber = 3; + tests.push_back( tVectorStatic ); + + Test tIntervalDynamic; + tIntervalDynamic.assign = new DynamicAssignmentAlgorithm( 1, REST_OF_THE_WORLD ); + tIntervalDynamic.description = "Dynamic assignment with interval."; + tIntervalDynamic.requiredNodesNumber = ALL; + tests.push_back( tIntervalDynamic ); + + Test tUniqueDynamic; + tUniqueDynamic.assign = new DynamicAssignmentAlgorithm( 1 ); + tUniqueDynamic.description = "Dynamic assignment with unique worker."; + tUniqueDynamic.requiredNodesNumber = 2; + tests.push_back( tUniqueDynamic ); + + Test tVectorDynamic; + tVectorDynamic.assign = new DynamicAssignmentAlgorithm( workers ); + tVectorDynamic.description = "Dynamic assignment with precise workers specified."; + tVectorDynamic.requiredNodesNumber = tVectorStatic.requiredNodesNumber; + tests.push_back( tVectorDynamic ); + + Test tWorldDynamic; + tWorldDynamic.assign = new DynamicAssignmentAlgorithm; + tWorldDynamic.description = "Dynamic assignment with whole world as workers."; + tWorldDynamic.requiredNodesNumber = ALL; + tests.push_back( tWorldDynamic ); + } + + for( unsigned int i = 0; i < tests.size(); ++i ) + { + // Instanciates a store with the functor, the master rank and size of packet (see ParallelApplyStore doc). + ParallelApplyStore< int > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 3 ); + // Updates the contained data + store.data( v ); + // Creates the job with the assignment algorithm, the master rank and the store + ParallelApply< int > job( *(tests[i].assign), eo::mpi::DEFAULT_MASTER, store ); + + // Only master writes information + if( job.isMaster() ) + { + cout << "Test : " << tests[i].description << endl; + } + + // Workers whose rank is inferior to required nodes number have to run the test, the other haven't anything to + // do. + if( Node::comm().rank() < tests[i].requiredNodesNumber ) + { + job.run(); + } + + // After the job run, the master checks the result with offset and originalV + if( job.isMaster() ) + { + // This job has to be instanciated, not launched, so as to tell the workers they're done with the parallel + // job. + EmptyJob stop( *(tests[i].assign), eo::mpi::DEFAULT_MASTER ); + ++offset; + for(int i = 0; i < v.size(); ++i) + { + cout << v[i] << ' '; + if( originalV[i] + offset != v[i] ) + { + cout << " <-- ERROR at this point." << endl; + exit( EXIT_FAILURE ); + } + } + cout << endl; + } + + // MPI synchronization (all the processes wait to be here). + Node::comm().barrier(); + + delete tests[i].assign; + } + return 0; +} + diff --git a/eo/test/mpi/t-mpi-wrapper.cpp b/eo/test/mpi/t-mpi-wrapper.cpp new file mode 100644 index 000000000..97d4b04a1 --- /dev/null +++ b/eo/test/mpi/t-mpi-wrapper.cpp @@ -0,0 +1,132 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ + +/* + * This file shows an example of how to wrap a handler of a job store. Here, the wrapped handler is the "IsFinished" + * one. The only function that has been added is that the wrapper prints a message on standard output, indicating what + * the wrapped function returns as a result. + * + * This test is performed on a parallel apply job, the same as in parallelApply. The main difference is when + * instanciating the store. + */ + +# include +# include +# include + +# include + +# include +using namespace std; + +using namespace eo::mpi; + +// Job functor. +struct plusOne : public eoUF< int&, void > +{ + void operator() ( int & x ) + { + ++x; + } +}; + +/* + * Shows the wrapped result of IsFinished, prints a message and returns the wrapped value. + * times is an integer counting how many time the wrapper (hence the wrapped too) has been called. + */ +template< class EOT > +struct ShowWrappedResult : public IsFinishedParallelApply +{ + using IsFinishedParallelApply::_wrapped; + + ShowWrappedResult ( IsFinishedParallelApply * w = 0 ) : IsFinishedParallelApply( w ), times( 0 ) + { + // empty + } + + bool operator()() + { + bool wrappedValue = _wrapped->operator()(); // (*_wrapped)(); + cout << times << ") Wrapped function would say that it is " << ( wrappedValue ? "":"not ") << "finished" << std::endl; + ++times; + return wrappedValue; + } + + private: + int times; +}; + +int main(int argc, char** argv) +{ + // eo::log << eo::setlevel( eo::debug ); + eo::log << eo::setlevel( eo::quiet ); + + Node::init( argc, argv ); + + srand( time(0) ); + vector v; + for( int i = 0; i < 1000; ++i ) + { + v.push_back( rand() ); + } + + int offset = 0; + vector originalV = v; + + plusOne plusOneInstance; + + StaticAssignmentAlgorithm assign( v.size() ); + + ParallelApplyStore< int > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 1 ); + store.data( v ); + // This is the only thing which changes: we wrap the IsFinished function. + // According to RAII, we'll delete the invokated wrapper at the end of the main ; the store won't delete it + // automatically. + IsFinishedParallelApply* wrapper = new ShowWrappedResult; + store.wrapIsFinished( wrapper ); + + ParallelApply job( assign, eo::mpi::DEFAULT_MASTER, store ); + // Equivalent to: + // Job< ParallelApplyData > job( assign, 0, store ); + job.run(); + EmptyJob stop( assign, eo::mpi::DEFAULT_MASTER ); + + if( job.isMaster() ) + { + ++offset; + for(int i = 0; i < v.size(); ++i) + { + cout << v[i] << ' '; + if( originalV[i] + offset != v[i] ) + { + cout << " <-- ERROR at this point." << endl; + exit( EXIT_FAILURE ); + } + } + cout << endl; + } + + delete wrapper; + + return 0; +} + diff --git a/eo/tutorial/Parallelization/css/deck.core.css b/eo/tutorial/Parallelization/css/deck.core.css new file mode 100644 index 000000000..a188b28fd --- /dev/null +++ b/eo/tutorial/Parallelization/css/deck.core.css @@ -0,0 +1,405 @@ +html { + height: 100%; +} + +body.deck-container { + overflow-y: auto; + position: static; +} + +.deck-container { + position: relative; + min-height: 100%; + margin: 0 auto; + padding: 0 48px; + font-size: 16px; + line-height: 1.25; + overflow: hidden; + /* Resets and base styles from HTML5 Boilerplate */ + /* End HTML5 Boilerplate adaptations */ +} +.js .deck-container { + visibility: hidden; +} +.ready .deck-container { + visibility: visible; +} +.touch .deck-container { + -webkit-text-size-adjust: none; + -moz-text-size-adjust: none; +} +.deck-container div, .deck-container span, .deck-container object, .deck-container iframe, +.deck-container h1, .deck-container h2, .deck-container h3, .deck-container h4, .deck-container h5, .deck-container h6, .deck-container p, .deck-container blockquote, .deck-container pre, +.deck-container abbr, .deck-container address, .deck-container cite, .deck-container code, .deck-container del, .deck-container dfn, .deck-container em, .deck-container img, .deck-container ins, .deck-container kbd, .deck-container q, .deck-container samp, +.deck-container small, .deck-container strong, .deck-container sub, .deck-container sup, .deck-container var, .deck-container b, .deck-container i, .deck-container dl, .deck-container dt, .deck-container dd, .deck-container ol, .deck-container ul, .deck-container li, +.deck-container fieldset, .deck-container form, .deck-container label, .deck-container legend, +.deck-container table, .deck-container caption, .deck-container tbody, .deck-container tfoot, .deck-container thead, .deck-container tr, .deck-container th, .deck-container td, +.deck-container article, .deck-container aside, .deck-container canvas, .deck-container details, .deck-container figcaption, .deck-container figure, +.deck-container footer, .deck-container header, .deck-container hgroup, .deck-container menu, .deck-container nav, .deck-container section, .deck-container summary, +.deck-container time, .deck-container mark, .deck-container audio, .deck-container video { + margin: 0; + padding: 0; + border: 0; + font-size: 100%; + font: inherit; + vertical-align: baseline; +} +.deck-container article, .deck-container aside, .deck-container details, .deck-container figcaption, .deck-container figure, +.deck-container footer, .deck-container header, .deck-container hgroup, .deck-container menu, .deck-container nav, .deck-container section { + display: block; +} +.deck-container blockquote, .deck-container q { + quotes: none; +} +.deck-container blockquote:before, .deck-container blockquote:after, .deck-container q:before, .deck-container q:after { + content: ""; + content: none; +} +.deck-container ins { + background-color: #ff9; + color: #000; + text-decoration: none; +} +.deck-container mark { + background-color: #ff9; + color: #000; + font-style: italic; + font-weight: bold; +} +.deck-container del { + text-decoration: line-through; +} +.deck-container abbr[title], .deck-container dfn[title] { + border-bottom: 1px dotted; + cursor: help; +} +.deck-container table { + border-collapse: collapse; + border-spacing: 0; +} +.deck-container hr { + display: block; + height: 1px; + border: 0; + border-top: 1px solid #ccc; + margin: 1em 0; + padding: 0; +} +.deck-container input, .deck-container select { + vertical-align: middle; +} +.deck-container select, .deck-container input, .deck-container textarea, .deck-container button { + font: 99% sans-serif; +} +.deck-container pre, .deck-container code, .deck-container kbd, .deck-container samp { + font-family: monospace, sans-serif; +} +.deck-container a { + -webkit-tap-highlight-color: rgba(0, 0, 0, 0); +} +.deck-container a:hover, .deck-container a:active { + outline: none; +} +.deck-container ul, .deck-container ol { + margin-left: 2em; + vertical-align: top; +} +.deck-container ol { + list-style-type: decimal; +} +.deck-container nav ul, .deck-container nav li { + margin: 0; + list-style: none; + list-style-image: none; +} +.deck-container small { + font-size: 85%; +} +.deck-container strong, .deck-container th { + font-weight: bold; +} +.deck-container td { + vertical-align: top; +} +.deck-container sub, .deck-container sup { + font-size: 75%; + line-height: 0; + position: relative; +} +.deck-container sup { + top: -0.5em; +} +.deck-container sub { + bottom: -0.25em; +} +.deck-container textarea { + overflow: auto; +} +.ie6 .deck-container legend, .ie7 .deck-container legend { + margin-left: -7px; +} +.deck-container input[type="radio"] { + vertical-align: text-bottom; +} +.deck-container input[type="checkbox"] { + vertical-align: bottom; +} +.ie7 .deck-container input[type="checkbox"] { + vertical-align: baseline; +} +.ie6 .deck-container input { + vertical-align: text-bottom; +} +.deck-container label, .deck-container input[type="button"], .deck-container input[type="submit"], .deck-container input[type="image"], .deck-container button { + cursor: pointer; +} +.deck-container button, .deck-container input, .deck-container select, .deck-container textarea { + margin: 0; +} +.deck-container input:invalid, .deck-container textarea:invalid { + border-radius: 1px; + -moz-box-shadow: 0px 0px 5px red; + -webkit-box-shadow: 0px 0px 5px red; + box-shadow: 0px 0px 5px red; +} +.deck-container input:invalid .no-boxshadow, .deck-container textarea:invalid .no-boxshadow { + background-color: #f0dddd; +} +.deck-container button { + width: auto; + overflow: visible; +} +.ie7 .deck-container img { + -ms-interpolation-mode: bicubic; +} +.deck-container, .deck-container select, .deck-container input, .deck-container textarea { + color: #444; +} +.deck-container a { + color: #607890; +} +.deck-container a:hover, .deck-container a:focus { + color: #036; +} +.deck-container a:link { + -webkit-tap-highlight-color: #fff; +} +.deck-container.deck-loading { + display: none; +} + +.slide { + width: auto; + min-height: 100%; + position: relative; +} +.slide h1 { + font-size: 4.5em; +} +.slide h1, .slide .vcenter { + font-weight: bold; + text-align: center; + padding-top: 1em; + max-height: 100%; +} +.csstransforms .slide h1, .csstransforms .slide .vcenter { + padding: 0 48px; + position: absolute; + left: 0; + right: 0; + top: 50%; + -webkit-transform: translate(0, -50%); + -moz-transform: translate(0, -50%); + -ms-transform: translate(0, -50%); + -o-transform: translate(0, -50%); + transform: translate(0, -50%); +} +.slide .vcenter h1 { + position: relative; + top: auto; + padding: 0; + -webkit-transform: none; + -moz-transform: none; + -ms-transform: none; + -o-transform: none; + transform: none; +} +.slide h2 { + font-size: 2.25em; + font-weight: bold; + padding-top: .5em; + margin: 0 0 .66666em 0; + border-bottom: 3px solid #888; +} +.slide h3 { + font-size: 1.4375em; + font-weight: bold; + margin-bottom: .30435em; +} +.slide h4 { + font-size: 1.25em; + font-weight: bold; + margin-bottom: .25em; +} +.slide h5 { + font-size: 1.125em; + font-weight: bold; + margin-bottom: .2222em; +} +.slide h6 { + font-size: 1em; + font-weight: bold; +} +.slide img, .slide iframe, .slide video { + display: block; + max-width: 100%; +} +.slide video, .slide iframe, .slide img { + display: block; + margin: 0 auto; +} +.slide p, .slide blockquote, .slide iframe, .slide img, .slide ul, .slide ol, .slide pre, .slide video { + margin-bottom: 1em; +} +.slide pre { + white-space: pre; + white-space: pre-wrap; + word-wrap: break-word; + padding: 1em; + border: 1px solid #888; +} +.slide em { + font-style: italic; +} +.slide li { + padding: .25em 0; + vertical-align: middle; +} + +.deck-before, .deck-previous, .deck-next, .deck-after { + position: absolute; + left: -999em; + top: -999em; +} + +.deck-current { + z-index: 2; +} + +.slide .slide { + visibility: hidden; + position: static; + min-height: 0; +} + +.deck-child-current { + position: static; + z-index: 2; +} +.deck-child-current .slide { + visibility: hidden; +} +.deck-child-current .deck-previous, .deck-child-current .deck-before, .deck-child-current .deck-current { + visibility: visible; +} + +@media screen and (max-device-width: 480px) { + /* html { -webkit-text-size-adjust:none; -ms-text-size-adjust:none; } */ +} +@media print { + * { + background: transparent !important; + color: black !important; + text-shadow: none !important; + filter: none !important; + -ms-filter: none !important; + -webkit-box-reflect: none !important; + -moz-box-reflect: none !important; + -webkit-box-shadow: none !important; + -moz-box-shadow: none !important; + box-shadow: none !important; + } + * :before, * :after { + display: none !important; + } + + a, a:visited { + color: #444 !important; + text-decoration: underline; + } + + a[href]:after { + content: " (" attr(href) ")"; + } + + abbr[title]:after { + content: " (" attr(title) ")"; + } + + .ir a:after, a[href^="javascript:"]:after, a[href^="#"]:after { + content: ""; + } + + pre, blockquote { + border: 1px solid #999; + page-break-inside: avoid; + } + + thead { + display: table-header-group; + } + + tr, img { + page-break-inside: avoid; + } + + @page { + margin: 0.5cm; +} + + p, h2, h3 { + orphans: 3; + widows: 3; + } + + h2, h3 { + page-break-after: avoid; + } + + .slide { + position: static !important; + visibility: visible !important; + display: block !important; + -webkit-transform: none !important; + -moz-transform: none !important; + -o-transform: none !important; + -ms-transform: none !important; + transform: none !important; + opacity: 1 !important; + } + + h1, .vcenter { + -webkit-transform: none !important; + -moz-transform: none !important; + -o-transform: none !important; + -ms-transform: none !important; + transform: none !important; + padding: 0 !important; + position: static !important; + } + + .deck-container > .slide { + page-break-after: always; + } + + .deck-container { + width: 100% !important; + height: auto !important; + padding: 0 !important; + display: block !important; + } + + script { + display: none; + } +} diff --git a/eo/tutorial/Parallelization/css/deck.goto.css b/eo/tutorial/Parallelization/css/deck.goto.css new file mode 100644 index 000000000..108e4f9c4 --- /dev/null +++ b/eo/tutorial/Parallelization/css/deck.goto.css @@ -0,0 +1,41 @@ +.deck-container .goto-form { + position: absolute; + z-index: 3; + bottom: 10px; + left: 50%; + height: 1.75em; + margin: 0 0 0 -9.125em; + line-height: 1.75em; + padding: 0.625em; + display: none; + background: #ccc; + overflow: hidden; +} +.borderradius .deck-container .goto-form { + -webkit-border-radius: 10px; + -moz-border-radius: 10px; + border-radius: 10px; +} +.deck-container .goto-form label { + font-weight: bold; +} +.deck-container .goto-form label, .deck-container .goto-form input { + display: inline-block; + font-family: inherit; +} + +.deck-goto .goto-form { + display: block; +} + +#goto-slide { + width: 8.375em; + margin: 0 0.625em; + height: 1.4375em; +} + +@media print { + .goto-form, #goto-slide { + display: none !important; + } +} diff --git a/eo/tutorial/Parallelization/css/deck.hash.css b/eo/tutorial/Parallelization/css/deck.hash.css new file mode 100644 index 000000000..28f07326b --- /dev/null +++ b/eo/tutorial/Parallelization/css/deck.hash.css @@ -0,0 +1,13 @@ +.deck-container .deck-permalink { + display: none; + position: absolute; + z-index: 4; + bottom: 30px; + right: 0; + width: 48px; + text-align: center; +} + +.no-history .deck-container:hover .deck-permalink { + display: block; +} diff --git a/eo/tutorial/Parallelization/css/deck.menu.css b/eo/tutorial/Parallelization/css/deck.menu.css new file mode 100644 index 000000000..c664a3f8e --- /dev/null +++ b/eo/tutorial/Parallelization/css/deck.menu.css @@ -0,0 +1,47 @@ +.deck-menu .slide { + background: #eee; + position: relative; + left: 0; + top: 0; + visibility: visible; + cursor: pointer; +} +.no-csstransforms .deck-menu > .slide { + float: left; + width: 22%; + height: 22%; + min-height: 0; + margin: 1%; + font-size: 0.22em; + overflow: hidden; + padding: 0 0.5%; +} +.csstransforms .deck-menu > .slide { + -webkit-transform: scale(0.22) !important; + -moz-transform: scale(0.22) !important; + -o-transform: scale(0.22) !important; + -ms-transform: scale(0.22) !important; + transform: scale(0.22) !important; + -webkit-transform-origin: 0 0; + -moz-transform-origin: 0 0; + -o-transform-origin: 0 0; + -ms-transform-origin: 0 0; + transform-origin: 0 0; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + width: 100%; + height: 100%; + overflow: hidden; + padding: 0 48px; + margin: 12px; +} +.deck-menu iframe, .deck-menu img, .deck-menu video { + max-width: 100%; +} +.deck-menu .deck-current, .no-touch .deck-menu .slide:hover { + background: #ddf; +} +.deck-menu.deck-container:hover .deck-prev-link, .deck-menu.deck-container:hover .deck-next-link { + display: none; +} diff --git a/eo/tutorial/Parallelization/css/deck.navigation.css b/eo/tutorial/Parallelization/css/deck.navigation.css new file mode 100644 index 000000000..e1ebad8b8 --- /dev/null +++ b/eo/tutorial/Parallelization/css/deck.navigation.css @@ -0,0 +1,43 @@ +.deck-container .deck-prev-link, .deck-container .deck-next-link { + display: none; + position: absolute; + z-index: 3; + top: 50%; + width: 32px; + height: 32px; + margin-top: -16px; + font-size: 20px; + font-weight: bold; + line-height: 32px; + vertical-align: middle; + text-align: center; + text-decoration: none; + color: #fff; + background: #888; +} +.borderradius .deck-container .deck-prev-link, .borderradius .deck-container .deck-next-link { + -webkit-border-radius: 16px; + -moz-border-radius: 16px; + border-radius: 16px; +} +.deck-container .deck-prev-link:hover, .deck-container .deck-prev-link:focus, .deck-container .deck-prev-link:active, .deck-container .deck-prev-link:visited, .deck-container .deck-next-link:hover, .deck-container .deck-next-link:focus, .deck-container .deck-next-link:active, .deck-container .deck-next-link:visited { + color: #fff; +} +.deck-container .deck-prev-link { + left: 8px; +} +.deck-container .deck-next-link { + right: 8px; +} +.deck-container:hover .deck-prev-link, .deck-container:hover .deck-next-link { + display: block; +} +.deck-container:hover .deck-prev-link.deck-nav-disabled, .touch .deck-container:hover .deck-prev-link, .deck-container:hover .deck-next-link.deck-nav-disabled, .touch .deck-container:hover .deck-next-link { + display: none; +} + +@media print { + .deck-prev-link, .deck-next-link { + display: none !important; + } +} diff --git a/eo/tutorial/Parallelization/css/deck.scale.css b/eo/tutorial/Parallelization/css/deck.scale.css new file mode 100644 index 000000000..d6a4eb0be --- /dev/null +++ b/eo/tutorial/Parallelization/css/deck.scale.css @@ -0,0 +1,28 @@ +/* Remove this line if you are embedding deck.js in a page and +using the scale extension. */ +.csstransforms { + overflow: hidden; +} + +.csstransforms .deck-container.deck-scale:not(.deck-menu) > .slide { + -webkit-box-sizing: padding-box; + -moz-box-sizing: padding-box; + box-sizing: padding-box; + width: 100%; + padding-bottom: 20px; +} +.csstransforms .deck-container.deck-scale:not(.deck-menu) > .slide > .deck-slide-scaler { + -webkit-transform-origin: 50% 0; + -moz-transform-origin: 50% 0; + -o-transform-origin: 50% 0; + -ms-transform-origin: 50% 0; + transform-origin: 50% 0; +} + +.csstransforms .deck-container.deck-menu .deck-slide-scaler { + -webkit-transform: none !important; + -moz-transform: none !important; + -o-transform: none !important; + -ms-transform: none !important; + transform: none !important; +} diff --git a/eo/tutorial/Parallelization/css/deck.status.css b/eo/tutorial/Parallelization/css/deck.status.css new file mode 100644 index 000000000..17d55ad0d --- /dev/null +++ b/eo/tutorial/Parallelization/css/deck.status.css @@ -0,0 +1,18 @@ +.deck-container .deck-status { + position: absolute; + bottom: 10px; + right: 5px; + color: #888; + z-index: 3; + margin: 0; +} + +body.deck-container .deck-status { + position: fixed; +} + +@media print { + .deck-status { + display: none; + } +} diff --git a/eo/tutorial/Parallelization/css/eompi.css b/eo/tutorial/Parallelization/css/eompi.css new file mode 100644 index 000000000..d089362ce --- /dev/null +++ b/eo/tutorial/Parallelization/css/eompi.css @@ -0,0 +1,9 @@ +.changed +{ + color: green; +} + +.specific +{ + color: red; +} diff --git a/eo/tutorial/Parallelization/css/horizontal-slide.css b/eo/tutorial/Parallelization/css/horizontal-slide.css new file mode 100644 index 000000000..4a4c6adf6 --- /dev/null +++ b/eo/tutorial/Parallelization/css/horizontal-slide.css @@ -0,0 +1,76 @@ +.csstransitions.csstransforms { + overflow-x: hidden; +} +.csstransitions.csstransforms .deck-container > .slide { + -webkit-transition: -webkit-transform 500ms ease-in-out; + -moz-transition: -moz-transform 500ms ease-in-out; + -ms-transition: -ms-transform 500ms ease-in-out; + -o-transition: -o-transform 500ms ease-in-out; + transition: transform 500ms ease-in-out; +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .slide { + position: absolute; + top: 0; + left: 0; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + width: 100%; + padding: 0 48px; +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .slide .slide { + position: relative; + left: 0; + top: 0; + -webkit-transition: -webkit-transform 500ms ease-in-out, opacity 500ms ease-in-out; + -moz-transition: -moz-transform 500ms ease-in-out, opacity 500ms ease-in-out; + -ms-transition: -ms-transform 500ms ease-in-out, opacity 500ms ease-in-out; + -o-transition: -o-transform 500ms ease-in-out, opacity 500ms ease-in-out; + transition: -webkit-transform 500ms ease-in-out, opacity 500ms ease-in-out; +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .slide .deck-next, .csstransitions.csstransforms .deck-container:not(.deck-menu) > .slide .deck-after { + visibility: visible; + -webkit-transform: translate3d(200%, 0, 0); + -moz-transform: translate(200%, 0); + -ms-transform: translate(200%, 0); + -o-transform: translate(200%, 0); + transform: translate3d(200%, 0, 0); +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .deck-previous { + -webkit-transform: translate3d(-200%, 0, 0); + -moz-transform: translate(-200%, 0); + -ms-transform: translate(-200%, 0); + -o-transform: translate(-200%, 0); + transform: translate3d(-200%, 0, 0); +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .deck-before { + -webkit-transform: translate3d(-400%, 0, 0); + -moz-transform: translate(-400%, 0); + -ms-transform: translate(-400%, 0); + -o-transform: translate(-400%, 0); + transform: translate3d(-400%, 0, 0); +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .deck-next { + -webkit-transform: translate3d(200%, 0, 0); + -moz-transform: translate(200%, 0); + -ms-transform: translate(200%, 0); + -o-transform: translate(200%, 0); + transform: translate3d(200%, 0, 0); +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .deck-after { + -webkit-transform: translate3d(400%, 0, 0); + -moz-transform: translate(400%, 0); + -ms-transform: translate(400%, 0); + -o-transform: translate(400%, 0); + transform: translate3d(400%, 0, 0); +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .deck-before .slide, .csstransitions.csstransforms .deck-container:not(.deck-menu) > .deck-previous .slide { + visibility: visible; +} +.csstransitions.csstransforms .deck-container:not(.deck-menu) > .deck-child-current { + -webkit-transform: none; + -moz-transform: none; + -ms-transform: none; + -o-transform: none; + transform: none; +} diff --git a/eo/tutorial/Parallelization/css/shjs.css b/eo/tutorial/Parallelization/css/shjs.css new file mode 100644 index 000000000..ec9d4088b --- /dev/null +++ b/eo/tutorial/Parallelization/css/shjs.css @@ -0,0 +1,67 @@ +pre.sh_sourceCode { + background-color: white; + color: black; + font-style: normal; + font-weight: normal; +} + +pre.sh_sourceCode .sh_keyword { color: blue; font-weight: bold; } /* language keywords */ +pre.sh_sourceCode .sh_type { color: darkgreen; } /* basic types */ +pre.sh_sourceCode .sh_usertype { color: teal; } /* user defined types */ +pre.sh_sourceCode .sh_string { color: red; font-family: monospace; } /* strings and chars */ +pre.sh_sourceCode .sh_regexp { color: orange; font-family: monospace; } /* regular expressions */ +pre.sh_sourceCode .sh_specialchar { color: pink; font-family: monospace; } /* e.g., \n, \t, \\ */ +pre.sh_sourceCode .sh_comment { color: brown; font-style: italic; } /* comments */ +pre.sh_sourceCode .sh_number { color: purple; } /* literal numbers */ +pre.sh_sourceCode .sh_preproc { color: darkblue; font-weight: bold; } /* e.g., #include, import */ +pre.sh_sourceCode .sh_symbol { color: darkred; } /* e.g., <, >, + */ +pre.sh_sourceCode .sh_function { color: black; font-weight: bold; } /* function calls and declarations */ +pre.sh_sourceCode .sh_cbracket { color: red; } /* block brackets (e.g., {, }) */ +pre.sh_sourceCode .sh_todo { font-weight: bold; background-color: cyan; } /* TODO and FIXME */ + +/* Predefined variables and functions (for instance glsl) */ +pre.sh_sourceCode .sh_predef_var { color: darkblue; } +pre.sh_sourceCode .sh_predef_func { color: darkblue; font-weight: bold; } + +/* for OOP */ +pre.sh_sourceCode .sh_classname { color: teal; } + +/* line numbers (not yet implemented) */ +pre.sh_sourceCode .sh_linenum { color: black; font-family: monospace; } + +/* Internet related */ +pre.sh_sourceCode .sh_url { color: blue; text-decoration: underline; font-family: monospace; } + +/* for ChangeLog and Log files */ +pre.sh_sourceCode .sh_date { color: blue; font-weight: bold; } +pre.sh_sourceCode .sh_time, pre.sh_sourceCode .sh_file { color: darkblue; font-weight: bold; } +pre.sh_sourceCode .sh_ip, pre.sh_sourceCode .sh_name { color: darkgreen; } + +/* for Prolog, Perl... */ +pre.sh_sourceCode .sh_variable { color: darkgreen; } + +/* for LaTeX */ +pre.sh_sourceCode .sh_italics { color: darkgreen; font-style: italic; } +pre.sh_sourceCode .sh_bold { color: darkgreen; font-weight: bold; } +pre.sh_sourceCode .sh_underline { color: darkgreen; text-decoration: underline; } +pre.sh_sourceCode .sh_fixed { color: green; font-family: monospace; } +pre.sh_sourceCode .sh_argument { color: darkgreen; } +pre.sh_sourceCode .sh_optionalargument { color: purple; } +pre.sh_sourceCode .sh_math { color: orange; } +pre.sh_sourceCode .sh_bibtex { color: blue; } + +/* for diffs */ +pre.sh_sourceCode .sh_oldfile { color: orange; } +pre.sh_sourceCode .sh_newfile { color: darkgreen; } +pre.sh_sourceCode .sh_difflines { color: blue; } + +/* for css */ +pre.sh_sourceCode .sh_selector { color: purple; } +pre.sh_sourceCode .sh_property { color: blue; } +pre.sh_sourceCode .sh_value { color: darkgreen; font-style: italic; } + +/* other */ +pre.sh_sourceCode .sh_section { color: black; font-weight: bold; } +pre.sh_sourceCode .sh_paren { color: red; } +pre.sh_sourceCode .sh_attribute { color: darkgreen; } + diff --git a/eo/tutorial/Parallelization/css/thales.css b/eo/tutorial/Parallelization/css/thales.css new file mode 100644 index 000000000..e30282966 --- /dev/null +++ b/eo/tutorial/Parallelization/css/thales.css @@ -0,0 +1,91 @@ +.deck-container em { + color:rgb(255,115,0); +} + +.deck-container s { + color:rgb(180,180,211); +} + +.deck-container { + font-family: "Helvetica Neue", sans-serif; + font-size: 1.75em; + color:RGB(50,50,101); + background:white url("./img/thales.jpg") no-repeat fixed bottom left; +} +.deck-container .slide { +} +.deck-container .slide h1 { + color:rgb(50,50,101); +} +.deck-container .slide h2 { + color:rgb(255,115,0); + border-bottom-color:lightgray; +} +.deck-container .slide h3 { + color:RGB(57,138,199); +} +.deck-container .slide pre { + border-color: #ccc; +} +.deck-container .slide blockquote { + font-size: 2em; + font-style: italic; + padding: 1em 2em; + color: #000; + border-left: 5px solid #ccc; + font-family:serif; +} +.deck-container .slide blockquote p { + margin: 0; +} +.deck-container .slide blockquote cite { + font-size: .5em; + font-style: normal; + font-weight: bold; + color: #888; +} +.deck-container .slide ::-moz-selection { + background: #c00; + color: #fff; +} +.deck-container .slide ::selection { + background: #c00; + color: #fff; +} +.deck-container .slide a, .deck-container .slide a:hover, .deck-container .slide a:focus, .deck-container .slide a:active, .deck-container .slide a:visited { + color:RGB(152,191,12); + text-decoration: none; +} +.deck-container .slide a:hover, .deck-container .slide a:focus { + text-decoration: underline; +} +.deck-container > .slide .deck-before, .deck-container > .slide .deck-previous { + opacity: 0.4; +} +.deck-container > .slide .deck-before:not(.deck-child-current) .deck-before, .deck-container > .slide .deck-before:not(.deck-child-current) .deck-previous, .deck-container > .slide .deck-previous:not(.deck-child-current) .deck-before, .deck-container > .slide .deck-previous:not(.deck-child-current) .deck-previous { + opacity: 1; +} +.deck-container > .slide .deck-child-current { + opacity: 1; +} +.deck-container .deck-prev-link, .deck-container .deck-next-link { + background: #ccc; + font-family: serif; +} +.deck-container .deck-prev-link, .deck-container .deck-prev-link:hover, .deck-container .deck-prev-link:focus, .deck-container .deck-prev-link:active, .deck-container .deck-prev-link:visited, .deck-container .deck-next-link, .deck-container .deck-next-link:hover, .deck-container .deck-next-link:focus, .deck-container .deck-next-link:active, .deck-container .deck-next-link:visited { + color: #fff; +} +.deck-container .deck-prev-link:hover, .deck-container .deck-prev-link:focus, .deck-container .deck-next-link:hover, .deck-container .deck-next-link:focus { + background: #c00; + text-decoration: none; +} +.deck-container .deck-status { + font-size: 0.6666em; +} +.deck-container.deck-menu .slide { + background: #eee; +} +.deck-container.deck-menu .deck-current, .no-touch .deck-container.deck-menu .slide:hover { + background: #ddf; +} + diff --git a/eo/tutorial/Parallelization/eompi.html b/eo/tutorial/Parallelization/eompi.html new file mode 100644 index 000000000..82533a796 --- /dev/null +++ b/eo/tutorial/Parallelization/eompi.html @@ -0,0 +1,903 @@ + + + + + + + + + + EO::MPI parallelization + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+

EO's Parallelization with MPI

+
+ +
+

What is parallelization?

+
    +

  • Divide and Conquer paradigm

    +

    A difficult problem can be splitted into simpler sub problems.

    +

    Therefore the sub problems should be solved faster.

    +
    • +

  • Parallelize.

    +

    Process serveral tasks together (concurrent programming).

    +

    By different ways : +

      +
    • Shared memory : Multitask, OpenMP, etc.
      • +
    • Message Passing Interface : MPI.
      • +
    • Other ways, undetailed here: coroutines, multi-thread / LightWeightProcess, + Field Programmable Gate Array (FPGA), General Purpose GPU (GPGPU), etc.
      • +
    +

    +
    • +
+
+ +
+

Shared memory (e.g : OpenMP)

+
    +
  • Multicore : more than one core per processor. +
      +
    • Processes several instruction streams together, each one manipulating different datas.
      • +
    • Different from superscalar processors, which can process more than one instruction during a + single processor cycle.
      • +
    • A multicore processor can however be superscalar.
      • +
    • For instance : IBM's Cell microprocessor (PlayStation3)
      • +
    +
    • + +
  • Symmetric multiprocessing : More than one processor which communicate through + a specific bus. +
      +
    • Bus contention is the principal limitating factor.
      • +
    +
    • + +
  • The main drawback is explicitly contained in the name. +
      +
    • Memory is shared
      • +
    • Bus contention?
      • +
    • Low memory?
      • +
    • Use of virtual memory (swap) and page faults?
      • +
    • => Can slow the speedup compared with the number of processors.
      • +
    +
    • +
+
+ +
+

Message Passing Interface (e.g : OpenMPI)

+

Memory isn't shared here, manipulated objects are sent on a network: there is communication between the machines + (called hosts)

+
    +
  • Cluster +
      +
    • Several machines network connected (for instance, in an Ethernet network).
      • +
    • Can have different specifications, but this can affect load balancing.
      • +
    • Most of the supercomputers are clusters.
      • +
    • For exemple : Beowulf Cluster, Sun Grid Engine...
      • +
    +
    • + +
  • Massive parallel processing +
      +
    • This is not machines but processors which are directly connected by network.
      • +
    • Like a cluster, but the networks are specific, so as to wire more processors.
      • +
    +
    • + +
  • Grid (a.k.a Distributed Computing) +
      +
    • Networks with potentially high latence and low bandwith, like Internet.
      • +
    • Example of programs : BOINC, Seti@home...
      • +
    +
    • +
+
+ +
+

Parallelization myths

+
+ +
+

A myth about speed: the car's enigma

+
    +
  • If a car has to travel 100 Km, but has yet traveled 60 Km in 1 hour (i.e it has an average + speed of 60 Km per hour), what is its maximal average speed on the whole traject + ?
    • + +

    +
  • Some hints: +
      +
    • The driver can use all the hilarious gas (N2O, nitrous oxyde, a.k.a nitro) he + needs.
      • +
    • Let's suppose even that the car can travel at speed of light, or teleport (despite general theory of + relativity).
      • +
    +
    • + +
  • The solution: 100 Km/h
    • +
  • The explanation: let's suppose that the car teleports after the first 60 Km. It would have + traveled 60 Km in 1 hour and 40 Km in 0 seconds, which is 100 Km in 1 hour: 100 Km/h.
    • +
+
+ +
+

A myth about speed: "Il dit qu'il voit pas le rapport ?"

+

+
    +
  • Let P be the parallelizable proportion of the program (~ the remaining distance to travel), which can be + processed by N machines.
    • +
  • Then 1 - P is the sequential (non parallelizable) proportion of the program (~ the already traveled distance), which + can be processed by 1 machine.
    • +
  • A sequential version would take 1 - P + P = 1 unit (of time) to terminate.
    • +
  • A parallel version would take (1-P) + P / N units to terminate.
    • +
  • The speedup (gain of speed) would then be: + + which tends to 1-P as N tends to infinity.
    • +
  • The maximal theoritical speedup (~ maximal average speed) would then be 1 / (1-P)
    • +

  • This result is known as Amdahl's law

    • +
+
+ +
+

A myth about data : the cat's enigma

+
    +
  • +

    If 3 cats catch 3 mices in 3 minutes, how many cats are needed to catch 9 mices in 9 + minutes ? + +

    +
    • +
  • The solution: 3 too !
    • +
  • A better question to ask would be: how many mices can catch 9 cats in 3 minutes ?
    • +
+
+ +
+

A myth about data

+
    +
  • The idea is no more to make a processing faster (for a constant amount of data) but to process more data (in + the same time).
    • +
  • In this case, the most hosts we add, the most data we can process at a time.
    • +
  • This doesn't contradict Amdahl's Law, which makes the assumption that the amount of data to process stays + the same.
    • +
  • There is also another way to calculate speedup here: + + where P is the number of processes, alpha the non parallelizable part of the program. +
    • +

  • This result is known as the Gustafson's Law

    • +
+
+ +
+

A metric: speedup

+
    +
  • Speedup refers to how much a parallel algorithm is faster than a corresponding sequential algorithm
    • +
  • It's a quantitative mesure, relative to the sequential version : + + where T1 is the time taken by the sequential version and Tp the time taken by the parallel version with p + processes. +
    • +
  • A speedup equal to one indicates that the version is as performant as the sequential version.
    • +
  • Practically, speedup may not be linear in number of processes (Amdahl's Law)
    • +
+
+ +
+

Parallelization in EO

+
+ +
+

Objectives

+
    +
  • Remember, tasks have to be independant from one to another.
    • +
  • Process data faster: what takes time in EO? +
      +
    • Evaluation!
      • +
    +
    • +
  • Process more data during the same time: where? +
      +
    • Multi-start!
      • +
    +
    • +
  • Other objectives : +
      +
    • Readily serialize EO objects.
      • +
    • Be able to easily implement other parallel algorithms.
      • +
    +
    • +
+
+ +
+

Evaluation: Long story short

+ 

+    int main( int argc, char **argv )
+    {
+        eo::mpi::Node::init( argc, argv );
+        // PUT EO STUFF HERE
+        // Let's make the assumption that pop is a eoPop<EOT>
+        // and evalFunc is an evaluation functor
+        eo::mpi::DynamicAssignmentAlgorithm assign;
+        eoParallelPopLoopEval<EOT> popEval( assign, eo::mpi::DEFAULT_MASTER, evalFunc );
+        popEval( pop, pop );
+    }
+
+
+ +
+

Serializing EO objects

+
    +
  • Serializing is writing a message from a binary source into a message transmissible data.
    • +
  • Several formats can be used +
      +
    • Binary directly - but all the hosts have to be the same!
      • +
    • Text based formats: XML, YAML, JSON,...
      • +
    +
    • +
  • So why use text? +
      +
    • It's human readable, more or less easily parsable.
      • +
    • It's independant from the data type representations on the machines (e.g: an int on a 32 bits and on + a 64 bits machines are not the same).
      • +
    • Main drawbacks: it takes more space and it needs a processing for encoding and decoding.
      • +
    +
    • +
+
+ +
+

eoserial : principle

+ +
    +
  • JSON serialization +
      +
    • Lighter than XML.
      • +
    • Easily parsable, the grammar is trivial.
      • +
    • Allows to represent tables, objects and texts: it's sufficient!
      • +
    +
    • +
  • What happens in your life when you're serializable? +
    • +
  • Implement interface eoserial::Persistent and your object can be saved and loaded, in JSON format.
    • +
  • No need to serialize the whole object, you choose what you need to save and load.
    • +
  • Everything can be serialized!
      +
    • Atomic types are directly serialized into eoserial::String (thanks to std::stringstream)
      • +
    • Arrays are serializable (into eoserial::Array), if what they contain is too.
      • +
    • Object can be serializable (into eoserial::Object), if what they contain is too.
      • +
    • +
+
+ +
+

eoserial : interface eoserial::Persistent

+ 
+    
+    # include <serial/eoSerial.h>
+
+    class MyObject : public eoserial::Persistent {
+        public:
+
+        // A persistent class needs a default empty ctor.
+        MyObject() {}
+
+        int id;
+
+        // Implementation of eoserial::Persistent::pack
+        // What to save when making a serialized object?
+        eoserial::Object* pack() const
+        {
+            eoserial::Object* obj = new eoserial::Object;
+            // eoserial::make creates a eoserial::String from a basic type
+            eoserial::String* idAsString = eoserial::make( id );
+            // the key "saved_id" will be associated to the JSON object idAsString
+            obj->add( "saved_id", idAsString );
+            // could have be done with
+            // (*obj)["saved_id"] = idAsString;
+            // as obj is a std::map pointer
+            return obj;
+        }
+
+        // Implementation of eoserial::Persistent::unpack
+        // What data to retrieve from a JSON object and where to put it?
+        void unpack(const eoserial::Object* json)
+        {
+            // retrieves the value from key "saved_id" in "*json" object and put it into member "id"
+            eoserial::unpack( *json, "saved_id" , id );
+        }
+    };
+    
+
+
+ +
+

eoserial : use it

+ 
+    
+# include <eoSerial.h>
+# include <fstream>
+# include <cassert>
+
+    int main(void)
+    {
+        MyObject instance;
+        instance.id = 42;
+
+        // Writes
+        eoserial::Object* obj = instance.pack();
+        std::ofstream ofile("filename");
+        obj->print( ofile );
+        ofile.close();
+        delete obj;
+
+        // Reads
+        std::ifstream ifile("filename");
+        std::stringstream ss;
+        while( ifile )
+        {
+            std::string s;
+            ifile >> s;
+            ss << s;
+        }
+        eoserial::Object* objCopy = eoserial::Parser::parse( ss.str() );
+        MyObject instanceCopy;
+        instanceCopy.unpack( objCopy );
+
+        assert( instanceCopy.id == instance.id );
+
+        return 0;
+    }
+    
+
+
+ +
+

eoserial : more complex uses

+ 

+    struct ComplexObject
+    {
+        bool someBooleanValue; // will be serialized into a string
+        MyObject obj; // Objects can contain other objects too
+        std::vector<int>; // and tables too!
+    };
+
+    int main(void)
+    {
+        ComplexObject co;
+        // let's imagine we've set values of co.
+        eoserial::Object* json = new eoserial::Object;
+        // serialize basic type
+        (*json)["some_boolean_value"] = eoserial::make( co.someBooleanValue );
+        // MyObject is Persistent, so eoserial knows how to serialize it
+        json->add( "my_object", &co.obj );
+        // Instead of having a "for" loop, let's automatically serialize the content of the array
+        json->add( "int_array",
+            eoserial::makeArray< std::vector<int>, eoserial::MakeAlgorithm >( co.array ) );
+        // Print everything on the standard output
+        json->print( std::cout );
+        delete json;
+        return 0;
+    }
+
+
+ +
+

MPI

+
    +
  • We know how to serialize our objects. Now, we need to transmit them over the network.
    • +
  • Message Passing Interface (MPI) is a norm.
    • +
  • OpenMPI implements it, in C, in the SPMD (Single Program Multiple Data) fashion. It is an active community + and the library is very well documented.
    • +
  • Boost::mpi gives it a C++ flavour (and tests each status code returned by MPI calls, throwing up exceptions + instead).
    • +
  • MPI helps by: +
      +
    • Managing the administration of roles: each MPI process has a rank and knows the whole + size of the cluster.
      • +
    • Regrouping outputs of different processes into one single output.
      • +
    • Managing the routing of messages and connections between the processes.
      • +
    • Launch a given number of processes via SSH, or a cluster engine (like SGE).
      • +
    +
    • +
  • MPI doesn't deal with: +
      +
    • Debugging: if one of your program segfaults, buy a parallel debugger or... Good luck!
      • +
    • More generally, knowing what happens: even the standard output becomes a shared resource without any + protection!
      • +
    +
    • +
+
+ +
+

Design of parallel algorithms

+
+ +
+

Some vocabulary

+
    +
  • In the most of cases, we want the results to be retrieved in one place. Besides, communication in MPI is + synchronous (it's a design choice making things are simpler).
    • +
  • One process will have particular responsabilities, like aggregating results: it's the master.
    • +
  • Other processes will be used to do the processing (it's the goal, after all?) : they're the + workers. Or slaves, but it may be patronizing and the master is rarely called + the patron.
    • +
  • As there is one master, the algorithm is said to be centralized. Some well-known parallel algorithms + use this paradigm: Google's MapReduce, Apache's Hadoop(free implementation of Google's one :-)),...
    • +
  • A job is the parallel algorithm seen in its globality (i.e., as a function).
    • +
  • A job is a set of tasks, which are the atomic, decomposed part which can be serialized and + processed by a worker, at a time.
    • +
+
+ +
+

Evaluation (1/2)

+

+ Let's see how we could implement our parallelized evaluation
+ It's feasible as evaluating an individual is independant from evaluating another one.
+ 


+        // On master side
+        function parallel_evaluate( population p )
+            foreach individual i in p,
+                send i to a worker
+                if there is no available worker,
+                    wait for any response (return)
+                    and retry
+                endif
+            endforeach
+            inform all the available workers that they are done (yes, it's a centralized algorithm)
+            wait for all remaining responses
+        endfunction
+
+        when receiving a response:
+            replace the evaluated individual in the population
+
+        // On worker side
+        function parallel_evaluate( evaluation function f )
+            wait for a individual i
+            apply f on it
+            send i to the master
+        endfunction
+
+

+
+ +
+

Evaluation (2/2)

+

But a parallelization algorithm is interesting only if the process time is higher than the + communication time. If process time is too short relatively to the communication time, we can do the following: + 


+        // On master side
+        function parallel_evaluate( population p, number of elements to send each time packet_size )
+            index = 0
+            while index < size
+                sentSize := how many individuals (<= packet_size) can we send to a worker?
+                find a worker. If there is no one, wait for any response (return) and retry
+                send the sentSize to the worker
+                send the individuals to the worker
+                index += sentSize
+            endwhile
+            inform all the available workers that they're done
+            wait for all remaining responses
+        endfunction
+
+        when receiving a response:
+            replace the evaluated individuals in the population
+
+        // On worker side
+        function parallel_evaluate( evaluation function f )
+        size := wait for a sentSize as described above
+            individuals := wait for size individuals
+            apply f on each of them
+            send back the individuals
+        endfunction
+
+
+ +
+

Multi start

+

The idea behing multi-start is to run many times the same algorithm (for instance, eoEasyEA), but with different + seeds: the workers launch the algorithm and send their solutions as they come to the master, which saves the + ultimate best solution.

+ 
+        // On master side
+        variable best_score (initialized at the worst value ever) // score can be fitness, for instance
+
+        function parallel_multistart( integer runs )
+            seeds = table of generated seeds, or fixed seeds, whose size is at least "runs"
+            for i := 0; i < runs; ++i
+                find a worker. If there is no one, wait for any response (return) and retry
+                send to the worker a different seed
+            endfor
+            inform all the available workers that they're done
+            wait for all remaining responses
+        endfunction
+
+        when receiving a response:
+            received_score := receive score from the worker.
+            If the received_score > best_score
+                send worker a message indicating that master is interested by the solution
+                receive the solution
+                updates the best_score
+            else
+                send worker a message indicating that master isn't interested by the solution
+            endif
+
+        // On worker side
+        function parallel_multistart( algorithm eoAlgo )
+            seed := wait for a seed
+            solution := eoAlgo( seed )
+            send solution score to master
+            master_is_interested := wait for the response
+            if master_is_interested
+                send solution to master
+            endif
+        endfunction
+
+
+ +
+

Common parts vs specific parts

+
    +
  • These two algorithms have common parts and specific parts.
    • +
  • Identifying them allows to design generic parallel algorithms.
    • +
  • In the following code sample, specific parts are in red. Everything else is hence generic.
    • +
+ + 

+        // On master side
+        function parallel_evaluate(population p, number of elements to send each time packet_size )
+            index = 0
+            while index < size
+                find a worker. If there is no one, wait for any response (return) and retry
+                sentSize := how many individuals (<= packet_size) can we send to a worker?
+                send the sentSize to the worker
+                send the individuals to the worker
+                index += sentSize
+            endwhile
+            inform all the available workers that they're done
+            wait for all remaining responses
+        endfunction
+
+        when receiving a response:
+            replace the evaluated individuals in the population
+
+        // On worker side
+        function parallel_evaluate( evaluation function f )
+        size := wait for a sentSize as described above
+            individuals := wait for size individuals
+            apply f on each of them
+            send back the individuals
+        endfunction
+
+ +
+ +
+

Common parts

+
    +
  • Master runs a loop.
    • +
  • Master has to manage workers (find them, wait for them, etc...)
    • +
  • Workers need to be informed if they have something to do or not (stop condition in master part)
    • +
  • Master needs to wait to get all the responses.
    • +
+
+ +
+

Specific parts

+
    +
  • Loop condition in the master's part.
    • +
  • What has to be sent to a worker by master?
    • +
  • What has to be done by a worker when it receives an order?
    • +
  • What has to be done when the master receives a response?
    • +
+
+ +
+

Generic parallel algorithm

+

The calls to specific parts are in red.

+ 

+    // Master side
+    function parallel_algorithm()
+        while ! isFinished()
+            worker := none
+            while worker is none
+                wait for a response and affect worker the origin of the response
+                handleResponse( worker )
+                worker = retrieve worker
+            endwhile
+            send worker a work order
+            sendTask( worker )
+        endwhile
+
+        foreach available worker
+            indicate worker it's done (send them a termination order)
+        endforeach
+
+        while all responses haven't be received
+            worker := none
+            wait for a response and affect worker the origin of the response
+            handleResponse( worker )
+            send worker a termination order
+        endwhile
+    endfunction
+
+    // Worker side
+    function parallel_algorithm()
+        order := receive order
+        while order is not termination order
+            processTask( )
+            order = receive order
+        endwhile
+    endfunction
+
+
+ +
+

TLDR;

+ +
+ +
+

Functors

+
    +
  • Using functors allows them to wrap and be wrapped (decorator pattern).
    • +
  • IsFinished : implements bool operator()(), indicating that the job is over.
    • +
  • SendTask : implements void operator()( int worker_rank ), indicating what to send to the + worker.
    • +
  • ProcessTask : implements void operator()(), indicating what the worker has to do when it receives a + task.
    • +
  • HandleResponse : implements void operator()( int worker_rank ), indicating what to do when + receiving a worker response.
    • +
  • Implementing these 4 functors is sufficient for a parallel algorithm!
    • +
  • You can also wrap the existing one to add functionalities.
    • +
+
+ +
+

Stores

+
    +
  • These 4 functors can use about the same data.
    • +
  • This data needs to be shared : all the functors are templated on a JobData structure.
    • +
  • A job needs data and functors to be launched.
    • +
  • Several jobs can use the same data and functors.
    • +
  • => Data and functors are saved into a store, which can be reused between different jobs.
    • +
+
+ +
+

Scheduling tasks between workers

+
    +
  • Until here, we don't know how to schedule tasks between workers.
    • +
  • Naive, simple solution: as soon as a worker has finished a task, give it a new task. Workers are put in a + queue, this is the dynamic assignment (scheduling).
    • +
  • If the worker's number of call is well-known, initially give to each worker a fixed amount of tasks. When a + worker has finished a task, give it another task only if it the amount of remaining tasks is positive ; else, + wait for another worker. Workers are managed with a fixed table, this is the static + assignment.
    • +
+
+ +
+

Let's go back to evaluation in EO

+
    +
  • The idea behind applying a functor to each element of a table is very generic. Google, Python and Javascript + call it map, we call it ParallelApply, according to existing + apply function, in apply.h.
    • +
  • There is also a ParallelApplyJob, a ParallelApplyStore which contains a + ParallelApplyData, a IsFinishedParallelApply, etc...
    • +
  • This is what is used when calling parallel evaluation.
    • +
+
+ +
+

Customizing evaluation: reminder

+ 

+    int main( int argc, char **argv )
+    {
+        eo::mpi::Node::init( argc, argv );
+        // PUT EO STUFF HERE
+        // Let's make the assumption that pop is a eoPop<EOT>
+        // and evalFunc is an evaluation functor
+        eo::mpi::DynamicAssignmentAlgorithm assign;
+        eoParallelPopLoopEval<EOT> popEval( assign, eo::mpi::DEFAULT_MASTER, evalFunc );
+        // The store is hidden behind this call, but it can be given at eoParallelPopLoopEval constructor!
+        popEval( pop, pop );
+    }
+
+
+ +
+

Customizing evaluation: the idea

+
    +
  • We would like to retrieve best individuals, as soon as they're processed, and print their fitness in the + standard output, for instance.
    • +
  • We can wrap one of the 4 functors.
    • +
  • Master side or worker side? +
      +
    • Master side: we want to retrieve the global best individual, not the best individual in + population slices.
      • +
    • We have 3 choices: IsFinished, HandleResponse, SendTask.
      • +
    • So which one? +
        +
      • The functor HandleResponse should be reimplemented: in a sequential version, it would be done just + after the evaluation of an individual. The HandleResponse is the nearest functor called after having + received the result.
        • +
      +
      • +
    +
    • +
  • How to do it? +
      +
    1. Retrieve which slice has been processed by the worker.
      2. +
    2. Call the embedded HandleResponse.
      3. +
    3. Compare the fitnesses of individuals in the slice to the global best individual.
      4. +
    +
    • +
+
+ +
+

Customizing evaluation: implementation!

+ 

+    // Our objective is to minimize fitness, for instance
+    struct CatBestAnswers : public eo::mpi::HandleResponseParallelApply<EOT>
+    {
+        CatBestAnswers()
+        {
+            best.fitness( 1000000000. );
+        }
+
+        void operator()(int wrkRank)
+        {
+            // Retrieve informations about the slice processed by the worker
+            int index = _data->assignedTasks[wrkRank].index;
+            int size = _data->assignedTasks[wrkRank].size;
+            // call to the wrapped function HERE
+            (*_wrapped)( wrkRank );
+            // Compare fitnesses of evaluated individuals with the best saved
+            for(int i = index; i < index+size; ++i)
+            {
+                if( best.fitness() < _data->table()[ i ].fitness() )
+                {
+                    eo::log << eo::quiet << "Better solution found:" << _data->table()[i].fitness() << std::endl;
+                    best = _data->table()[ i ];
+                }
+            }
+        }
+
+        protected:
+
+        EOT best;
+    };
+
+
+ +
+

Using customized handler

+ 

+        int main( int argc, char **argv )
+        {
+            eo::mpi::Node::init( argc, argv );
+            // PUT EO STUFF HERE
+            // Let's make the assumption that pop is a eoPop<EOT>
+            // and evalFunc is an evaluation functor
+            eo::mpi::DynamicAssignmentAlgorithm assign;
+            // What was used before
+            // eoParallelPopLoopEval<EOT> popEval( assign, eo::mpi::DEFAULT_MASTER, evalFunc );
+            // What's new
+            eo::mpi::ParallelApplyStore< EOT > store( evalFunc, eo::mpi::DEFAULT_MASTER );
+            CatBestAnswer catBestAnswers;
+            store.wrapHandleResponse( &catBestAnswers );
+
+            eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, &store );
+            // What doesn't change
+            popEval( pop, pop );
+        }
+
+
+ +
+

Thank you for your attention

+
+ +
+

Remarks

+
    +
  • This presentation is made of HTML5, CSS3, JavaScript, thanks to frameworks + Deck.js (slides) and SHJS (syntax + highlighting). +
  • If you have any complaint to make, please refer to Johann Dreo.
    • +
  • If you have any question or compliment, please refer to me + (Benjamin Bouvier).
    • +
+
+ + +← +→ + + +

+ + / + +

+ + +
+ + + + +
+ + +# + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eo/tutorial/Parallelization/img/generic_parallel.dia b/eo/tutorial/Parallelization/img/generic_parallel.dia new file mode 100644 index 000000000..693475c5b Binary files /dev/null and b/eo/tutorial/Parallelization/img/generic_parallel.dia differ diff --git a/eo/tutorial/Parallelization/img/generic_parallel.png b/eo/tutorial/Parallelization/img/generic_parallel.png new file mode 100644 index 000000000..90bbcf170 Binary files /dev/null and b/eo/tutorial/Parallelization/img/generic_parallel.png differ diff --git a/eo/tutorial/Parallelization/img/serialisation.dia b/eo/tutorial/Parallelization/img/serialisation.dia new file mode 100644 index 000000000..2a9333506 Binary files /dev/null and b/eo/tutorial/Parallelization/img/serialisation.dia differ diff --git a/eo/tutorial/Parallelization/img/serialisation.png b/eo/tutorial/Parallelization/img/serialisation.png new file mode 100644 index 000000000..76ac0a9c0 Binary files /dev/null and b/eo/tutorial/Parallelization/img/serialisation.png differ diff --git a/eo/tutorial/Parallelization/js/deck.core.js b/eo/tutorial/Parallelization/js/deck.core.js new file mode 100644 index 000000000..6fbeb5ce0 --- /dev/null +++ b/eo/tutorial/Parallelization/js/deck.core.js @@ -0,0 +1,498 @@ +/*! +Deck JS - deck.core +Copyright (c) 2011 Caleb Troughton +Dual licensed under the MIT license and GPL license. +https://github.com/imakewebthings/deck.js/blob/master/MIT-license.txt +https://github.com/imakewebthings/deck.js/blob/master/GPL-license.txt +*/ + +/* +The deck.core module provides all the basic functionality for creating and +moving through a deck. It does so by applying classes to indicate the state of +the deck and its slides, allowing CSS to take care of the visual representation +of each state. It also provides methods for navigating the deck and inspecting +its state, as well as basic key bindings for going to the next and previous +slides. More functionality is provided by wholly separate extension modules +that use the API provided by core. +*/ +(function($, deck, document, undefined) { + var slides, // Array of all the uh, slides... + current, // Array index of the current slide + $container, // Keeping this cached + + events = { + /* + This event fires whenever the current slide changes, whether by way of + next, prev, or go. The callback function is passed two parameters, from + and to, equal to the indices of the old slide and the new slide + respectively. If preventDefault is called on the event within this handler + the slide change does not occur. + + $(document).bind('deck.change', function(event, from, to) { + alert('Moving from slide ' + from + ' to ' + to); + }); + */ + change: 'deck.change', + + /* + This event fires at the beginning of deck initialization, after the options + are set but before the slides array is created. This event makes a good hook + for preprocessing extensions looking to modify the deck. + */ + beforeInitialize: 'deck.beforeInit', + + /* + This event fires at the end of deck initialization. Extensions should + implement any code that relies on user extensible options (key bindings, + element selectors, classes) within a handler for this event. Native + events associated with Deck JS should be scoped under a .deck event + namespace, as with the example below: + + var $d = $(document); + $.deck.defaults.keys.myExtensionKeycode = 70; // 'h' + $d.bind('deck.init', function() { + $d.bind('keydown.deck', function(event) { + if (event.which === $.deck.getOptions().keys.myExtensionKeycode) { + // Rock out + } + }); + }); + */ + initialize: 'deck.init' + }, + + options = {}, + $d = $(document), + + /* + Internal function. Updates slide and container classes based on which + slide is the current slide. + */ + updateStates = function() { + var oc = options.classes, + osc = options.selectors.container, + old = $container.data('onSlide'), + $all = $(); + + // Container state + $container.removeClass(oc.onPrefix + old) + .addClass(oc.onPrefix + current) + .data('onSlide', current); + + // Remove and re-add child-current classes for nesting + $('.' + oc.current).parentsUntil(osc).removeClass(oc.childCurrent); + slides[current].parentsUntil(osc).addClass(oc.childCurrent); + + // Remove previous states + $.each(slides, function(i, el) { + $all = $all.add(el); + }); + $all.removeClass([ + oc.before, + oc.previous, + oc.current, + oc.next, + oc.after + ].join(" ")); + + // Add new states back in + slides[current].addClass(oc.current); + if (current > 0) { + slides[current-1].addClass(oc.previous); + } + if (current + 1 < slides.length) { + slides[current+1].addClass(oc.next); + } + if (current > 1) { + $.each(slides.slice(0, current - 1), function(i, el) { + el.addClass(oc.before); + }); + } + if (current + 2 < slides.length) { + $.each(slides.slice(current+2), function(i, el) { + el.addClass(oc.after); + }); + } + }, + + /* Methods exposed in the jQuery.deck namespace */ + methods = { + + /* + jQuery.deck(selector, options) + + selector: string | jQuery | array + options: object, optional + + Initializes the deck, using each element matched by selector as a slide. + May also be passed an array of string selectors or jQuery objects, in + which case each selector in the array is considered a slide. The second + parameter is an optional options object which will extend the default + values. + + $.deck('.slide'); + + or + + $.deck([ + '#first-slide', + '#second-slide', + '#etc' + ]); + */ + init: function(elements, opts) { + var startTouch, + tolerance, + esp = function(e) { + e.stopPropagation(); + }; + + options = $.extend(true, {}, $[deck].defaults, opts); + slides = []; + current = 0; + $container = $(options.selectors.container); + tolerance = options.touch.swipeTolerance; + + // Pre init event for preprocessing hooks + $d.trigger(events.beforeInitialize); + + // Hide the deck while states are being applied to kill transitions + $container.addClass(options.classes.loading); + + // Fill slides array depending on parameter type + if ($.isArray(elements)) { + $.each(elements, function(i, e) { + slides.push($(e)); + }); + } + else { + $(elements).each(function(i, e) { + slides.push($(e)); + }); + } + + /* Remove any previous bindings, and rebind key events */ + $d.unbind('keydown.deck').bind('keydown.deck', function(e) { + if (e.which === options.keys.next || $.inArray(e.which, options.keys.next) > -1) { + methods.next(); + e.preventDefault(); + } + else if (e.which === options.keys.previous || $.inArray(e.which, options.keys.previous) > -1) { + methods.prev(); + e.preventDefault(); + } + }); + + /* Bind touch events for swiping between slides on touch devices */ + $container.unbind('touchstart.deck').bind('touchstart.deck', function(e) { + if (!startTouch) { + startTouch = $.extend({}, e.originalEvent.targetTouches[0]); + } + }) + .unbind('touchmove.deck').bind('touchmove.deck', function(e) { + $.each(e.originalEvent.changedTouches, function(i, t) { + if (startTouch && t.identifier === startTouch.identifier) { + if (t.screenX - startTouch.screenX > tolerance || t.screenY - startTouch.screenY > tolerance) { + $[deck]('prev'); + startTouch = undefined; + } + else if (t.screenX - startTouch.screenX < -1 * tolerance || t.screenY - startTouch.screenY < -1 * tolerance) { + $[deck]('next'); + startTouch = undefined; + } + return false; + } + }); + e.preventDefault(); + }) + .unbind('touchend.deck').bind('touchend.deck', function(t) { + $.each(t.originalEvent.changedTouches, function(i, t) { + if (startTouch && t.identifier === startTouch.identifier) { + startTouch = undefined; + } + }); + }) + .scrollLeft(0).scrollTop(0) + /* Stop propagation of key events within editable elements of slides */ + .undelegate('input, textarea, select, button, meter, progress, [contentEditable]', 'keydown', esp) + .delegate('input, textarea, select, button, meter, progress, [contentEditable]', 'keydown', esp); + + /* + Kick iframe videos, which dont like to redraw w/ transforms. + Remove this if Webkit ever fixes it. + */ + $.each(slides, function(i, $el) { + $el.unbind('webkitTransitionEnd.deck').bind('webkitTransitionEnd.deck', + function(event) { + if ($el.hasClass($[deck]('getOptions').classes.current)) { + var embeds = $(this).find('iframe').css('opacity', 0); + window.setTimeout(function() { + embeds.css('opacity', 1); + }, 100); + } + }); + }); + + if (slides.length) { + updateStates(); + } + + // Show deck again now that slides are in place + $container.removeClass(options.classes.loading); + $d.trigger(events.initialize); + }, + + /* + jQuery.deck('go', index) + + index: integer | string + + Moves to the slide at the specified index if index is a number. Index is + 0-based, so $.deck('go', 0); will move to the first slide. If index is a + string this will move to the slide with the specified id. If index is out + of bounds or doesn't match a slide id the call is ignored. + */ + go: function(index) { + var e = $.Event(events.change), + ndx; + + /* Number index, easy. */ + if (typeof index === 'number' && index >= 0 && index < slides.length) { + ndx = index; + } + /* Id string index, search for it and set integer index */ + else if (typeof index === 'string') { + $.each(slides, function(i, $slide) { + if ($slide.attr('id') === index) { + ndx = i; + return false; + } + }); + }; + + /* Out of bounds, id doesn't exist, illegal input, eject */ + if (typeof ndx === 'undefined') return; + + $d.trigger(e, [current, ndx]); + if (e.isDefaultPrevented()) { + /* Trigger the event again and undo the damage done by extensions. */ + $d.trigger(events.change, [ndx, current]); + } + else { + current = ndx; + updateStates(); + } + }, + + /* + jQuery.deck('next') + + Moves to the next slide. If the last slide is already active, the call + is ignored. + */ + next: function() { + methods.go(current+1); + }, + + /* + jQuery.deck('prev') + + Moves to the previous slide. If the first slide is already active, the + call is ignored. + */ + prev: function() { + methods.go(current-1); + }, + + /* + jQuery.deck('getSlide', index) + + index: integer, optional + + Returns a jQuery object containing the slide at index. If index is not + specified, the current slide is returned. + */ + getSlide: function(index) { + var i = typeof index !== 'undefined' ? index : current; + if (typeof i != 'number' || i < 0 || i >= slides.length) return null; + return slides[i]; + }, + + /* + jQuery.deck('getSlides') + + Returns all slides as an array of jQuery objects. + */ + getSlides: function() { + return slides; + }, + + /* + jQuery.deck('getContainer') + + Returns a jQuery object containing the deck container as defined by the + container option. + */ + getContainer: function() { + return $container; + }, + + /* + jQuery.deck('getOptions') + + Returns the options object for the deck, including any overrides that + were defined at initialization. + */ + getOptions: function() { + return options; + }, + + /* + jQuery.deck('extend', name, method) + + name: string + method: function + + Adds method to the deck namespace with the key of name. This doesn’t + give access to any private member data — public methods must still be + used within method — but lets extension authors piggyback on the deck + namespace rather than pollute jQuery. + + $.deck('extend', 'alert', function(msg) { + alert(msg); + }); + + // Alerts 'boom' + $.deck('alert', 'boom'); + */ + extend: function(name, method) { + methods[name] = method; + } + }; + + /* jQuery extension */ + $[deck] = function(method, arg) { + if (methods[method]) { + return methods[method].apply(this, Array.prototype.slice.call(arguments, 1)); + } + else { + return methods.init(method, arg); + } + }; + + /* + The default settings object for a deck. All deck extensions should extend + this object to add defaults for any of their options. + + options.classes.after + This class is added to all slides that appear after the 'next' slide. + + options.classes.before + This class is added to all slides that appear before the 'previous' + slide. + + options.classes.childCurrent + This class is added to all elements in the DOM tree between the + 'current' slide and the deck container. For standard slides, this is + mostly seen and used for nested slides. + + options.classes.current + This class is added to the current slide. + + options.classes.loading + This class is applied to the deck container during loading phases and is + primarily used as a way to short circuit transitions between states + where such transitions are distracting or unwanted. For example, this + class is applied during deck initialization and then removed to prevent + all the slides from appearing stacked and transitioning into place + on load. + + options.classes.next + This class is added to the slide immediately following the 'current' + slide. + + options.classes.onPrefix + This prefix, concatenated with the current slide index, is added to the + deck container as you change slides. + + options.classes.previous + This class is added to the slide immediately preceding the 'current' + slide. + + options.selectors.container + Elements matched by this CSS selector will be considered the deck + container. The deck container is used to scope certain states of the + deck, as with the onPrefix option, or with extensions such as deck.goto + and deck.menu. + + options.keys.next + The numeric keycode used to go to the next slide. + + options.keys.previous + The numeric keycode used to go to the previous slide. + + options.touch.swipeTolerance + The number of pixels the users finger must travel to produce a swipe + gesture. + */ + $[deck].defaults = { + classes: { + after: 'deck-after', + before: 'deck-before', + childCurrent: 'deck-child-current', + current: 'deck-current', + loading: 'deck-loading', + next: 'deck-next', + onPrefix: 'on-slide-', + previous: 'deck-previous' + }, + + selectors: { + container: '.deck-container' + }, + + keys: { + // enter, space, page down, right arrow, down arrow, + next: [13, 32, 34, 39, 40], + // backspace, page up, left arrow, up arrow + previous: [8, 33, 37, 38] + }, + + touch: { + swipeTolerance: 60 + } + }; + + $d.ready(function() { + $('html').addClass('ready'); + }); + + /* + FF + Transforms + Flash video don't get along... + Firefox will reload and start playing certain videos after a + transform. Blanking the src when a previously shown slide goes out + of view prevents this. + */ + $d.bind('deck.change', function(e, from, to) { + var oldFrames = $[deck]('getSlide', from).find('iframe'), + newFrames = $[deck]('getSlide', to).find('iframe'); + + oldFrames.each(function() { + var $this = $(this), + curSrc = $this.attr('src'); + + if(curSrc) { + $this.data('deck-src', curSrc).attr('src', ''); + } + }); + + newFrames.each(function() { + var $this = $(this), + originalSrc = $this.data('deck-src'); + + if (originalSrc) { + $this.attr('src', originalSrc); + } + }); + }); +})(jQuery, 'deck', document); diff --git a/eo/tutorial/Parallelization/js/deck.goto.js b/eo/tutorial/Parallelization/js/deck.goto.js new file mode 100644 index 000000000..eedba10b1 --- /dev/null +++ b/eo/tutorial/Parallelization/js/deck.goto.js @@ -0,0 +1,170 @@ +/*! +Deck JS - deck.goto +Copyright (c) 2011 Caleb Troughton +Dual licensed under the MIT license and GPL license. +https://github.com/imakewebthings/deck.js/blob/master/MIT-license.txt +https://github.com/imakewebthings/deck.js/blob/master/GPL-license.txt +*/ + +/* +This module adds the necessary methods and key bindings to show and hide a form +for jumping to any slide number/id in the deck (and processes that form +accordingly). The form-showing state is indicated by the presence of a class on +the deck container. +*/ +(function($, deck, undefined) { + var $d = $(document); + + /* + Extends defaults/options. + + options.classes.goto + This class is added to the deck container when showing the Go To Slide + form. + + options.selectors.gotoDatalist + The element that matches this selector is the datalist element that will + be populated with options for each of the slide ids. In browsers that + support the datalist element, this provides a drop list of slide ids to + aid the user in selecting a slide. + + options.selectors.gotoForm + The element that matches this selector is the form that is submitted + when a user hits enter after typing a slide number/id in the gotoInput + element. + + options.selectors.gotoInput + The element that matches this selector is the text input field for + entering a slide number/id in the Go To Slide form. + + options.keys.goto + The numeric keycode used to show the Go To Slide form. + + options.countNested + If false, only top level slides will be counted when entering a + slide number. + */ + $.extend(true, $[deck].defaults, { + classes: { + goto: 'deck-goto' + }, + + selectors: { + gotoDatalist: '#goto-datalist', + gotoForm: '.goto-form', + gotoInput: '#goto-slide' + }, + + keys: { + goto: 71 // g + }, + + countNested: true + }); + + /* + jQuery.deck('showGoTo') + + Shows the Go To Slide form by adding the class specified by the goto class + option to the deck container. + */ + $[deck]('extend', 'showGoTo', function() { + $[deck]('getContainer').addClass($[deck]('getOptions').classes.goto); + $($[deck]('getOptions').selectors.gotoInput).focus(); + }); + + /* + jQuery.deck('hideGoTo') + + Hides the Go To Slide form by removing the class specified by the goto class + option from the deck container. + */ + $[deck]('extend', 'hideGoTo', function() { + $($[deck]('getOptions').selectors.gotoInput).blur(); + $[deck]('getContainer').removeClass($[deck]('getOptions').classes.goto); + }); + + /* + jQuery.deck('toggleGoTo') + + Toggles between showing and hiding the Go To Slide form. + */ + $[deck]('extend', 'toggleGoTo', function() { + $[deck]($[deck]('getContainer').hasClass($[deck]('getOptions').classes.goto) ? 'hideGoTo' : 'showGoTo'); + }); + + $d.bind('deck.init', function() { + var opts = $[deck]('getOptions'), + $datalist = $(opts.selectors.gotoDatalist), + slideTest = $.map([ + opts.classes.before, + opts.classes.previous, + opts.classes.current, + opts.classes.next, + opts.classes.after + ], function(el, i) { + return '.' + el; + }).join(', '), + rootCounter = 1; + + // Bind key events + $d.unbind('keydown.deckgoto').bind('keydown.deckgoto', function(e) { + var key = $[deck]('getOptions').keys.goto; + + if (e.which === key || $.inArray(e.which, key) > -1) { + e.preventDefault(); + $[deck]('toggleGoTo'); + } + }); + + /* Populate datalist and work out countNested*/ + $.each($[deck]('getSlides'), function(i, $slide) { + var id = $slide.attr('id'), + $parentSlides = $slide.parentsUntil(opts.selectors.container, slideTest); + + if (id) { + $datalist.append('