diff --git a/eo/.gitignore b/eo/.gitignore new file mode 100644 index 000000000..2bfc9fc77 --- /dev/null +++ b/eo/.gitignore @@ -0,0 +1,5 @@ +*.swp +debug/ +release/ +*CMakeFiles* +*Makefile diff --git a/eo/CMakeLists.txt b/eo/CMakeLists.txt index 63de5b229..8089c4164 100644 --- a/eo/CMakeLists.txt +++ b/eo/CMakeLists.txt @@ -45,10 +45,12 @@ ENABLE_LANGUAGE(C) ### 2) Include required modules / configuration files ##################################################################################### -FIND_PACKAGE(OpenMP) -IF(OPENMP_FOUND) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") +IF(WITH_OMP) + FIND_PACKAGE(OpenMP) + IF(OPENMP_FOUND) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + ENDIF() ENDIF() INCLUDE(CMakeBackwardCompatibilityCXX) diff --git a/eo/eo-conf.cmake b/eo/eo-conf.cmake index 9d3d5c3a8..7b1ab6419 100644 --- a/eo/eo-conf.cmake +++ b/eo/eo-conf.cmake @@ -6,3 +6,13 @@ SET(PROJECT_VERSION_PATCH 0) SET(PROJECT_VERSION_MISC "-edge") # ADD_DEFINITIONS(-DDEPRECATED_MESSAGES) # disable warning deprecated function messages +# If you plan to use OpenMP, put the following boolean to true : +SET(WITH_OMP FALSE CACHE BOOL "Use OpenMP ?" FORCE) + +# If you plan to use MPI, precise here where are the static libraries from +# openmpi and boost::mpi. + +SET(WITH_MPI FALSE CACHE BOOL "Use mpi ?" FORCE) +SET(MPI_DIR "/mpi/directory" CACHE PATH "OpenMPI directory" FORCE) +SET(BOOST_DIR "/boost/directory" CACHE PATH "Boost directory" FORCE) + diff --git a/eo/src/CMakeLists.txt b/eo/src/CMakeLists.txt index c03bc509a..caeb673ad 100644 --- a/eo/src/CMakeLists.txt +++ b/eo/src/CMakeLists.txt @@ -5,7 +5,32 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) ###################################################################################### -### 2) Define the eo target +### 2) Optional: add MPI and Boost MPI dependencies. +###################################################################################### + +IF(WITH_MPI) + MESSAGE("[EO] Compilation with MPI and BoostMPI.") + + SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") + + # headers location + INCLUDE_DIRECTORIES(${MPI_DIR}/include) + INCLUDE_DIRECTORIES(${BOOST_DIR}/include) + + # lib location + LINK_DIRECTORIES(${MPI_DIR}/lib) + LINK_DIRECTORIES(${BOOST_DIR}/lib) + + # for conditional compilation in code + ADD_DEFINITIONS(-DWITH_MPI) + + LINK_LIBRARIES(boost_mpi boost_serialization) + + ADD_SUBDIRECTORY(mpi) +ENDIF() + +###################################################################################### +### 3) Define the eo target ###################################################################################### SET(EO_LIB_OUTPUT_PATH ${EO_BINARY_DIR}/lib) @@ -27,14 +52,14 @@ FILE(GLOB HDRS *.h eo) INSTALL(FILES ${HDRS} DESTINATION include/eo COMPONENT headers) ###################################################################################### -### 3) Optionnal: define your target(s)'s version: no effect for windows +### 4) Optionnal: define your target(s)'s version: no effect for windows ###################################################################################### SET(EO_VERSION ${GLOBAL_VERSION}) SET_TARGET_PROPERTIES(eo PROPERTIES VERSION "${EO_VERSION}") ###################################################################################### -### 4) Where must cmake go now ? +### 5) Where must cmake go now ? ###################################################################################### ADD_SUBDIRECTORY(do) @@ -43,6 +68,7 @@ ADD_SUBDIRECTORY(ga) ADD_SUBDIRECTORY(gp) ADD_SUBDIRECTORY(other) ADD_SUBDIRECTORY(utils) +ADD_SUBDIRECTORY(serial) IF(ENABLE_PYEO) ADD_SUBDIRECTORY(pyeo) diff --git a/eo/src/apply.h b/eo/src/apply.h index 01256e059..2ead48c5c 100644 --- a/eo/src/apply.h +++ b/eo/src/apply.h @@ -36,6 +36,12 @@ #include #endif +# ifdef WITH_MPI +# include +# include +# include +# endif // WITH_MPI + /** Applies a unary function to a std::vector of things. @@ -51,29 +57,29 @@ void apply(eoUF& _proc, std::vector& _pop) double t1 = 0; if ( eo::parallel.enableResults() ) - { - t1 = omp_get_wtime(); - } + { + t1 = omp_get_wtime(); + } if (!eo::parallel.isDynamic()) - { + { #pragma omp parallel for if(eo::parallel.isEnabled()) //default(none) shared(_proc, _pop, size) - for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } - } + for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } + } else - { + { #pragma omp parallel for schedule(dynamic) if(eo::parallel.isEnabled()) - //doesnot work with gcc 4.1.2 - //default(none) shared(_proc, _pop, size) - for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } - } + //doesnot work with gcc 4.1.2 + //default(none) shared(_proc, _pop, size) + for (size_t i = 0; i < size; ++i) { _proc(_pop[i]); } + } if ( eo::parallel.enableResults() ) - { - double t2 = omp_get_wtime(); - eoLogger log; - log << eo::file(eo::parallel.prefix()) << t2 - t1 << ' '; - } + { + double t2 = omp_get_wtime(); + eoLogger log; + log << eo::file(eo::parallel.prefix()) << t2 - t1 << ' '; + } #else // _OPENMP @@ -82,6 +88,21 @@ void apply(eoUF& _proc, std::vector& _pop) #endif // !_OPENMP } +#ifdef WITH_MPI +template +void parallelApply( + std::vector& _pop, + eo::mpi::AssignmentAlgorithm& _algo, + int _masterRank, + eo::mpi::ParallelEvalStore & _store ) +{ + _store.data( _pop ); + _algo.reinit( _pop.size() ); + eo::mpi::ParallelApply job( _algo, _masterRank, _store ); + job.run(); +} +#endif + /** This is a variant of apply which is called in parallel thanks to OpenMP. diff --git a/eo/src/eo b/eo/src/eo index 8cebd23a0..23df6ee5f 100644 --- a/eo/src/eo +++ b/eo/src/eo @@ -141,6 +141,9 @@ #include // includes eoRealBounds.h #include // no eoIntVectorBounds +// Serialization stuff +#include + // aliens #include #include diff --git a/eo/src/eoEasyEA.h b/eo/src/eoEasyEA.h index 2c7c5474c..4d74932f7 100644 --- a/eo/src/eoEasyEA.h +++ b/eo/src/eoEasyEA.h @@ -102,6 +102,33 @@ template class eoEasyEA: public eoAlgo offspring.reserve(_offspringSize); // This line avoids an incremental resize of offsprings. } + /** + * @brief Ctor allowing to specify which pop eval function we're going to use. + * + * Ctor taking a breed and merge, an overload of ctor to define an offspring size, and + * the pop eval function used. This allows to precise if we would like to use the + * parallel evaluation, for instance. + */ + eoEasyEA( + eoContinue& _continuator, + eoEvalFunc& _eval, + eoPopEvalFunc& _pop_eval, + eoBreed& _breed, + eoReplacement& _replace, + unsigned _offspringSize + ) : continuator(_continuator), + eval (_eval), + loopEval(_eval), + popEval(_pop_eval), + selectTransform(dummySelect, dummyTransform), + breed(_breed), + mergeReduce(dummyMerge, dummyReduce), + replace(_replace), + isFirstCall(true) + { + offspring.reserve(_offspringSize); // This line avoids an incremental resize of offsprings. + } + /* eoEasyEA(eoContinue & _continuator, eoPopEvalFunc & _pop_eval, @@ -219,6 +246,8 @@ template class eoEasyEA: public eoAlgo /// Apply a few generation of evolution to the population. virtual void operator()(eoPop& _pop) { + + eo::log << "[EasyEA] Call to operator()" << std::endl; if (isFirstCall) { size_t total_capacity = _pop.capacity() + offspring.capacity(); @@ -227,22 +256,33 @@ template class eoEasyEA: public eoAlgo isFirstCall = false; } + // TODO TODOB delete all log traces + std::cout << "[EasyEA] After is first call." << std::endl; + eoPop empty_pop; + std::cout << "[EasyEA] After empty_pop." << std::endl; popEval(empty_pop, _pop); // A first eval of pop. + std::cout << "[EasyEA] After pop_eval." << std::endl; do { try { + std::cout << "[EasyEA] Beginning try." << std::endl; unsigned pSize = _pop.size(); + std::cout << "[EasyEA] psize determinated." << std::endl; offspring.clear(); // new offspring + std::cout << "[EasyEA] offspring cleared." << std::endl; breed(_pop, offspring); + std::cout << "[EasyEA] After breed, evaluating pop." << std::endl; popEval(_pop, offspring); // eval of parents + offspring if necessary + std::cout << "[EasyEA] After evaluation, replacing pop." << std::endl; replace(_pop, offspring); // after replace, the new pop. is in _pop + std::cout << "[EasyEA] After replacing, continuator." << std::endl; if (pSize > _pop.size()) throw std::runtime_error("Population shrinking!"); diff --git a/eo/src/eoPopEvalFunc.h b/eo/src/eoPopEvalFunc.h index aba74d9ab..d37eebafa 100644 --- a/eo/src/eoPopEvalFunc.h +++ b/eo/src/eoPopEvalFunc.h @@ -77,6 +77,72 @@ private: eoEvalFunc & eval; }; +#ifdef WITH_MPI +// TODO TODOB commenter +template +class eoParallelPopLoopEval : public eoPopEvalFunc +{ + public: + /** Ctor: set value of embedded eoEvalFunc */ + eoParallelPopLoopEval( + // Job parameters + eo::mpi::AssignmentAlgorithm& _assignAlgo, + int _masterRank, + // Default parameters for store + eoEvalFunc & _eval, + int _packetSize = 1 + ) : + assignAlgo( _assignAlgo ), + masterRank( _masterRank ), + needToDeleteStore( true ) + { + store = new eo::mpi::ParallelEvalStore( _eval, _masterRank, _packetSize ); + } + + eoParallelPopLoopEval( + // Job parameters + eo::mpi::AssignmentAlgorithm& _assignAlgo, + int _masterRank, + eo::mpi::ParallelEvalStore* _store + ) : + assignAlgo( _assignAlgo ), + masterRank( _masterRank ), + store( _store ), + needToDeleteStore( false ) + { + // empty + } + + ~eoParallelPopLoopEval() + { + if( eo::mpi::Node::comm().rank() == masterRank ) + { + eo::mpi::EmptyJob job( assignAlgo, masterRank ); + job.run(); + } + + if( needToDeleteStore ) + { + delete store; + } + } + + /** Do the job: simple loop over the offspring */ + void operator()( eoPop & _parents, eoPop & _offspring ) + { + (void)_parents; + parallelApply(_offspring, assignAlgo, masterRank, *store); + } + + private: + eo::mpi::AssignmentAlgorithm & assignAlgo; + int masterRank; + + eo::mpi::ParallelEvalStore* store; + bool needToDeleteStore; +}; +#endif + ///////////////////////////////////////////////////////////// // eoTimeVaryingLoopEval ///////////////////////////////////////////////////////////// diff --git a/eo/src/mpi/CMakeLists.txt b/eo/src/mpi/CMakeLists.txt new file mode 100644 index 000000000..0e22e47e0 --- /dev/null +++ b/eo/src/mpi/CMakeLists.txt @@ -0,0 +1,32 @@ +###################################################################################### +### 1) Include the sources +###################################################################################### + +INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/src) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +###################################################################################### +### 2) Define the eompi target +###################################################################################### + +SET(EOMPI_LIB_OUTPUT_PATH ${EO_BINARY_DIR}/lib) +SET(LIBRARY_OUTPUT_PATH ${EOMPI_LIB_OUTPUT_PATH}) + +SET(EOMPI_SOURCES + eoMpi.cpp + ) + +ADD_LIBRARY(eompi STATIC ${EOMPI_SOURCES}) +INSTALL(TARGETS eompi ARCHIVE DESTINATION lib COMPONENT libraries) + +FILE(GLOB HDRS *.h) +INSTALL(FILES ${HDRS} DESTINATION include/eo/mpi COMPONENT headers) + +###################################################################################### +### 3) Optionnal +###################################################################################### + +SET(EOMPI_VERSION ${GLOBAL_VERSION}) +SET_TARGET_PROPERTIES(eompi PROPERTIES VERSION "${EOMPI_VERSION}") + +###################################################################################### diff --git a/eo/src/mpi/eoMpi.cpp b/eo/src/mpi/eoMpi.cpp new file mode 100644 index 000000000..9ffc84bda --- /dev/null +++ b/eo/src/mpi/eoMpi.cpp @@ -0,0 +1,12 @@ +# include "eoMpi.h" + +// MpiNode* MpiNodeStore::singleton; +namespace eo +{ + namespace mpi + { + bmpi::communicator Node::_comm; + eoTimerStat timerStat; + } +} + diff --git a/eo/src/mpi/eoMpi.h b/eo/src/mpi/eoMpi.h new file mode 100644 index 000000000..0ef80c120 --- /dev/null +++ b/eo/src/mpi/eoMpi.h @@ -0,0 +1,391 @@ +# ifndef __EO_MPI_H__ +# define __EO_MPI_H__ + +# include +# include +# include +# include + +# include +# include +# include +# include + +# include "eoMpiNode.h" +# include "eoMpiAssignmentAlgorithm.h" + +// TODO TODOB comment! + +namespace eo +{ + namespace mpi + { + extern eoTimerStat timerStat; + + namespace Channel + { + const int Commands = 0; + const int Messages = 1; + } + + namespace Message + { + const int Continue = 0; + const int Finish = 1; // TODO commentaire : différence entre finir une tâche et arrêter le worker à expliciter. + const int Kill = 2; + } + + const int DEFAULT_MASTER = 0; + + template< typename JobData, typename Wrapped > + struct SharedDataFunction + { + SharedDataFunction( Wrapped * w = 0 ) : _wrapped( w ) + { + // empty + } + + void wrapped( Wrapped * w ) + { + _wrapped = w; + } + + void data( JobData* _d ) + { + d = _d; + if( _wrapped ) + { + _wrapped->data( _d ); + } + } + + protected: + JobData* d; + Wrapped* _wrapped; + }; + + template< typename JobData > + struct SendTaskFunction : public eoUF, public SharedDataFunction< JobData, SendTaskFunction > + { + public: + + SendTaskFunction( SendTaskFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~SendTaskFunction() {} + }; + + template< typename JobData > + struct HandleResponseFunction : public eoUF, public SharedDataFunction< JobData, HandleResponseFunction > + { + public: + + HandleResponseFunction( HandleResponseFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~HandleResponseFunction() {} + }; + + template< typename JobData > + struct ProcessTaskFunction : public eoF, public SharedDataFunction< JobData, ProcessTaskFunction > + { + public: + + ProcessTaskFunction( ProcessTaskFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~ProcessTaskFunction() {} + }; + + template< typename JobData > + struct IsFinishedFunction : public eoF, public SharedDataFunction< JobData, IsFinishedFunction > + { + public: + + IsFinishedFunction( IsFinishedFunction* w = 0 ) : SharedDataFunction >( w ) + { + // empty + } + + virtual ~IsFinishedFunction() {} + }; + + template< typename JobData > + struct JobStore + { + JobStore( + SendTaskFunction* stf, + HandleResponseFunction* hrf, + ProcessTaskFunction* ptf, + IsFinishedFunction* iff + ) : + _stf( stf ), _hrf( hrf ), _ptf( ptf ), _iff( iff ) + { + // empty + } + + JobStore() + { + // empty + } + + SendTaskFunction & sendTask() { return *_stf; } + HandleResponseFunction & handleResponse() { return *_hrf; } + ProcessTaskFunction & processTask() { return *_ptf; } + IsFinishedFunction & isFinished() { return *_iff; } + + void sendTask( SendTaskFunction* stf ) { _stf = stf; } + void handleResponse( HandleResponseFunction* hrf ) { _hrf = hrf; } + void processTask( ProcessTaskFunction* ptf ) { _ptf = ptf; } + void isFinished( IsFinishedFunction* iff ) { _iff = iff; } + + void wrapSendTask( SendTaskFunction* stf ) + { + if( stf ) + { + stf->wrapped( _stf ); + _stf = stf; + } + } + + void wrapHandleResponse( HandleResponseFunction* hrf ) + { + if( hrf ) + { + hrf->wrapped( _hrf ); + _hrf = hrf; + } + } + + void wrapProcessTask( ProcessTaskFunction* ptf ) + { + if( ptf ) + { + ptf->wrapped( _ptf ); + _ptf = ptf; + } + } + + void wrapIsFinished( IsFinishedFunction* iff ) + { + if( iff ) + { + iff->wrapped( _iff ); + _iff = iff; + } + } + + // TODO commenter : laissé à la couche d'en dessous car impossible d'initialiser une donnée membre d'une classe mère depuis une classe fille. + virtual JobData* data() = 0; + + protected: + + // TODO commenter : Utiliser des pointeurs pour éviter d'écraser les fonctions wrappées + SendTaskFunction< JobData >* _stf; + HandleResponseFunction< JobData >* _hrf; + ProcessTaskFunction< JobData >* _ptf; + IsFinishedFunction< JobData >* _iff; + }; + + template< class JobData > + class Job + { + public: + Job( AssignmentAlgorithm& _algo, + int _masterRank, + JobStore & store + ) : + assignmentAlgo( _algo ), + masterRank( _masterRank ), + comm( Node::comm() ), + // Functors + sendTask( store.sendTask() ), + handleResponse( store.handleResponse() ), + processTask( store.processTask() ), + isFinished( store.isFinished() ) + { + _isMaster = Node::comm().rank() == _masterRank; + + sendTask.data( store.data() ); + handleResponse.data( store.data() ); + processTask.data( store.data() ); + isFinished.data( store.data() ); + } + + protected: + + struct FinallyBlock + { + FinallyBlock( + int _totalWorkers, + AssignmentAlgorithm& _algo, + Job< JobData > & _that + ) : + totalWorkers( _totalWorkers ), + assignmentAlgo( _algo ), + that( _that ), + // global field + comm( Node::comm() ) + { + // empty + } + + ~FinallyBlock() + { +# ifndef NDEBUG + eo::log << eo::debug; + eo::log << "[M" << comm.rank() << "] Frees all the idle." << std::endl; +# endif + // frees all the idle workers + timerStat.start("master_wait_for_idles"); + std::vector idles = assignmentAlgo.idles(); + for(unsigned int i = 0; i < idles.size(); ++i) + { + comm.send( idles[i], Channel::Commands, Message::Finish ); + } + timerStat.stop("master_wait_for_idles"); + +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Waits for all responses." << std::endl; +# endif + // wait for all responses + timerStat.start("master_wait_for_all_responses"); + while( assignmentAlgo.availableWorkers() != totalWorkers ) + { + bmpi::status status = comm.probe( bmpi::any_source, bmpi::any_tag ); + int wrkRank = status.source(); + that.handleResponse( wrkRank ); + comm.send( wrkRank, Channel::Commands, Message::Finish ); + assignmentAlgo.confirm( wrkRank ); + } + timerStat.stop("master_wait_for_all_responses"); +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Leaving master task." << std::endl; +# endif + } + + protected: + + int totalWorkers; + AssignmentAlgorithm& assignmentAlgo; + Job< JobData > & that; + + bmpi::communicator & comm; + }; + + void master( ) + { + int totalWorkers = assignmentAlgo.availableWorkers(); +# ifndef NDEBUG + eo::log << eo::debug; + eo::log << "[M" << comm.rank() << "] Have " << totalWorkers << " workers." << std::endl; +# endif + try { + FinallyBlock finally( totalWorkers, assignmentAlgo, *this ); + while( ! isFinished() ) + { + timerStat.start("master_wait_for_assignee"); + int assignee = assignmentAlgo.get( ); + while( assignee <= 0 ) + { +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Waitin' for node..." << std::endl; +# endif + bmpi::status status = comm.probe( bmpi::any_source, bmpi::any_tag ); + int wrkRank = status.source(); +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Node " << wrkRank << " just terminated." << std::endl; +# endif + handleResponse( wrkRank ); + assignmentAlgo.confirm( wrkRank ); + assignee = assignmentAlgo.get( ); + } + timerStat.stop("master_wait_for_assignee"); +# ifndef NDEBUG + eo::log << "[M" << comm.rank() << "] Assignee : " << assignee << std::endl; +# endif + + timerStat.start("master_wait_for_send"); + comm.send( assignee, Channel::Commands, Message::Continue ); + sendTask( assignee ); + timerStat.stop("master_wait_for_send"); + } + } catch( const std::exception & e ) + { + std::string s = e.what(); + s.append( " in eoMpi loop"); + throw std::runtime_error( s ); + } + } + + void worker( ) + { + int order; +# ifndef NDEBUG + eo::log << eo::debug; +# endif + timerStat.start("worker_wait_for_order"); + comm.recv( masterRank, Channel::Commands, order ); + timerStat.stop("worker_wait_for_order"); + + while( true ) + { +# ifndef NDEBUG + eo::log << "[W" << comm.rank() << "] Waiting for an order..." << std::endl; +# endif + if ( order == Message::Kill ) + { +# ifndef NDEBUG + eo::log << "[W" << comm.rank() << "] Leaving worker task." << std::endl; +# endif + return; + } else if( order == Message::Continue ) + { +# ifndef NDEBUG + eo::log << "[W" << comm.rank() << "] Processing task..." << std::endl; +# endif + processTask( ); + } + + timerStat.start("worker_wait_for_order"); + comm.recv( masterRank, Channel::Commands, order ); + timerStat.stop("worker_wait_for_order"); + } + } + + public: + + void run( ) + { + ( _isMaster ) ? master( ) : worker( ); + } + + bool isMaster( ) + { + return _isMaster; + } + + protected: + + AssignmentAlgorithm& assignmentAlgo; + int masterRank; + bmpi::communicator& comm; + + SendTaskFunction & sendTask; + HandleResponseFunction & handleResponse; + ProcessTaskFunction & processTask; + IsFinishedFunction & isFinished; + + bool _isMaster; + }; + } +} + +# endif // __EO_MPI_H__ + diff --git a/eo/src/mpi/eoMpiAssignmentAlgorithm.h b/eo/src/mpi/eoMpiAssignmentAlgorithm.h new file mode 100644 index 000000000..c3d24a196 --- /dev/null +++ b/eo/src/mpi/eoMpiAssignmentAlgorithm.h @@ -0,0 +1,219 @@ +# ifndef __MPI_ASSIGNMENT_ALGORITHM_H__ +# define __MPI_ASSIGNMENT_ALGORITHM_H__ + +# include +# include "eoMpiNode.h" + +namespace eo +{ + namespace mpi + { + const int REST_OF_THE_WORLD = -1; + + struct AssignmentAlgorithm + { + virtual int get( ) = 0; + virtual int availableWorkers( ) = 0; + virtual void confirm( int wrkRank ) = 0; + virtual std::vector idles( ) = 0; + virtual void reinit( int runs ) = 0; + }; + + struct DynamicAssignmentAlgorithm : public AssignmentAlgorithm + { + public: + + DynamicAssignmentAlgorithm( ) + { + for(int i = 1; i < Node::comm().size(); ++i) + { + availableWrk.push_back( i ); + } + } + + DynamicAssignmentAlgorithm( int unique ) + { + availableWrk.push_back( unique ); + } + + DynamicAssignmentAlgorithm( const std::vector & workers ) + { + availableWrk = workers; + } + + DynamicAssignmentAlgorithm( int first, int last ) + { + if( last == REST_OF_THE_WORLD ) + { + last = Node::comm().size() - 1; + } + + for( int i = first; i <= last; ++i) + { + availableWrk.push_back( i ); + } + } + + virtual int get( ) + { + int assignee = -1; + if (! availableWrk.empty() ) + { + assignee = availableWrk.back(); + availableWrk.pop_back(); + } + return assignee; + } + + int availableWorkers() + { + return availableWrk.size(); + } + + void confirm( int rank ) + { + availableWrk.push_back( rank ); + } + + std::vector idles( ) + { + return availableWrk; + } + + void reinit( int _ ) + { + ++_; + // nothing to do + } + + protected: + std::vector< int > availableWrk; + }; + + struct StaticAssignmentAlgorithm : public AssignmentAlgorithm + { + public: + StaticAssignmentAlgorithm( std::vector& workers, int runs ) + { + init( workers, runs ); + } + + StaticAssignmentAlgorithm( int first, int last, int runs ) + { + std::vector workers; + + if( last == REST_OF_THE_WORLD ) + { + last = Node::comm().size() - 1; + } + + for(int i = first; i <= last; ++i) + { + workers.push_back( i ); + } + init( workers, runs ); + } + + StaticAssignmentAlgorithm( int runs ) + { + std::vector workers; + for(int i = 1; i < Node::comm().size(); ++i) + { + workers.push_back( i ); + } + init( workers, runs ); + } + + StaticAssignmentAlgorithm( int unique, int runs ) + { + std::vector workers; + workers.push_back( unique ); + init( workers, runs ); + } + + private: + void init( std::vector & workers, int runs ) + { + unsigned int nbWorkers = workers.size(); + freeWorkers = nbWorkers; + attributions.reserve( nbWorkers ); + busy.resize( nbWorkers, false ); + + // Let be the euclidean division of runs by nbWorkers : + // runs == q * nbWorkers + r, 0 <= r < nbWorkers + // This one liner affects q requests to each worker + for (unsigned int i = 0; i < nbWorkers; attributions[i++] = runs / nbWorkers) ; + // The first line computes r and the one liner affects the remaining + // r requests to workers, in ascending order + unsigned int diff = runs - (runs / nbWorkers) * nbWorkers; + for (unsigned int i = 0; i < diff; ++attributions[i++]); + + realRank = workers; + } + + public: + int get( ) + { + int assignee = -1; + for( unsigned i = 0; i < busy.size(); ++i ) + { + if( !busy[i] && attributions[i] > 0 ) + { + busy[i] = true; + --freeWorkers; + assignee = realRank[ i ]; + break; + } + } + return assignee; + } + + int availableWorkers( ) + { + return freeWorkers; + } + + std::vector idles() + { + std::vector ret; + for(unsigned int i = 0; i < busy.size(); ++i) + { + if( !busy[i] ) + { + ret.push_back( realRank[i] ); + } + } + return ret; + } + + void confirm( int rank ) + { + int i = -1; // i is the real index in table + for( unsigned int j = 0; j < realRank.size(); ++j ) + { + if( realRank[j] == rank ) + { + i = j; + break; + } + } + + --attributions[ i ]; + busy[ i ] = false; + ++freeWorkers; + } + + void reinit( int runs ) + { + init( realRank, runs ); + } + + private: + std::vector attributions; + std::vector realRank; + std::vector busy; + unsigned int freeWorkers; + }; + } +} +# endif // __MPI_ASSIGNMENT_ALGORITHM_H__ diff --git a/eo/src/mpi/eoMpiNode.h b/eo/src/mpi/eoMpiNode.h new file mode 100644 index 000000000..9f1ea7b53 --- /dev/null +++ b/eo/src/mpi/eoMpiNode.h @@ -0,0 +1,31 @@ +# ifndef __MPI_NODE_H__ +# define __MPI_NODE_H__ + +# include +namespace bmpi = boost::mpi; + +namespace eo +{ + namespace mpi + { + class Node + { + public: + + static void init( int argc, char** argv ) + { + static bmpi::environment env( argc, argv ); + } + + static bmpi::communicator& comm() + { + return _comm; + } + + protected: + static bmpi::communicator _comm; + }; + } +} +# endif // __MPI_NODE_H__ + diff --git a/eo/src/mpi/eoMultiParallelApply.h b/eo/src/mpi/eoMultiParallelApply.h new file mode 100644 index 000000000..85b1689e6 --- /dev/null +++ b/eo/src/mpi/eoMultiParallelApply.h @@ -0,0 +1,59 @@ +# ifndef __EO_MULTI_PARALLEL_APPLY_H__ +# define __EO_MULTI_PARALLEL_APPLY_H__ + +# include "eoParallelApply.h" + +namespace eo +{ + namespace mpi + { + template< class EOT > + class ProcessTaskParallelEval : public ProcessTaskParallelApply + { + public: + + using ProcessTaskParallelApply::_wrapped; + using ProcessTaskParallelApply::d; + + void operator()() + { + int order = Message::Continue; + while( order != Message::Finish ) + { + _wrapped->operator()(); + d->comm.recv( d->masterRank, Channel::Commands, order ); + } + } + + ~ProcessTaskParallelEval() + { + delete _wrapped; + } + }; + + template< class EOT > + struct ParallelEvalStore : public ParallelApplyStore< EOT > + { + using ParallelApplyStore::wrapProcessTask; + + ParallelEvalStore( + eoUF & _proc, + int _masterRank, + // long _maxTime = 0, + int _packetSize = 1 + ) : + ParallelApplyStore< EOT >( _proc, *( new std::vector ), _masterRank, _packetSize ) + // FIXME memory leak because of vector ==> use const correctness + { + wrapProcessTask( new ProcessTaskParallelEval ); + } + + void data( std::vector& _pop ) + { + ParallelApplyStore::_data.init( _pop ); + } + }; + } +} +# endif // __EO_PARALLEL_APPLY_H__ + diff --git a/eo/src/mpi/eoParallelApply.h b/eo/src/mpi/eoParallelApply.h new file mode 100644 index 000000000..a02254716 --- /dev/null +++ b/eo/src/mpi/eoParallelApply.h @@ -0,0 +1,228 @@ +# ifndef __EO_PARALLEL_APPLY_H__ +# define __EO_PARALLEL_APPLY_H__ + +# include "eoMpi.h" + +# include +# include + +namespace eo +{ + namespace mpi + { + struct ParallelApplyAssignment + { + int index; + int size; + }; + + template + struct ParallelApplyData + { + ParallelApplyData( + eoUF & _proc, + std::vector & _pop, + int _masterRank, + // long _maxTime = 0, + int _packetSize + ) : + _data( &_pop ), func( _proc ), index( 0 ), size( _pop.size() ), packetSize( _packetSize ), masterRank( _masterRank ), comm( Node::comm() ) + { + if ( _packetSize <= 0 ) + { + throw std::runtime_error("Packet size should not be negative."); + } + tempArray = new EOT[ _packetSize ]; + } + + void init( std::vector& _pop ) + { + index = 0; + size = _pop.size(); + _data = &_pop; + assignedTasks.clear(); + } + + ~ParallelApplyData() + { + delete [] tempArray; + } + + std::vector& data() + { + return *_data; + } + + std::vector * _data; + eoUF & func; + int index; + int size; + std::map< int /* worker rank */, ParallelApplyAssignment /* min indexes in vector */> assignedTasks; + int packetSize; + EOT* tempArray; + + int masterRank; + bmpi::communicator& comm; + }; + + template< class EOT > + class SendTaskParallelApply : public SendTaskFunction< ParallelApplyData > + { + public: + using SendTaskFunction< ParallelApplyData >::d; + + SendTaskParallelApply( SendTaskParallelApply * w = 0 ) : SendTaskFunction< ParallelApplyData >( w ) + { + // empty + } + + void operator()(int wrkRank) + { + int futureIndex; + + if( d->index + d->packetSize < d->size ) + { + futureIndex = d->index + d->packetSize; + } else { + futureIndex = d->size; + } + + int sentSize = futureIndex - d->index ; + + d->comm.send( wrkRank, 1, sentSize ); + + eo::log << eo::progress << "Evaluating individual " << d->index << std::endl; + + d->assignedTasks[ wrkRank ].index = d->index; + d->assignedTasks[ wrkRank ].size = sentSize; + + d->comm.send( wrkRank, 1, & ( (d->data())[ d->index ] ) , sentSize ); + d->index = futureIndex; + } + }; + + template< class EOT > + class HandleResponseParallelApply : public HandleResponseFunction< ParallelApplyData > + { + public: + using HandleResponseFunction< ParallelApplyData >::d; + + HandleResponseParallelApply( HandleResponseParallelApply * w = 0 ) : HandleResponseFunction< ParallelApplyData >( w ) + { + // empty + } + + void operator()(int wrkRank) + { + d->comm.recv( wrkRank, 1, & (d->data()[ d->assignedTasks[wrkRank].index ] ), d->assignedTasks[wrkRank].size ); + } + }; + + template< class EOT > + class ProcessTaskParallelApply : public ProcessTaskFunction< ParallelApplyData > + { + public: + using ProcessTaskFunction< ParallelApplyData >::d; + + ProcessTaskParallelApply( ProcessTaskParallelApply * w = 0 ) : ProcessTaskFunction< ParallelApplyData >( w ) + { + // empty + } + + void operator()() + { + int recvSize; + + d->comm.recv( d->masterRank, 1, recvSize ); + d->comm.recv( d->masterRank, 1, d->tempArray, recvSize ); + timerStat.start("worker_processes"); + for( int i = 0; i < recvSize ; ++i ) + { + d->func( d->tempArray[ i ] ); + } + timerStat.stop("worker_processes"); + d->comm.send( d->masterRank, 1, d->tempArray, recvSize ); + } + }; + + template< class EOT > + class IsFinishedParallelApply : public IsFinishedFunction< ParallelApplyData > + { + public: + using IsFinishedFunction< ParallelApplyData >::d; + + IsFinishedParallelApply( IsFinishedParallelApply * w = 0 ) : IsFinishedFunction< ParallelApplyData >( w ) + { + // empty + } + + bool operator()() + { + return d->index == d->size; + } + }; + + template< class EOT > + struct ParallelApplyStore : public JobStore< ParallelApplyData > + { + using JobStore< ParallelApplyData >::_stf; + using JobStore< ParallelApplyData >::_hrf; + using JobStore< ParallelApplyData >::_ptf; + using JobStore< ParallelApplyData >::_iff; + + ParallelApplyStore( + eoUF & _proc, + std::vector& _pop, + int _masterRank, + // long _maxTime = 0, + int _packetSize = 1, + // JobStore functors + SendTaskParallelApply * stpa = new SendTaskParallelApply, + HandleResponseParallelApply* hrpa = new HandleResponseParallelApply, + ProcessTaskParallelApply* ptpa = new ProcessTaskParallelApply, + IsFinishedParallelApply* ifpa = new IsFinishedParallelApply + ) : + _data( _proc, _pop, _masterRank, _packetSize ) + { + _stf = stpa; + _hrf = hrpa; + _ptf = ptpa; + _iff = ifpa; + } + + ParallelApplyData* data() { return &_data; } + + virtual ~ParallelApplyStore() + { + delete _stf; + delete _hrf; + delete _ptf; + delete _iff; + } + + protected: + ParallelApplyData _data; + }; + + // TODO commentaire : impossible de faire un typedef sur un template sans passer + // par un traits => complique la tâche de l'utilisateur pour rien. + template< typename EOT > + class ParallelApply : public Job< ParallelApplyData > + { + public: + + ParallelApply( + AssignmentAlgorithm & algo, + int _masterRank, + ParallelApplyStore & store + ) : + Job< ParallelApplyData >( algo, _masterRank, store ) + { + // empty + } + }; + } +} +# endif // __EO_PARALLEL_APPLY_H__ + + diff --git a/eo/src/mpi/eoTerminateJob.h b/eo/src/mpi/eoTerminateJob.h new file mode 100644 index 000000000..d85aeeb05 --- /dev/null +++ b/eo/src/mpi/eoTerminateJob.h @@ -0,0 +1,121 @@ +# ifndef __EO_TERMINATE_H__ +# define __EO_TERMINATE_H__ + +# include "eoMpi.h" + +namespace eo +{ + namespace mpi + { + struct DummySendTaskFunction : public SendTaskFunction + { + void operator()( int _ ) + { + ++_; + } + }; + + struct DummyHandleResponseFunction : public HandleResponseFunction + { + void operator()( int _ ) + { + ++_; + } + }; + + struct DummyProcessTaskFunction : public ProcessTaskFunction + { + void operator()() + { + // nothing! + } + }; + + struct DummyIsFinishedFunction : public IsFinishedFunction + { + bool operator()() + { + return true; + } + }; + + struct DummyJobStore : public JobStore + { + using JobStore::_stf; + using JobStore::_hrf; + using JobStore::_ptf; + using JobStore::_iff; + + DummyJobStore() + { + _stf = new DummySendTaskFunction; + _hrf = new DummyHandleResponseFunction; + _ptf = new DummyProcessTaskFunction; + _iff = new DummyIsFinishedFunction; + } + + ~DummyJobStore() + { + delete _stf; + delete _hrf; + delete _ptf; + delete _iff; + } + + void* data() { return 0; } + }; + + struct EmptyJob : public Job + { + EmptyJob( AssignmentAlgorithm& algo, int masterRank ) : + Job( algo, masterRank, *(new DummyJobStore) ) + // FIXME memory leak => will be corrected by using const correctness + { + // empty + } + + ~EmptyJob() + { + std::vector< int > idles = assignmentAlgo.idles(); + for(unsigned i = 0, size = idles.size(); i < size; ++i) + { + comm.send( idles[i], Channel::Commands, Message::Kill ); + } + } + }; + + /* + class TerminateJob : public Job + { + public: + TerminateJob( AssignmentAlgorithm& algo, int _ ) + : Job( algo, _ ) + { + // empty + } + + void sendTask( int wrkRank ) + { + // empty + } + + void handleResponse( int wrkRank ) + { + // empty + } + + void processTask( ) + { + // empty + } + + bool isFinished() + { + return true; + } + }; + */ + } +} + +# endif // __EO_TERMINATE_H__ diff --git a/eo/src/serial/Array.cpp b/eo/src/serial/Array.cpp new file mode 100644 index 000000000..f70b00e83 --- /dev/null +++ b/eo/src/serial/Array.cpp @@ -0,0 +1,38 @@ +# include "Array.h" + +namespace eoserial +{ + +std::ostream& Array::print( std::ostream& out ) const +{ + out << "["; + bool first = true; + for (ArrayChildren::const_iterator it = begin(), + end = this->end(); + it != end; + ++it) + { + if ( first ) + { + first = false; + } else { + out << ", "; + } + (*it)->print( out ); + } + out << "]\n"; + return out; +} + +Array::~Array() +{ + for (ArrayChildren::iterator it = begin(), + end = this->end(); + it != end; + ++it) + { + delete *it; + } +} + +} // namespace eoserial diff --git a/eo/src/serial/Array.h b/eo/src/serial/Array.h new file mode 100644 index 000000000..69231980c --- /dev/null +++ b/eo/src/serial/Array.h @@ -0,0 +1,149 @@ +# ifndef __EOSERIAL_ARRAY_H__ +# define __EOSERIAL_ARRAY_H__ + +# include +# include + +# include "Entity.h" +# include "Serializable.h" + +# include "Object.h" +# include "String.h" + +namespace eoserial +{ + +// Forward declaration for below declarations. +class Array; + +/* + * Declarations of functions present in Utils.h + * These are put here to avoid instead of including the file Utils.h, which would + * cause a circular inclusion. + */ +template< class T > +void unpack( const Array & array, unsigned int index, T & value ); + +void unpackObject( const Array & array, unsigned int index, Persistent & value ); + +template< class Container, template class UnpackAlgorithm > +void unpackArray( const Array & array, unsigned int index, Container & container ); + +/** + * @brief Represents a JSON array. + * + * Wrapper for an array, so as to be used as a JSON object. + */ +class Array : public eoserial::Entity, public std::vector< eoserial::Entity* > +{ +protected: + typedef std::vector< eoserial::Entity* > ArrayChildren; + +public: + /** + * @brief Adds the serializable object as a JSON object. + * @param obj Object which implemnets JsonSerializable. + */ + void push_back( const eoserial::Printable* obj ) + { + ArrayChildren::push_back( obj->pack() ); + } + + /** + * @brief Proxy for vector::push_back. + */ + void push_back( eoserial::Entity* json ) + { + ArrayChildren::push_back( json ); + } + + /** + * @brief Prints the JSON array into the given stream. + * @param out The stream + */ + virtual std::ostream& print( std::ostream& out ) const; + + /** + * @brief Dtor + */ + ~Array(); + + /* + * The following parts allows the user to automatically deserialize an eoserial::Array into a + * standard container, by giving the algorithm which will be used to deserialize contained entities. + */ + + /** + * @brief Functor which determines how to retrieve the real value contained in a eoserial::Entity at + * a given place. + * + * It will be applied for each contained variable in the array. + */ + template + struct BaseAlgorithm + { + /** + * @brief Main operator. + * + * @param array The eoserial::Array from which we're reading. + * @param i The index of the contained value. + * @param container The standard (STL) container in which we'll push back the read value. + */ + virtual void operator()( const eoserial::Array& array, unsigned int i, Container & container ) const = 0; + }; + + /** + * @brief BaseAlgorithm for retrieving primitive variables. + * + * This one should be used to retrieve primitive (and types which implement operator>>) variables, for instance + * int, double, std::string, etc... + */ + template + struct UnpackAlgorithm : public BaseAlgorithm + { + void operator()( const eoserial::Array& array, unsigned int i, C & container ) const + { + typename C::value_type t; + unpack( array, i, t ); + container.push_back( t ); + } + }; + + /** + * @brief BaseAlgorithm for retrieving eoserial::Persistent objects. + * + * This one should be used to retrieve objects which implement eoserial::Persistent. + */ + template + struct UnpackObjectAlgorithm : public BaseAlgorithm + { + void operator()( const eoserial::Array& array, unsigned int i, C & container ) const + { + typename C::value_type t; + unpackObject( array, i, t ); + container.push_back( t ); + } + }; + + /** + * @brief General algorithm for array deserialization. + * + * Applies the BaseAlgorithm to each contained variable in the eoserial::Array. + */ + template class UnpackAlgorithm> + inline void deserialize( Container & array ) + { + UnpackAlgorithm< Container > algo; + for( unsigned int i = 0, size = this->size(); + i < size; + ++i) + { + algo( *this, i, array ); + } + } +}; + +} // namespace eoserial + +# endif // __EOSERIAL_ARRAY_H__ + diff --git a/eo/src/serial/CMakeLists.txt b/eo/src/serial/CMakeLists.txt new file mode 100644 index 000000000..5358b30a4 --- /dev/null +++ b/eo/src/serial/CMakeLists.txt @@ -0,0 +1,35 @@ +###################################################################################### +### 1) Include the sources +###################################################################################### + +INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/src) +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) + +###################################################################################### +### 2) Define the eoserial target +###################################################################################### + +SET(EOSERIAL_LIB_OUTPUT_PATH ${EO_BINARY_DIR}/lib) +SET(LIBRARY_OUTPUT_PATH ${EOSERIAL_LIB_OUTPUT_PATH}) + +SET(EOSERIAL_SOURCES + Array.cpp + Object.cpp + Parser.cpp + String.cpp + ) + +ADD_LIBRARY(eoserial STATIC ${EOSERIAL_SOURCES}) +INSTALL(TARGETS eoserial ARCHIVE DESTINATION lib COMPONENT libraries) + +FILE(GLOB HDRS *.h) +INSTALL(FILES ${HDRS} DESTINATION include/eo/serial COMPONENT headers) + +###################################################################################### +### 3) Optionnal +###################################################################################### + +SET(EOSERIAL_VERSION ${GLOBAL_VERSION}) +SET_TARGET_PROPERTIES(eoserial PROPERTIES VERSION "${EOSERIAL_VERSION}") + +###################################################################################### diff --git a/eo/src/serial/Entity.h b/eo/src/serial/Entity.h new file mode 100644 index 000000000..df10002df --- /dev/null +++ b/eo/src/serial/Entity.h @@ -0,0 +1,34 @@ +# ifndef __EOSERIAL_ENTITY_H__ +# define __EOSERIAL_ENTITY_H__ + +# include +# include + +namespace eoserial +{ + +/** + * @brief JSON entity + * + * This class represents a JSON entity, which can be JSON objects, + * strings or arrays. It is the base class for the JSON hierarchy. + */ +class Entity +{ +public: + + /** + * Virtual dtor (base class). + */ + virtual ~Entity() { /* empty */ } + + /** + * @brief Prints the content of a JSON object into a stream. + * @param out The stream in which we're printing. + */ + virtual std::ostream& print( std::ostream& out ) const = 0; +}; + +} // namespace eoserial + +# endif // __ENTITY_H__ diff --git a/eo/src/serial/Object.cpp b/eo/src/serial/Object.cpp new file mode 100644 index 000000000..3e557a126 --- /dev/null +++ b/eo/src/serial/Object.cpp @@ -0,0 +1,40 @@ +# include "Object.h" + +using namespace eoserial; + +namespace eoserial +{ + +std::ostream& Object::print( std::ostream& out ) const +{ + out << '{'; + bool first = true; + for(JsonValues::const_iterator it = begin(), end = this->end(); + it != end; + ++it) + { + if ( first ) + { + first = false; + } else { + out << ", "; + } + + out << '"' << it->first << "\":"; // key + it->second->print( out ); // value + } + out << "}\n"; + return out; +} + +Object::~Object() +{ + for(JsonValues::iterator it = begin(), end = this->end(); + it != end; + ++it) + { + delete it->second; + } +} + +} // namespace eoserial diff --git a/eo/src/serial/Object.h b/eo/src/serial/Object.h new file mode 100644 index 000000000..367692527 --- /dev/null +++ b/eo/src/serial/Object.h @@ -0,0 +1,67 @@ +# ifndef __EOSERIAL_OBJECT_H__ +# define __EOSERIAL_OBJECT_H__ + +# include +# include +# include + +# include "Entity.h" +# include "Serializable.h" + +namespace eoserial +{ + +/** + * @brief JSON Object + * + * This class represents a JSON object, which is basically a dictionnary + * of keys (strings) and values (JSON entities). + */ +class Object : public eoserial::Entity, public std::map< std::string, eoserial::Entity* > +{ +public: + typedef std::map JsonValues; + + /** + * @brief Adds a pair into the JSON object. + * @param key The key associated with the eoserial object + * @param eoserial The JSON object as created with framework. + */ + void add( const std::string& key, eoserial::Entity* json ) + { + (*this)[ key ] = json; + } + + /** + * @brief Adds a pair into the JSON object. + * @param key The key associated with the eoserial object + * @param obj A JSON-serializable object + */ + void add( const std::string& key, const eoserial::Printable* obj ) + { + (*this)[ key ] = obj->pack(); + } + + /** + * @brief Deserializes a Serializable class instance from this JSON object. + * @param obj The object we want to rebuild. + */ + void deserialize( eoserial::Persistent & obj ) + { + obj.unpack( this ); + } + + /** + * @brief Dtor + */ + ~Object(); + + /** + * @brief Prints the content of a JSON object into a stream. + */ + virtual std::ostream& print( std::ostream& out ) const; +}; + +} // namespace eoserial +# endif // __EOSERIAL_OBJECT_H__ + diff --git a/eo/src/serial/Parser.cpp b/eo/src/serial/Parser.cpp new file mode 100644 index 000000000..c7822d292 --- /dev/null +++ b/eo/src/serial/Parser.cpp @@ -0,0 +1,153 @@ +# include +# include +# include +# include + +# include "Parser.h" + +# include "Array.h" +# include "Object.h" +# include "String.h" + +// in debug mode only +// # define DEBUG(x) std::cout << x << std::endl; +# define DEBUG(x) + +using namespace eoserial; + +namespace eoserial +{ + +/** + * @brief Parses a string contained between double quotes. + * + * Strings can contain escaped double quotes. + * @param str The string we're parsing. + * @param pos The index of current position in parsed string. + * This index will be updated so as to allow the parser to + * continue. + */ +static std::string parseString(const std::string& str, size_t & pos) +{ + // example : "hello" + // example 2 : "\"world\"" + // for hello: + // firstQuote == 0, secondQuote == 6 + // sub string should be from firstQuote+1 to secondQuote-1 + // so its size should be (secondQuote-1 -(firstQuote+1) + 1) + std::string value; + size_t firstQuote = str.find( '"', pos ); + size_t secondQuote; + + /* instead of just seeking the second quote, we need to ensure + // that there is no escaped quote before this one. + // actually this is harder than that. Using backslashes + // to escape double quotes mean that backslashes have to be + // escaped to. + // example : "text\\" to symbolize : text\ + // example : "text\\\" to symbolize : text\" + // In fact, we should find if number of backslashes is odd; in this case, + // the double quotes are escaped and we should find the next one. + */ + int backslashesCount; + do { + ++pos; + secondQuote = str.find( '"', pos ); + size_t i = secondQuote - 1; + + // Find the backslashes + backslashesCount = 0; + while ( str[ i ] == '\\' ) + { + --i; + ++backslashesCount; + } + pos = secondQuote; + } while( backslashesCount % 2 == 1 ); + + value = str.substr( firstQuote+1, secondQuote-firstQuote-1 ); + pos = secondQuote + 1; + return value; +} + +/** + * @brief Moves the given index pos to the next character which is + * neither a coma, a space nor a new line. + * + * @param str The string in which we want to ignores those characters. + * @param pos The index of current position in parsed string. + */ +static void ignoreChars(const std::string& str, size_t & pos) +{ + // ignore white spaces and comas + for (char current = str[ pos ]; + current == ',' || current == ' ' || current == '\n'; + current = str[ ++pos ]); +} + +String* Parser::parseJsonString(const std::string & str, size_t & pos) +{ + return new String( parseString( str, pos ) ); +} + +Object* Parser::parse(const std::string & str) +{ + size_t initial(0); // we begin at position 0 + return static_cast( parseRight(str, initial) ); +} + +Entity* Parser::parseRight(const std::string & str, size_t & pos) +{ + Entity* value = 0; + + if ( str[ pos ] == '{' ) + { + // next one is an object + DEBUG("We read an object.") + Object* obj = new Object; + pos += 1; + while( pos < str.size() && str[ pos ] != '}' ) + { + parseLeft( str, pos, obj ); + ignoreChars( str, pos ); + } + DEBUG("We just finished to read an object ! ") + pos += 1; // we're on the }, go to the next char + value = obj; + } + else if ( str[ pos ] == '"' ) + { + // next one is a string + DEBUG("We read a string") + value = parseJsonString( str, pos ); + } + else if ( str[ pos ] == '[' ) + { + // next one is an array + DEBUG("We read an array") + Array* array = new Array; + pos += 1; + while( pos < str.size() && str[ pos ] != ']' ) + { + Entity* child = parseRight( str, pos ); + if ( child ) + array->push_back( child ); + } + DEBUG("We've finished to read our array.") + pos += 1; // we're on the ], go to the next char + value = array; + } + ignoreChars( str, pos ); + return value; +} + +void Parser::parseLeft(const std::string & str, size_t & pos, Object* eoserial) +{ + std::string key = parseString(str, pos); + ++pos; // the colon + DEBUG("We've read the key ") + (*eoserial)[ key ] = parseRight( str, pos ); +} + +} // namespace eoserial + diff --git a/eo/src/serial/Parser.h b/eo/src/serial/Parser.h new file mode 100644 index 000000000..f0a94ee21 --- /dev/null +++ b/eo/src/serial/Parser.h @@ -0,0 +1,78 @@ +# ifndef __EOSERIAL_PARSER_H__ +# define __EOSERIAL_PARSER_H__ + +# include "Entity.h" +# include "String.h" +# include "Object.h" + +/** + * This file contains a tiny JSON parser used in DAE. This parser just handles + * a subset of JSON grammar, with the following restrictions : + * - all strings must be surrounded by double quotes. + * - everything which is not an object or an array is considered to be a string + * (even integers, booleans,...). + * - no syntax check is done. We trust the programmer and he has to ensure that + * every JSON string he produces is valid. + * + * @author Benjamin BOUVIER + */ + +namespace eoserial +{ + +/** + * @brief Parser from a JSON source. + * + * This parser does just retrieve values and does NOT check the structure of + * the input. This implies that if the input is not correct, the result is undefined + * and can result to a failure on execution. + */ +class Parser +{ + public: + + /** + * @brief Parses the given string and returns the JSON object read. + */ + static eoserial::Object* parse(const std::string & str); + + protected: + + /** + * @brief Parses the right part of a JSON object as a string. + * + * The right part of an object can be a string (for instance : + * "key":"value"), a JSON array (for instance: "key":["1"]) or + * another JSON object (for instance: "key":{"another_key":"value"}). + * + * The right parts are found after keys (which are parsed by parseLeft) + * and in arrays. + * + * @param str The string we're parsing. + * @param pos The index of the current position in the string. + * @return The JSON object matching the right part. + */ + static eoserial::Entity* parseRight(const std::string & str, size_t & pos); + + /** + * @brief Parses the left value of a key-value pair, which is the key. + * + * @param str The string we're parsing. + * @param pos The index of the current position in the string. + * @param eoserial The current JSON object for which we're adding a key-value pair. + */ + static void parseLeft(const std::string & str, size_t & pos, eoserial::Object* json); + + /** + * @brief Retrieves a string in a JSON content. + * + * @param str The string we're parsing. + * @param pos The index of the current position of parsing, + * which will be updated. + */ + static eoserial::String* parseJsonString(const std::string & str, size_t & pos); +}; + +} // namespace eoserial + +# endif // __EOSERIAL_PARSER_H__ diff --git a/eo/src/serial/Serializable.h b/eo/src/serial/Serializable.h new file mode 100644 index 000000000..482a918a8 --- /dev/null +++ b/eo/src/serial/Serializable.h @@ -0,0 +1,44 @@ +# ifndef __EOSERIAL_SERIALIZABLE_H__ +# define __EOSERIAL_SERIALIZABLE_H__ + +# include + +namespace eoserial +{ + +class Object; // to avoid recursive inclusion with JsonObject + +/** + * @brief Interface showing that object can be written to a eoserial type + * (currently JSON). + */ +class Printable +{ +public: + /** + * @brief Serializes the object to JSON format. + * @return A JSON object created with new. + */ + virtual eoserial::Object* pack() const = 0; +}; + +/** + * @brief Interface showing that object can be eoserialized (written and read + * from an input). + * + * Note : Persistent objects should have a default non-arguments constructor. + */ +class Persistent : public Printable +{ + public: + /** + * @brief Loads class fields from a JSON object. + * @param json A JSON object. Programmer doesn't have to delete it, it + * is automatically done. + */ + virtual void unpack(const eoserial::Object* json) = 0; +}; + +} // namespace eoserial + +# endif // __EOSERIAL_SERIALIZABLE_H__ diff --git a/eo/src/serial/String.cpp b/eo/src/serial/String.cpp new file mode 100644 index 000000000..deba05a06 --- /dev/null +++ b/eo/src/serial/String.cpp @@ -0,0 +1,11 @@ +# include "String.h" + +namespace eoserial +{ + std::ostream& String::print( std::ostream& out ) const + { + out << '"' << *this << '"'; + return out; + } +} // namespace eoserial + diff --git a/eo/src/serial/String.h b/eo/src/serial/String.h new file mode 100644 index 000000000..6d81937f3 --- /dev/null +++ b/eo/src/serial/String.h @@ -0,0 +1,81 @@ +# ifndef __EOSERIAL_STRING_H__ +# define __EOSERIAL_STRING_H__ + +# include +# include +# include + +# include "Entity.h" + +namespace eoserial +{ + +/** + * @brief JSON String + * + * Wrapper for string, so as to be used as a JSON object. + */ +class String : public eoserial::Entity, public std::string +{ + public: + + /** + * @brief Default ctor. + * @param str The string we want to wrap. + */ + String( const std::string& str ) : std::string( str ) {} + + /** + * @brief Ctor used only on parsing. + */ + String( ) {} + + /** + * @brief Prints out the string. + */ + virtual std::ostream& print( std::ostream& out ) const; + + /** + * @brief Deserializes the current String into a given primitive type value. + * @param value The value in which we're writing. + */ + template + inline void deserialize( T & value ); + + protected: + // Copy and reaffectation are forbidden + explicit String( const String& _ ); + String& operator=( const String& _ ); +}; + +/** + * @brief Casts a eoserial::String into a primitive value, or in a type which at + * least overload operator>>. + * + * @param value A reference to the variable we're writing into. + * + * It's not necessary to specify the variable type, which can be infered by compiler when + * invoking. + */ +template +inline void String::deserialize( T & value ) +{ + std::stringstream ss; + ss.precision(std::numeric_limits::digits10 + 1); + ss << *this; + ss >> value; +} + +/** + * @brief Specialization for strings, which don't need to be converted through + * a stringstream. + */ +template<> +inline void String::deserialize( std::string & value ) +{ + value = *this; +} + +} // namespace eoserial + +# endif // __EOSERIAL_STRING_H__ diff --git a/eo/src/serial/Utils.h b/eo/src/serial/Utils.h new file mode 100644 index 000000000..f40c08d92 --- /dev/null +++ b/eo/src/serial/Utils.h @@ -0,0 +1,168 @@ +# ifndef __EOSERIAL_UTILS_H__ +# define __EOSERIAL_UTILS_H__ + +# include "Array.h" +# include "Object.h" +# include "String.h" + +namespace eoserial +{ + /***************************** + * DESERIALIZATION FUNCTIONS * + ***************************** + These functions are useful for casting eoserial::objects into simple, primitive + variables or into class instance which implement eoserial::Persistent. + + The model is always quite the same : + - the first argument is the containing object (which is a eoserial::Entity, + an object or an array) + - the second argument is the key or index, + - the last argument is the value in which we're writing. + */ + + template< class T > + inline void unpack( const Object & obj, const std::string & key, T & value ) + { + static_cast( obj.find( key )->second )->deserialize( value ); + } + + inline void unpackObject( const Object & obj, const std::string & key, Persistent & value ) + { + static_cast( obj.find( key )->second )->deserialize( value ); + } + + template< class Container, template class UnpackAlgorithm > + inline void unpackArray( const Object & obj, const std::string & key, Container & array ) + { + static_cast( obj.find( key )->second )->deserialize< Container, UnpackAlgorithm >( array ); + } + + template< class T > + inline void unpack( const Array & array, unsigned int index, T & value ) + { + static_cast( array[ index ] )->deserialize( value ); + } + + inline void unpackObject( const Array & array, unsigned int index, Persistent & value ) + { + static_cast( array[ index ] )->deserialize( value ); + } + + template< class Container, template class UnpackAlgorithm > + inline void unpackArray( const Array & array, unsigned int index, Container & container ) + { + static_cast( array[ index ] )->deserialize< Container, UnpackAlgorithm >( container ); + } + + /******************************* + *** SERIALIZATION FUNCTIONS *** + ******************************* + These functions are useful for casting classic objects and + eoserial::Persistent objects into eoserial entities which + can be manipulated by the framework. + */ + + /** + * @brief Casts a value of a stream-serializable type (i.e, which implements + * operator <<) into a JsonString. + * + * This is used when serializing the objects : all primitives types should be + * converted into strings to get more easily manipulated. + * + * @param value The value we're converting. + * @return JsonString wrapper for the value. + */ + template + String* make( const T & value ) + { + std::stringstream ss; + ss.precision(std::numeric_limits::digits10 + 1); + ss << value; + return new String( ss.str() ); + } + + /** + * @brief Specialization for strings : no need to convert as they're still + * usable as strings. + */ + template<> + inline String* make( const std::string & value ) + { + return new String( value ); + } + + /* + * These functions are useful for automatically serializing STL containers into + * eoserial arrays which could be used by the framework. + **/ + + /** + * @brief Functor which explains how to push the value into the eoserial::Array. + */ + template< class T > + struct PushAlgorithm + { + /** + * @brief Main operator. + * + * @param array The eoserial::array in which we're writing. + * @param value The variable we are writing. + */ + virtual void operator()( Array & array, const T & value ) = 0; + }; + + /** + * @brief Push algorithm for primitive variables. + * + * This one should be used when inserting primitive (and types which implement + * operator<<) variables. + */ + template< class T > + struct MakeAlgorithm : public PushAlgorithm + { + void operator()( Array & array, const T & value ) + { + array.push_back( make( value ) ); + } + }; + + /** + * @brief Push algorithm for eoserial::Persistent variables. + */ + template< class T > + struct SerializablePushAlgorithm : public PushAlgorithm + { + void operator()( Array & array, const T & obj ) + { + // obj address is not saved into array.push_back. + array.push_back( &obj ); + } + }; + + /** + * @brief Casts a STL container (vector or list, for instance) + * into a eoserial::Array. + * + * @þaram PushAlgorithm The algorithm used for inserting new element in the eoserial::Array. + * This algorithm is directly called, so it is its own charge to invoke push_back on the + * eoserial::Array. + */ + template< class Container, template class PushAlgorithm > + Array* makeArray( const Container & array ) + { + Array* returned_array = new Array; + typedef typename Container::const_iterator iterator; + typedef typename Container::value_type Type; + PushAlgorithm< Type > algo; + for ( + iterator it = array.begin(), end = array.end(); + it != end; + ++it) + { + algo( *returned_array, *it ); + } + return returned_array; + } +} // namespace eoserial + +# endif //__EOSERIAL_UTILS_H__ diff --git a/eo/src/serial/eoSerial.h b/eo/src/serial/eoSerial.h new file mode 100644 index 000000000..a10f6c014 --- /dev/null +++ b/eo/src/serial/eoSerial.h @@ -0,0 +1,12 @@ +# ifndef __EOSERIAL_HEADERS__ +# define __EOSERIAL_HEADERS__ + +# include "Object.h" +# include "Serializable.h" +# include "Array.h" +# include "Object.h" +# include "String.h" +# include "Parser.h" +# include "Utils.h" + +# endif // __EOSERIAL_HEADERS__ diff --git a/eo/src/serial/json_example b/eo/src/serial/json_example new file mode 100644 index 000000000..7ecb3edda --- /dev/null +++ b/eo/src/serial/json_example @@ -0,0 +1,8 @@ +{"a":"b", +"obj": + {"obj_a":"obj_}b","subobj_a": + {"subk":"subv"} + }, +"c":"d", +"array":["1","2",{"\"array\"_obj\"":"array_ov]"}, ["3"], "4"] +} diff --git a/eo/src/utils/eoTimer.h b/eo/src/utils/eoTimer.h new file mode 100644 index 000000000..c97ab5d35 --- /dev/null +++ b/eo/src/utils/eoTimer.h @@ -0,0 +1,145 @@ +# ifndef __TIMER_H__ +# define __TIMER_H__ + +# include +# include + +# include +# include + +# ifdef WITH_MPI +# include +# include +# include +# endif + +// TODO TODOB commenter +class eoTimer +{ + public: + + eoTimer() + { + restart(); + } + + void restart() + { + uuremainder = 0; + usremainder = 0; + wc_start = time(NULL); + getrusage( RUSAGE_SELF, &_start ); + } + + long int usertime() + { + struct rusage _now; + getrusage( RUSAGE_SELF, &_now ); + long int result = _now.ru_utime.tv_sec - _start.ru_utime.tv_sec; + if( _now.ru_utime.tv_sec == _start.ru_utime.tv_sec ) + { + uuremainder += _now.ru_utime.tv_usec - _start.ru_utime.tv_usec; + if( uuremainder > 1000000) + { + ++result; + uuremainder = 0; + } + } + return result; + } + + long int systime() + { + struct rusage _now; + getrusage( RUSAGE_SELF, &_now ); + long int result = _now.ru_stime.tv_sec - _start.ru_stime.tv_sec; + if( _now.ru_stime.tv_sec == _start.ru_stime.tv_sec ) + { + usremainder += _now.ru_stime.tv_usec - _start.ru_stime.tv_usec; + if( usremainder > 1000000) + { + ++result; + usremainder = 0; + } + } + return result; + } + + double wallclock() + { + return std::difftime( std::time(NULL) , wc_start ); + } + + protected: + struct rusage _start; + long int uuremainder; + long int usremainder; + time_t wc_start; +}; + +class eoTimerStat +{ + public: + + struct Stat + { + std::vector utime; + std::vector stime; + std::vector wtime; +#ifdef WITH_MPI + // Gives access to boost serialization + friend class boost::serialization::access; + + /** + * Serializes the statistique in a boost archive (useful for boost::mpi) + */ + template + void serialize( Archive & ar, const unsigned int version ) + { + ar & utime & stime & wtime; + (void) version; // avoid compilation warning + } +# endif + }; + +#ifdef WITH_MPI + // Gives access to boost serialization + friend class boost::serialization::access; + + /** + * Serializes the map of statistics in a boost archive (useful for boost::mpi) + */ + template + void serialize( Archive & ar, const unsigned int version ) + { + ar & _stats; + (void) version; // avoid compilation warning + } +# endif + + void start( const std::string & key ) + { + _timers[ key ].restart(); + } + + void stop( const std::string& key ) + { + Stat & s = _stats[ key ]; + eoTimer & t = _timers[ key ]; + s.utime.push_back( t.usertime() ); + s.stime.push_back( t.systime() ); + s.wtime.push_back( t.wallclock() ); + } + + std::map< std::string, Stat > stats() + { + return _stats; + } + + protected: + std::map< std::string, Stat > _stats; + std::map< std::string, eoTimer > _timers; +}; + +# endif // __TIMER_H__ + diff --git a/eo/test/mpi/eval.cpp b/eo/test/mpi/eval.cpp new file mode 100644 index 000000000..312c9d657 --- /dev/null +++ b/eo/test/mpi/eval.cpp @@ -0,0 +1,170 @@ +//----------------------------------------------------------------------------- +// t-eoMpiParallel.cpp +//----------------------------------------------------------------------------- + +#include +#include + +#include +// #include +#include "../real_value.h" + +#include + +#include + +#include + +#include +using namespace std; + +//----------------------------------------------------------------------------- + +class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial::Persistent +{ + public: + + eoRealSerializable(unsigned size = 0, double value = 0.0): + eoReal(size, value) {} + + eoserial::Object* pack() const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "array", + eoserial::makeArray< vector, eoserial::MakeAlgorithm > + ( *this ) + ); + return obj; + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpackArray< vector, eoserial::Array::UnpackAlgorithm > + ( *obj, "array", *this ); + } + + // Gives access to boost serialization + friend class boost::serialization::access; + + /** + * Serializes the decomposition in a boost archive (useful for boost::mpi) + */ + template + void save( Archive & ar, const unsigned int version ) const + { + std::stringstream ss; + printOn( ss ); + std::string asStr = ss.str(); + ar & asStr; + + (void) version; // avoid compilation warning + } + + /** + * Deserializes the decomposition from a boost archive (useful for boost:mpi) + */ + template + void load( Archive & ar, const unsigned int version ) + { + std::string asStr; + ar & asStr; + std::stringstream ss; + ss << asStr; + readFrom( ss ); + + (void) version; // avoid compilation warning + } + + // Indicates that boost save and load operations are not the same. + BOOST_SERIALIZATION_SPLIT_MEMBER() + +}; + +typedef eoRealSerializable EOT; + +struct CatBestAnswers : public eo::mpi::HandleResponseParallelApply +{ + CatBestAnswers() + { + best.fitness( 1000000000. ); + } + + using eo::mpi::HandleResponseParallelApply::_wrapped; + using eo::mpi::HandleResponseParallelApply::d; + + void operator()(int wrkRank) + { + int index = d->assignedTasks[wrkRank].index; + int size = d->assignedTasks[wrkRank].size; + (*_wrapped)( wrkRank ); + for(int i = index; i < index+size; ++i) + { + if( best.fitness() < d->data()[ i ].fitness() ) + { + eo::log << eo::quiet << "Better solution found:" << d->data()[i].fitness() << std::endl; + best = d->data()[ i ]; + } + } + } + + protected: + + EOT best; +}; + +int main(int ac, char** av) +{ + eo::mpi::Node::init( ac, av ); + eo::log << eo::setlevel( eo::quiet ); + + eoParser parser(ac, av); + + unsigned int popSize = parser.getORcreateParam((unsigned int)100, "popSize", "Population Size", 'P', "Evolution Engine").value(); + unsigned int dimSize = parser.getORcreateParam((unsigned int)10, "dimSize", "Dimension Size", 'd', "Evolution Engine").value(); + + uint32_t seedParam = parser.getORcreateParam((uint32_t)0, "seed", "Random number seed", 0).value(); + if (seedParam == 0) { seedParam = time(0); } + + make_parallel(parser); + make_help(parser); + + rng.reseed( seedParam ); + + eoUniformGenerator< double > gen(-5, 5); + eoInitFixedLength< EOT > init( dimSize, gen ); + + eoEvalFuncPtr< EOT, double, const std::vector< double >& > mainEval( real_value ); + eoEvalFuncCounter< EOT > eval( mainEval ); + + int rank = eo::mpi::Node::comm().rank(); + eo::mpi::DynamicAssignmentAlgorithm assign; + if( rank == eo::mpi::DEFAULT_MASTER ) + { + eoPop< EOT > pop( popSize, init ); + + eo::log << "Size of population : " << popSize << std::endl; + + eo::mpi::ParallelEvalStore< EOT > store( eval, eo::mpi::DEFAULT_MASTER ); + store.wrapHandleResponse( new CatBestAnswers ); + + eoParallelPopLoopEval< EOT > popEval( eval, assign, &store, eo::mpi::DEFAULT_MASTER, 3 ); + eo::log << eo::quiet << "Before first evaluation." << std::endl; + popEval( pop, pop ); + eo::log << eo::quiet << "After first evaluation." << std::endl; + + pop = eoPop< EOT >( popSize, init ); + popEval( pop, pop ); + eo::log << eo::quiet << "After second evaluation." << std::endl; + + eo::log << eo::quiet << "DONE!" << std::endl; + } else + { + eoPop< EOT > pop( popSize, init ); + eoParallelPopLoopEval< EOT > popEval( eval, assign, eo::mpi::DEFAULT_MASTER, 3 ); + popEval( pop, pop ); + } + + return 0; +} + +//----------------------------------------------------------------------------- diff --git a/eo/test/mpi/multipleRoles.cpp b/eo/test/mpi/multipleRoles.cpp new file mode 100644 index 000000000..ce10f6214 --- /dev/null +++ b/eo/test/mpi/multipleRoles.cpp @@ -0,0 +1,108 @@ +# include +# include + +# include + +# include + +# include +using namespace std; + +using namespace eo::mpi; + +// Role map +// 0 : general master +// 1, 2 : worker of general job, master of subjob +// 3 to 7 : workers of subjob + +struct SubWork: public eoUF< int&, void > +{ + void operator() ( int & x ) + { + cout << "Subwork phase." << endl; + ++x; + } +}; + +void subtask( vector& v, int rank ) +{ + vector workers; + workers.push_back( rank + 2 ); + workers.push_back( rank + 4 ); + DynamicAssignmentAlgorithm algo( workers ); + SubWork sw; + + ParallelApplyStore store( sw, v, rank ); + ParallelApply job( algo, rank, store ); + job.run(); +} + +struct Work: public eoUF< vector&, void > +{ + void operator() ( vector& v ) + { + cout << "Work phase..." << endl; + subtask( v, Node::comm().rank() ); + for( int i = 0; i < v.size(); ++i ) + { + v[i] *= 2; + } + } +}; + +int main(int argc, char** argv) +{ + // eo::log << eo::setlevel( eo::debug ); + Node::init( argc, argv ); + vector v; + + v.push_back(1); + v.push_back(3); + v.push_back(3); + v.push_back(7); + v.push_back(42); + + vector< vector > metaV; + metaV.push_back( v ); + metaV.push_back( v ); + + switch( Node::comm().rank() ) + { + case 0: + case 1: + case 2: + { + Work w; + DynamicAssignmentAlgorithm algo( 1, 2 ); + ParallelApplyStore< vector > store( w, metaV, 0 ); + ParallelApply< vector > job( algo, 0, store ); + job.run(); + if( job.isMaster() ) + { + v = metaV[0]; + cout << "Results : " << endl; + for(int i = 0; i < v.size(); ++i) + { + cout << v[i] << ' '; + } + cout << endl; + } + } + break; + + default: + { + // all the other nodes are sub workers + int rank = Node::comm().rank(); + if ( rank == 3 or rank == 5 ) + { + subtask( v, 1 ); + } else { + subtask( v, 2 ); + } + } + break; + } + + return 0; +} diff --git a/eo/test/mpi/parallelApply.cpp b/eo/test/mpi/parallelApply.cpp new file mode 100644 index 000000000..d562ef0ec --- /dev/null +++ b/eo/test/mpi/parallelApply.cpp @@ -0,0 +1,148 @@ +# include +# include + +# include + +# include +using namespace std; + +using namespace eo::mpi; + +struct plusOne : public eoUF< int&, void > +{ + void operator() ( int & x ) + { + ++x; + } +}; + +struct Test +{ + AssignmentAlgorithm * assign; + string description; + int requiredNodesNumber; // nb : chosen nodes ranks must be sequential +}; + +// These tests require at least 3 processes to be launched. +int main(int argc, char** argv) +{ + // eo::log << eo::setlevel( eo::debug ); + eo::log << eo::setlevel( eo::quiet ); + bool launchOnlyOne = false ; // Set this to true if you wanna launch only the first test. + + Node::init( argc, argv ); + + srand( time(0) ); + vector v; + for( int i = 0; i < 1000; ++i ) + { + v.push_back( rand() ); + } + + int offset = 0; + vector originalV = v; + + plusOne plusOneInstance; + + vector< Test > tests; + + const int ALL = Node::comm().size(); + + Test tIntervalStatic; + tIntervalStatic.assign = new StaticAssignmentAlgorithm( 1, REST_OF_THE_WORLD, v.size() ); + tIntervalStatic.description = "Correct static assignment with interval."; + tIntervalStatic.requiredNodesNumber = ALL; + tests.push_back( tIntervalStatic ); + + if( !launchOnlyOne ) + { + Test tWorldStatic; + tWorldStatic.assign = new StaticAssignmentAlgorithm( v.size() ); + tWorldStatic.description = "Correct static assignment with whole world as workers."; + tWorldStatic.requiredNodesNumber = ALL; + tests.push_back( tWorldStatic ); + + Test tStaticOverload; + tStaticOverload.assign = new StaticAssignmentAlgorithm( v.size()+100 ); + tStaticOverload.description = "Static assignment with too many runs."; + tStaticOverload.requiredNodesNumber = ALL; + tests.push_back( tStaticOverload ); + + Test tUniqueStatic; + tUniqueStatic.assign = new StaticAssignmentAlgorithm( 1, v.size() ); + tUniqueStatic.description = "Correct static assignment with unique worker."; + tUniqueStatic.requiredNodesNumber = 2; + tests.push_back( tUniqueStatic ); + + Test tVectorStatic; + vector workers; + workers.push_back( 1 ); + workers.push_back( 2 ); + tVectorStatic.assign = new StaticAssignmentAlgorithm( workers, v.size() ); + tVectorStatic.description = "Correct static assignment with precise workers specified."; + tVectorStatic.requiredNodesNumber = 3; + tests.push_back( tVectorStatic ); + + Test tIntervalDynamic; + tIntervalDynamic.assign = new DynamicAssignmentAlgorithm( 1, REST_OF_THE_WORLD ); + tIntervalDynamic.description = "Dynamic assignment with interval."; + tIntervalDynamic.requiredNodesNumber = ALL; + tests.push_back( tIntervalDynamic ); + + Test tUniqueDynamic; + tUniqueDynamic.assign = new DynamicAssignmentAlgorithm( 1 ); + tUniqueDynamic.description = "Dynamic assignment with unique worker."; + tUniqueDynamic.requiredNodesNumber = 2; + tests.push_back( tUniqueDynamic ); + + Test tVectorDynamic; + tVectorDynamic.assign = new DynamicAssignmentAlgorithm( workers ); + tVectorDynamic.description = "Dynamic assignment with precise workers specified."; + tVectorDynamic.requiredNodesNumber = tVectorStatic.requiredNodesNumber; + tests.push_back( tVectorDynamic ); + + Test tWorldDynamic; + tWorldDynamic.assign = new DynamicAssignmentAlgorithm; + tWorldDynamic.description = "Dynamic assignment with whole world as workers."; + tWorldDynamic.requiredNodesNumber = ALL; + tests.push_back( tWorldDynamic ); + } + + for( unsigned int i = 0; i < tests.size(); ++i ) + { + ParallelApplyStore< int > store( plusOneInstance, v, eo::mpi::DEFAULT_MASTER, 3 ); + // Job< JobData > job( *(tests[i].assign), eo::mpi::DEFAULT_MASTER, store ); + ParallelApply< int > job( *(tests[i].assign), eo::mpi::DEFAULT_MASTER, store ); + + if( job.isMaster() ) + { + cout << "Test : " << tests[i].description << endl; + } + + if( Node::comm().rank() < tests[i].requiredNodesNumber ) + { + job.run(); + } + + if( job.isMaster() ) + { + ++offset; + for(int i = 0; i < v.size(); ++i) + { + cout << v[i] << ' '; + if( originalV[i] + offset != v[i] ) + { + cout << " <-- ERROR at this point." << endl; + exit( EXIT_FAILURE ); + } + } + cout << endl; + } + + Node::comm().barrier(); + + delete tests[i].assign; + } + return 0; +} + diff --git a/eo/test/mpi/wrapper.cpp b/eo/test/mpi/wrapper.cpp new file mode 100644 index 000000000..d8b72c98e --- /dev/null +++ b/eo/test/mpi/wrapper.cpp @@ -0,0 +1,88 @@ +# include +# include + +# include + +# include +using namespace std; + +using namespace eo::mpi; + +struct plusOne : public eoUF< int&, void > +{ + void operator() ( int & x ) + { + ++x; + } +}; + +template< class EOT > +struct ShowWrappedResult : public IsFinishedParallelApply +{ + using IsFinishedParallelApply::_wrapped; + + ShowWrappedResult ( IsFinishedParallelApply * w = 0 ) : IsFinishedParallelApply( w ), times( 0 ) + { + // empty + } + + bool operator()() + { + bool wrappedValue = _wrapped->operator()(); // (*_wrapped)(); + cout << times << ") Wrapped function would say that it is " << ( wrappedValue ? "":"not ") << "finished" << std::endl; + ++times; + return wrappedValue; + } + + private: + int times; +}; + +// These tests require at least 3 processes to be launched. +int main(int argc, char** argv) +{ + // eo::log << eo::setlevel( eo::debug ); + eo::log << eo::setlevel( eo::quiet ); + + Node::init( argc, argv ); + + srand( time(0) ); + vector v; + for( int i = 0; i < 1000; ++i ) + { + v.push_back( rand() ); + } + + int offset = 0; + vector originalV = v; + + plusOne plusOneInstance; + + StaticAssignmentAlgorithm assign( v.size() ); + + ParallelApplyStore< int > store( plusOneInstance, v, eo::mpi::DEFAULT_MASTER, 1 ); + store.wrapIsFinished( new ShowWrappedResult ); + + ParallelApply job( assign, eo::mpi::DEFAULT_MASTER, store ); + // Equivalent to: + // Job< ParallelApplyData > job( assign, 0, store ); + job.run(); + + if( job.isMaster() ) + { + ++offset; + for(int i = 0; i < v.size(); ++i) + { + cout << v[i] << ' '; + if( originalV[i] + offset != v[i] ) + { + cout << " <-- ERROR at this point." << endl; + exit( EXIT_FAILURE ); + } + } + cout << endl; + } + + return 0; +} +