From 5e97fbf7775fc4bf2a30259bcdf7b5ab8942bc10 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Mon, 23 Jul 2012 10:50:28 +0200 Subject: [PATCH 01/31] Added eompi.html to gitignore exceptions --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a6f796491..c1a3ba42c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ !lib/ !src/ !test/ +!eompi.html build/ From 1f5719c6474259fcde046eec8ad7f37dc872cf0c Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Mon, 23 Jul 2012 15:03:47 +0200 Subject: [PATCH 02/31] Put back example t-mpi-eval to use a customized store. --- eo/test/mpi/t-mpi-eval.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eo/test/mpi/t-mpi-eval.cpp b/eo/test/mpi/t-mpi-eval.cpp index 5216123ac..dc00a5485 100644 --- a/eo/test/mpi/t-mpi-eval.cpp +++ b/eo/test/mpi/t-mpi-eval.cpp @@ -196,14 +196,12 @@ int main(int ac, char** av) eo::log << "Size of population : " << popSize << std::endl; - /* eo::mpi::ParallelApplyStore< EOT > store( eval, eo::mpi::DEFAULT_MASTER ); store.wrapHandleResponse( new CatBestAnswers ); eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, &store ); - */ - eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, eval, 5 ); + //eoParallelPopLoopEval< EOT > popEval( assign, eo::mpi::DEFAULT_MASTER, eval, 5 ); eo::log << eo::quiet << "Before first evaluation." << std::endl; popEval( pop, pop ); From dc58ab7739fd8d6e970f25da87c161de84bfc1cb Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 15:40:54 +0200 Subject: [PATCH 03/31] Use own implementation of Boost::mpi API instead of real Boost::mpi. --- eo/src/mpi/eoMpi.cpp | 8 ++ eo/src/mpi/eoMpiNode.h | 4 +- eo/src/mpi/implMpi.hpp | 283 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 eo/src/mpi/implMpi.hpp diff --git a/eo/src/mpi/eoMpi.cpp b/eo/src/mpi/eoMpi.cpp index ab7543284..250d90bf4 100644 --- a/eo/src/mpi/eoMpi.cpp +++ b/eo/src/mpi/eoMpi.cpp @@ -9,3 +9,11 @@ namespace eo } } +namespace mpi +{ + void broadcast( communicator & comm, int value, int root ) + { + comm; // unused + MPI_Bcast( &value, 1, MPI_INT, root, MPI_COMM_WORLD ); + } +} diff --git a/eo/src/mpi/eoMpiNode.h b/eo/src/mpi/eoMpiNode.h index 27c03312c..f4ffa8a0f 100644 --- a/eo/src/mpi/eoMpiNode.h +++ b/eo/src/mpi/eoMpiNode.h @@ -22,8 +22,8 @@ Authors: # ifndef __MPI_NODE_H__ # define __MPI_NODE_H__ -# include -namespace bmpi = boost::mpi; +# include "implMpi.hpp" +namespace bmpi = mpi; namespace eo { diff --git a/eo/src/mpi/implMpi.hpp b/eo/src/mpi/implMpi.hpp new file mode 100644 index 000000000..4bf71dcc5 --- /dev/null +++ b/eo/src/mpi/implMpi.hpp @@ -0,0 +1,283 @@ +# ifndef __EO_MPI_HPP__ +# define __EO_MPI_HPP__ + +# include +# include + +# include + +namespace mpi +{ + const int any_source = MPI_ANY_SOURCE; + const int any_tag = MPI_ANY_TAG; + + class environment + { + public: + + environment(int argc, char**argv) + { + MPI_Init(&argc, &argv); + } + + ~environment() + { + MPI_Finalize(); + } + }; + + class status + { + public: + + status( const MPI_Status & s ) + { + _source = s.MPI_SOURCE; + _tag = s.MPI_TAG; + _error = s.MPI_ERROR; + } + + int tag() { return _tag; } + int error() { return _error; } + int source() { return _source; } + + private: + int _source; + int _tag; + int _error; + }; + + class communicator + { + public: + + communicator( ) + { + _rank = -1; + _size = -1; + + _buf = 0; + _bufsize = -1; + } + + ~communicator() + { + if( _buf ) + { + delete _buf; + _buf = 0; + } + } + + int rank() + { + if ( _rank == -1 ) + { + MPI_Comm_rank( MPI_COMM_WORLD, &_rank ); + } + return _rank; + } + + int size() + { + if ( _size == -1 ) + { + MPI_Comm_size( MPI_COMM_WORLD, &_size ); + } + return _size; + } + + /* + * SEND / RECV INT + */ + void send( int dest, int tag, int n ) + { + //send( dest, tag, &n, 1 ); + MPI_Send( &n, 1, MPI_INT, dest, tag, MPI_COMM_WORLD ); + } + + void recv( int src, int tag, int& n ) + { + MPI_Status stat; + MPI_Recv( &n, 1, MPI_INT, src, tag, MPI_COMM_WORLD , &stat ); + //recv( src, tag, &n, 1 ); + } + + /* + void send( int dest, int tag, int* table, int size ) + { + MPI_Send( table, size, MPI_INT, dest, tag, MPI_COMM_WORLD ); + } + + void recv( int src, int tag, int* table, int size ) + { + MPI_Status stat; + MPI_Recv( table, size, MPI_INT, src, tag, MPI_COMM_WORLD , &stat ); + } + */ + + /* + * SEND / RECV STRING + */ + void send( int dest, int tag, const std::string& str ) + { + int size = str.size() + 1; + send( dest, tag, size ); + MPI_Send( (char*)str.c_str(), size, MPI_CHAR, dest, tag, MPI_COMM_WORLD); + } + + void recv( int src, int tag, std::string& str ) + { + int size = -1; + MPI_Status stat; + recv( src, tag, size ); + + if( _buf == 0 ) + { + _buf = new char[ size ]; + _bufsize = size; + } else if( _bufsize < size ) + { + delete [] _buf; + _buf = new char[ size ]; + _bufsize = size; + } + MPI_Recv( _buf, size, MPI_CHAR, src, tag, MPI_COMM_WORLD, &stat ); + str.assign( _buf ); + } + + /* + * SEND / RECV Objects + */ + void send( int dest, int tag, const eoserial::Persistent & persistent ) + { + eoserial::Object* obj = persistent.pack(); + std::stringstream ss; + obj->print( ss ); + delete obj; + send( dest, tag, ss.str() ); + } + + /* + void send( int dest, int tag, eoserial::Persistent* table, int size ) + { + // Puts all the values into an array + eoserial::Array* array = new eoserial::Array; + + std::cout << "DEBUG EO: @ premier: " << table << " / @ second: " << table+1 << std::endl; + + for( int i = 0; i < size; ++i ) + { + array->push_back( table[i].pack() ); + } + + // Encapsulates the array into an object + eoserial::Object* obj = new eoserial::Object; + obj->add( "array", array ); + std::stringstream ss; + obj->print( ss ); + delete obj; + + // Sends the object as a string + send( dest, tag, ss.str() ); + } + */ + + template< class T > + void send( int dest, int tag, T* table, int size ) + { + // Puts all the values into an array + eoserial::Array* array = new eoserial::Array; + + for( int i = 0; i < size; ++i ) + { + array->push_back( table[i].pack() ); + } + + // Encapsulates the array into an object + eoserial::Object* obj = new eoserial::Object; + obj->add( "array", array ); + std::stringstream ss; + obj->print( ss ); + delete obj; + + // Sends the object as a string + send( dest, tag, ss.str() ); + } + + + void recv( int src, int tag, eoserial::Persistent & persistent ) + { + std::string asText; + recv( src, tag, asText ); + eoserial::Object* obj = eoserial::Parser::parse( asText ); + persistent.unpack( obj ); + delete obj; + } + + /* + void recv( int src, int tag, eoserial::Persistent* table, int size ) + { + // Receives the string which contains the object + std::string asText; + recv( src, tag, asText ); + + // Parses the object and retrieves the table + eoserial::Object* obj = eoserial::Parser::parse( asText ); + eoserial::Array* array = static_cast( (*obj)["array"] ); + + // Retrieves all the values from the array + for( int i = 0; i < size; ++i ) + { + eoserial::unpackObject( *array, i, table[i] ); + } + delete obj; + } + */ + + template< class T > + void recv( int src, int tag, T* table, int size ) + { + // Receives the string which contains the object + std::string asText; + recv( src, tag, asText ); + + // Parses the object and retrieves the table + eoserial::Object* obj = eoserial::Parser::parse( asText ); + eoserial::Array* array = static_cast( (*obj)["array"] ); + + // Retrieves all the values from the array + for( int i = 0; i < size; ++i ) + { + eoserial::unpackObject( *array, i, table[i] ); + } + delete obj; + } + + /* + * Other methods + */ + status probe( int src = any_source, int tag = any_tag ) + { + MPI_Status stat; + MPI_Probe( src, tag, MPI_COMM_WORLD , &stat ); + return status( stat ); + } + + void barrier() + { + MPI_Barrier( MPI_COMM_WORLD ); + } + + private: + int _rank; + int _size; + + char* _buf; + int _bufsize; + }; + + void broadcast( communicator & comm, int value, int root ); +} // namespace mpi + +# endif //__EO_MPI_HPP__ From 0e567783272d4963eea1f12c0e85dfc8a7eb210f Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 15:41:25 +0200 Subject: [PATCH 04/31] Updated the tests to conform the new Boost-like API --- eo/test/mpi/t-mpi-common.h | 43 +++++++++++++++++++++++++ eo/test/mpi/t-mpi-eval.cpp | 30 +++++++++++++---- eo/test/mpi/t-mpi-multipleRoles.cpp | 50 +++++++++++++++++++++-------- eo/test/mpi/t-mpi-parallelApply.cpp | 14 ++++---- eo/test/mpi/t-mpi-wrapper.cpp | 20 ++++++------ 5 files changed, 120 insertions(+), 37 deletions(-) create mode 100644 eo/test/mpi/t-mpi-common.h diff --git a/eo/test/mpi/t-mpi-common.h b/eo/test/mpi/t-mpi-common.h new file mode 100644 index 000000000..4217af28b --- /dev/null +++ b/eo/test/mpi/t-mpi-common.h @@ -0,0 +1,43 @@ +# ifndef __T_MPI_COMMON_H__ +# define __T_MPI_COMMON_H__ + +#include + +template< class T > +struct SerializableBase : public eoserial::Persistent +{ + public: + + operator T&() + { + return _value; + } + + SerializableBase() : _value() + { + // empty + } + + SerializableBase( T base ) : _value( base ) + { + // empty + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpack( *obj, "value", _value ); + } + + eoserial::Object* pack(void) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add("value", eoserial::make( _value ) ); + return obj; + } + + private: + T _value; +}; + + +# endif // __T_MPI_COMMON_H__ diff --git a/eo/test/mpi/t-mpi-eval.cpp b/eo/test/mpi/t-mpi-eval.cpp index dc00a5485..4187c4e63 100644 --- a/eo/test/mpi/t-mpi-eval.cpp +++ b/eo/test/mpi/t-mpi-eval.cpp @@ -34,7 +34,7 @@ Authors: #include -#include +// #include #include using namespace std; @@ -55,6 +55,14 @@ class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial eoserial::makeArray< vector, eoserial::MakeAlgorithm > ( *this ) ); + + bool invalidFitness = invalid(); + obj->add("invalid", eoserial::make( invalidFitness ) ); + if( !invalidFitness ) + { + double fitnessVal = fitness(); + obj->add("fitness", eoserial::make( fitnessVal ) ); + } return obj; } @@ -62,14 +70,23 @@ class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial { eoserial::unpackArray< vector, eoserial::Array::UnpackAlgorithm > ( *obj, "array", *this ); + + bool invalidFitness; + eoserial::unpack( *obj, "invalid", invalidFitness ); + if( invalidFitness ) { + invalidate(); + } else { + double fitnessVal; + eoserial::unpack( *obj, "fitness", fitnessVal ); + fitness( fitnessVal ); + } } + /* // Gives access to boost serialization friend class boost::serialization::access; - /** - * Serializes the decomposition in a boost archive (useful for boost::mpi) - */ + template void save( Archive & ar, const unsigned int version ) const { @@ -81,9 +98,7 @@ class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial (void) version; // avoid compilation warning } - /** - * Deserializes the decomposition from a boost archive (useful for boost:mpi) - */ + template void load( Archive & ar, const unsigned int version ) { @@ -98,6 +113,7 @@ class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial // Indicates that boost save and load operations are not the same. BOOST_SERIALIZATION_SPLIT_MEMBER() + */ }; diff --git a/eo/test/mpi/t-mpi-multipleRoles.cpp b/eo/test/mpi/t-mpi-multipleRoles.cpp index 36d588b82..db4b1bb6c 100644 --- a/eo/test/mpi/t-mpi-multipleRoles.cpp +++ b/eo/test/mpi/t-mpi-multipleRoles.cpp @@ -41,7 +41,9 @@ Authors: # include # include -# include +# include "t-mpi-common.h" + +// # include # include @@ -50,10 +52,30 @@ using namespace std; using namespace eo::mpi; -// The real job to execute, for the subworkers: add one to each element of a table. -struct SubWork: public eoUF< int&, void > +template< class T > +struct SerializableVector : public std::vector, public eoserial::Persistent { - void operator() ( int & x ) + public: + + void unpack( const eoserial::Object* obj ) + { + this->clear(); + eoserial::Array* vector = static_cast( obj->find("vector")->second ); + vector->deserialize< std::vector, eoserial::Array::UnpackObjectAlgorithm >( *this ); + } + + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add("vector", eoserial::makeArray< std::vector, eoserial::SerializablePushAlgorithm >( *this ) ); + return obj; + } +}; + +// The real job to execute, for the subworkers: add one to each element of a table. +struct SubWork: public eoUF< SerializableBase&, void > +{ + void operator() ( SerializableBase & x ) { cout << "Subwork phase." << endl; ++x; @@ -62,7 +84,7 @@ struct SubWork: public eoUF< int&, void > // Function called by both subworkers and delegates. // v is the vector to process, rank is the MPI rank of the sub master -void subtask( vector& v, int rank ) +void subtask( vector< SerializableBase >& v, int rank ) { // Attach workers according to nodes. // Submaster with rank 1 will have ranks 3 and 5 as subworkers. @@ -74,9 +96,9 @@ void subtask( vector& v, int rank ) SubWork sw; // Launch the job! - ParallelApplyStore store( sw, rank ); + ParallelApplyStore< SerializableBase > store( sw, rank ); store.data( v ); - ParallelApply job( algo, rank, store ); + ParallelApply< SerializableBase > job( algo, rank, store ); job.run(); EmptyJob stop( algo, rank ); } @@ -85,9 +107,9 @@ void subtask( vector& v, int rank ) // each result by two). // Note that this work receives a vector of integers as an entry, while subworkers task's operator receives a simple // integer. -struct Work: public eoUF< vector&, void > +struct Work: public eoUF< SerializableVector< SerializableBase >&, void > { - void operator() ( vector& v ) + void operator() ( SerializableVector< SerializableBase >& v ) { cout << "Work phase..." << endl; subtask( v, Node::comm().rank() ); @@ -103,10 +125,10 @@ int main(int argc, char** argv) // eo::log << eo::setlevel( eo::debug ); Node::init( argc, argv ); if( Node::comm().size() != 7 ) { - throw std::runtime_error("World size should be 7."); + // throw std::runtime_error("World size should be 7."); } - vector v; + SerializableVector< SerializableBase > v; v.push_back(1); v.push_back(3); @@ -116,7 +138,7 @@ int main(int argc, char** argv) // As submasters' operator receives a vector as an input, and ParallelApply takes a vector of // operator's input as an input, we have to deal with a vector of vector of integers for the master task. - vector< vector > metaV; + vector< SerializableVector< SerializableBase > > metaV; // Here, we send twice the same vector. We could also have splitted the first vector into two vectors, one // containing the beginning and another one containing the end. metaV.push_back( v ); @@ -132,9 +154,9 @@ int main(int argc, char** argv) { Work w; DynamicAssignmentAlgorithm algo( 1, 2 ); - ParallelApplyStore< vector > store( w, 0 ); + ParallelApplyStore< SerializableVector< SerializableBase > > store( w, 0 ); store.data( metaV ); - ParallelApply< vector > job( algo, 0, store ); + ParallelApply< SerializableVector< SerializableBase > > job( algo, 0, store ); job.run(); if( job.isMaster() ) { diff --git a/eo/test/mpi/t-mpi-parallelApply.cpp b/eo/test/mpi/t-mpi-parallelApply.cpp index 7cefa203a..df90f0f56 100644 --- a/eo/test/mpi/t-mpi-parallelApply.cpp +++ b/eo/test/mpi/t-mpi-parallelApply.cpp @@ -40,6 +40,8 @@ Authors: # include # include +# include "t-mpi-common.h" + # include # include @@ -50,9 +52,9 @@ using namespace eo::mpi; /* * The function to be called on each element of the table: just increment the value. */ -struct plusOne : public eoUF< int&, void > +struct plusOne : public eoUF< SerializableBase&, void > { - void operator() ( int & x ) + void operator() ( SerializableBase & x ) { ++x; } @@ -79,7 +81,7 @@ int main(int argc, char** argv) // Initializes a vector with random values. srand( time(0) ); - vector v; + vector< SerializableBase > v; for( int i = 0; i < 1000; ++i ) { v.push_back( rand() ); @@ -90,7 +92,7 @@ int main(int argc, char** argv) // incremented and we can compare the returned value of each element to the value of each element in originalV + // offset. If the two values are different, there has been a problem. int offset = 0; - vector originalV = v; + vector< SerializableBase > originalV = v; // Instanciates the functor to apply on each element plusOne plusOneInstance; @@ -166,11 +168,11 @@ int main(int argc, char** argv) for( unsigned int i = 0; i < tests.size(); ++i ) { // Instanciates a store with the functor, the master rank and size of packet (see ParallelApplyStore doc). - ParallelApplyStore< int > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 3 ); + ParallelApplyStore< SerializableBase > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 3 ); // Updates the contained data store.data( v ); // Creates the job with the assignment algorithm, the master rank and the store - ParallelApply< int > job( *(tests[i].assign), eo::mpi::DEFAULT_MASTER, store ); + ParallelApply< SerializableBase > job( *(tests[i].assign), eo::mpi::DEFAULT_MASTER, store ); // Only master writes information if( job.isMaster() ) diff --git a/eo/test/mpi/t-mpi-wrapper.cpp b/eo/test/mpi/t-mpi-wrapper.cpp index 97d4b04a1..7da45ad4e 100644 --- a/eo/test/mpi/t-mpi-wrapper.cpp +++ b/eo/test/mpi/t-mpi-wrapper.cpp @@ -33,6 +33,8 @@ Authors: # include # include +# include "t-mpi-common.h" + # include # include @@ -41,9 +43,9 @@ using namespace std; using namespace eo::mpi; // Job functor. -struct plusOne : public eoUF< int&, void > +struct plusOne : public eoUF< SerializableBase&, void > { - void operator() ( int & x ) + void operator() ( SerializableBase& x ) { ++x; } @@ -83,28 +85,28 @@ int main(int argc, char** argv) Node::init( argc, argv ); srand( time(0) ); - vector v; + vector< SerializableBase > v; for( int i = 0; i < 1000; ++i ) { v.push_back( rand() ); } int offset = 0; - vector originalV = v; + vector< SerializableBase > originalV = v; plusOne plusOneInstance; StaticAssignmentAlgorithm assign( v.size() ); - ParallelApplyStore< int > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 1 ); + ParallelApplyStore< SerializableBase > store( plusOneInstance, eo::mpi::DEFAULT_MASTER, 1 ); store.data( v ); // This is the only thing which changes: we wrap the IsFinished function. // According to RAII, we'll delete the invokated wrapper at the end of the main ; the store won't delete it // automatically. - IsFinishedParallelApply* wrapper = new ShowWrappedResult; - store.wrapIsFinished( wrapper ); + ShowWrappedResult< SerializableBase > wrapper; + store.wrapIsFinished( &wrapper ); - ParallelApply job( assign, eo::mpi::DEFAULT_MASTER, store ); + ParallelApply< SerializableBase > job( assign, eo::mpi::DEFAULT_MASTER, store ); // Equivalent to: // Job< ParallelApplyData > job( assign, 0, store ); job.run(); @@ -125,8 +127,6 @@ int main(int argc, char** argv) cout << endl; } - delete wrapper; - return 0; } From 191b280371e69af31ea36d4be354c6b76a31dc52 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 16:16:56 +0200 Subject: [PATCH 05/31] Code organisation, splitted headers files in implementation files, as much as possible (impossible for templates functions, thank you C++) --- eo/src/mpi/CMakeLists.txt | 3 + eo/src/mpi/eoMpi.cpp | 26 ++- eo/src/mpi/eoMpi.h | 12 +- eo/src/mpi/eoMpiAssignmentAlgorithm.cpp | 204 +++++++++++++++++ eo/src/mpi/eoMpiAssignmentAlgorithm.h | 188 ++-------------- eo/src/mpi/eoMpiNode.cpp | 19 ++ eo/src/mpi/eoMpiNode.h | 18 +- eo/src/mpi/implMpi.cpp | 146 ++++++++++++ eo/src/mpi/implMpi.h | 129 +++++++++++ eo/src/mpi/implMpi.hpp | 283 ------------------------ 10 files changed, 550 insertions(+), 478 deletions(-) create mode 100644 eo/src/mpi/eoMpiAssignmentAlgorithm.cpp create mode 100644 eo/src/mpi/eoMpiNode.cpp create mode 100644 eo/src/mpi/implMpi.cpp create mode 100644 eo/src/mpi/implMpi.h delete mode 100644 eo/src/mpi/implMpi.hpp diff --git a/eo/src/mpi/CMakeLists.txt b/eo/src/mpi/CMakeLists.txt index 0e22e47e0..00a338777 100644 --- a/eo/src/mpi/CMakeLists.txt +++ b/eo/src/mpi/CMakeLists.txt @@ -14,6 +14,9 @@ SET(LIBRARY_OUTPUT_PATH ${EOMPI_LIB_OUTPUT_PATH}) SET(EOMPI_SOURCES eoMpi.cpp + eoMpiAssignmentAlgorithm.cpp + eoMpiNode.cpp + implMpi.cpp ) ADD_LIBRARY(eompi STATIC ${EOMPI_SOURCES}) diff --git a/eo/src/mpi/eoMpi.cpp b/eo/src/mpi/eoMpi.cpp index 250d90bf4..56422c13d 100644 --- a/eo/src/mpi/eoMpi.cpp +++ b/eo/src/mpi/eoMpi.cpp @@ -4,16 +4,24 @@ namespace eo { namespace mpi { - bmpi::communicator Node::_comm; + /********************************************** + * *********** GLOBALS ************************ + * *******************************************/ eoTimerStat timerStat; - } -} -namespace mpi -{ - void broadcast( communicator & comm, int value, int root ) - { - comm; // unused - MPI_Bcast( &value, 1, MPI_INT, root, MPI_COMM_WORLD ); + namespace Channel + { + const int Commands = 0; + const int Messages = 1; + } + + namespace Message + { + const int Continue = 0; + const int Finish = 1; + const int Kill = 2; + } + + const int DEFAULT_MASTER = 0; } } diff --git a/eo/src/mpi/eoMpi.h b/eo/src/mpi/eoMpi.h index ee92e0878..92c2ba99b 100644 --- a/eo/src/mpi/eoMpi.h +++ b/eo/src/mpi/eoMpi.h @@ -143,8 +143,8 @@ namespace eo */ namespace Channel { - const int Commands = 0; - const int Messages = 1; + extern const int Commands; + extern const int Messages; } /** @@ -157,9 +157,9 @@ namespace eo */ namespace Message { - const int Continue = 0; - const int Finish = 1; - const int Kill = 2; + extern const int Continue; + extern const int Finish; + extern const int Kill; } /** @@ -167,7 +167,7 @@ namespace eo * * @ingroup MPI */ - const int DEFAULT_MASTER = 0; + extern const int DEFAULT_MASTER; /** * @brief Base class for the 4 algorithm functors. diff --git a/eo/src/mpi/eoMpiAssignmentAlgorithm.cpp b/eo/src/mpi/eoMpiAssignmentAlgorithm.cpp new file mode 100644 index 000000000..e51fc2b51 --- /dev/null +++ b/eo/src/mpi/eoMpiAssignmentAlgorithm.cpp @@ -0,0 +1,204 @@ +# include "eoMpiAssignmentAlgorithm.h" + +# include "eoMpiNode.h" + +namespace eo +{ + namespace mpi + { + const int REST_OF_THE_WORLD = -1; + + /******************************************************** + * DYNAMIC ASSIGNMENT ALGORITHM ************************* + *******************************************************/ + + DynamicAssignmentAlgorithm::DynamicAssignmentAlgorithm( ) + { + for(int i = 1; i < Node::comm().size(); ++i) + { + availableWrk.push_back( i ); + } + } + + DynamicAssignmentAlgorithm::DynamicAssignmentAlgorithm( int unique ) + { + availableWrk.push_back( unique ); + } + + DynamicAssignmentAlgorithm::DynamicAssignmentAlgorithm( const std::vector & workers ) + { + availableWrk = workers; + } + + DynamicAssignmentAlgorithm::DynamicAssignmentAlgorithm( int first, int last ) + { + if( last == REST_OF_THE_WORLD ) + { + last = Node::comm().size() - 1; + } + + for( int i = first; i <= last; ++i) + { + availableWrk.push_back( i ); + } + } + + int DynamicAssignmentAlgorithm::get( ) + { + int assignee = -1; + if (! availableWrk.empty() ) + { + assignee = availableWrk.back(); + availableWrk.pop_back(); + } + return assignee; + } + + int DynamicAssignmentAlgorithm::availableWorkers() + { + return availableWrk.size(); + } + + void DynamicAssignmentAlgorithm::confirm( int rank ) + { + availableWrk.push_back( rank ); + } + + std::vector DynamicAssignmentAlgorithm::idles( ) + { + return availableWrk; + } + + void DynamicAssignmentAlgorithm::reinit( int _ ) + { + ++_; + // nothing to do + } + + /******************************************************** + * STATIC ASSIGNMENT ALGORITHM ************************** + *******************************************************/ + + StaticAssignmentAlgorithm::StaticAssignmentAlgorithm( std::vector& workers, int runs ) + { + init( workers, runs ); + } + + StaticAssignmentAlgorithm::StaticAssignmentAlgorithm( int first, int last, int runs ) + { + std::vector workers; + + if( last == REST_OF_THE_WORLD ) + { + last = Node::comm().size() - 1; + } + + for(int i = first; i <= last; ++i) + { + workers.push_back( i ); + } + init( workers, runs ); + } + + StaticAssignmentAlgorithm::StaticAssignmentAlgorithm( int runs ) + { + std::vector workers; + for(int i = 1; i < Node::comm().size(); ++i) + { + workers.push_back( i ); + } + + init( workers, runs ); + } + + StaticAssignmentAlgorithm::StaticAssignmentAlgorithm( int unique, int runs ) + { + std::vector workers; + workers.push_back( unique ); + init( workers, runs ); + } + + void StaticAssignmentAlgorithm::init( std::vector & workers, int runs ) + { + unsigned int nbWorkers = workers.size(); + freeWorkers = nbWorkers; + + busy.clear(); + busy.resize( nbWorkers, false ); + realRank = workers; + + if( runs <= 0 ) + { + return; + } + + attributions.clear(); + attributions.reserve( nbWorkers ); + + // Let be the euclidean division of runs by nbWorkers : + // runs == q * nbWorkers + r, 0 <= r < nbWorkers + // This one liner affects q requests to each worker + for (unsigned int i = 0; i < nbWorkers; attributions[i++] = runs / nbWorkers) ; + // The first line computes r and the one liner affects the remaining + // r requests to workers, in ascending order + unsigned int diff = runs - (runs / nbWorkers) * nbWorkers; + for (unsigned int i = 0; i < diff; ++attributions[i++]); + } + + int StaticAssignmentAlgorithm::get( ) + { + int assignee = -1; + for( unsigned i = 0; i < busy.size(); ++i ) + { + if( !busy[i] && attributions[i] > 0 ) + { + busy[i] = true; + --freeWorkers; + assignee = realRank[ i ]; + break; + } + } + return assignee; + } + + int StaticAssignmentAlgorithm::availableWorkers( ) + { + return freeWorkers; + } + + std::vector StaticAssignmentAlgorithm::idles() + { + std::vector ret; + for(unsigned int i = 0; i < busy.size(); ++i) + { + if( !busy[i] ) + { + ret.push_back( realRank[i] ); + } + } + return ret; + } + + void StaticAssignmentAlgorithm::confirm( int rank ) + { + int i = -1; // i is the real index in table + for( unsigned int j = 0; j < realRank.size(); ++j ) + { + if( realRank[j] == rank ) + { + i = j; + break; + } + } + + --attributions[ i ]; + busy[ i ] = false; + ++freeWorkers; + } + + void StaticAssignmentAlgorithm::reinit( int runs ) + { + init( realRank, runs ); + } + } +} diff --git a/eo/src/mpi/eoMpiAssignmentAlgorithm.h b/eo/src/mpi/eoMpiAssignmentAlgorithm.h index 07f674157..3389db684 100644 --- a/eo/src/mpi/eoMpiAssignmentAlgorithm.h +++ b/eo/src/mpi/eoMpiAssignmentAlgorithm.h @@ -23,7 +23,6 @@ Authors: # define __MPI_ASSIGNMENT_ALGORITHM_H__ # include // std::vector -# include "eoMpiNode.h" namespace eo { @@ -35,7 +34,7 @@ namespace eo * * @ingroup MPI */ - const int REST_OF_THE_WORLD = -1; + extern const int REST_OF_THE_WORLD; /** * @brief Contains informations on the available workers and allows to find assignees for jobs. @@ -115,33 +114,21 @@ namespace eo /** * @brief Uses all the hosts whose rank is higher to 1, inclusive, as workers. */ - DynamicAssignmentAlgorithm( ) - { - for(int i = 1; i < Node::comm().size(); ++i) - { - availableWrk.push_back( i ); - } - } + DynamicAssignmentAlgorithm( ); /** * @brief Uses the unique host with given rank as a worker. * * @param unique MPI rank of the unique worker. */ - DynamicAssignmentAlgorithm( int unique ) - { - availableWrk.push_back( unique ); - } + DynamicAssignmentAlgorithm( int unique ); /** * @brief Uses the workers whose ranks are present in the argument as workers. * * @param workers std::vector containing MPI ranks of workers. */ - DynamicAssignmentAlgorithm( const std::vector & workers ) - { - availableWrk = workers; - } + DynamicAssignmentAlgorithm( const std::vector & workers ); /** * @brief Uses a range of ranks as workers. @@ -150,50 +137,17 @@ namespace eo * @param last The last worker to be included (inclusive). If last == eo::mpi::REST_OF_THE_WORLD, all * hosts whose rank is higher than first are taken. */ - DynamicAssignmentAlgorithm( int first, int last ) - { - if( last == REST_OF_THE_WORLD ) - { - last = Node::comm().size() - 1; - } + DynamicAssignmentAlgorithm( int first, int last ); - for( int i = first; i <= last; ++i) - { - availableWrk.push_back( i ); - } - } + virtual int get( ); - virtual int get( ) - { - int assignee = -1; - if (! availableWrk.empty() ) - { - assignee = availableWrk.back(); - availableWrk.pop_back(); - } - return assignee; - } + int availableWorkers(); - int availableWorkers() - { - return availableWrk.size(); - } + void confirm( int rank ); - void confirm( int rank ) - { - availableWrk.push_back( rank ); - } + std::vector idles( ); - std::vector idles( ) - { - return availableWrk; - } - - void reinit( int _ ) - { - ++_; - // nothing to do - } + void reinit( int _ ); protected: std::vector< int > availableWrk; @@ -223,10 +177,7 @@ namespace eo * @param workers std::vector of MPI ranks of workers which will be used. * @param runs Fixed amount of runs, strictly positive. */ - StaticAssignmentAlgorithm( std::vector& workers, int runs ) - { - init( workers, runs ); - } + StaticAssignmentAlgorithm( std::vector& workers, int runs ); /** * @brief Uses a range of workers. @@ -236,21 +187,7 @@ namespace eo * workers from the first one are taken as workers. * @param runs Fixed amount of runs, strictly positive. */ - StaticAssignmentAlgorithm( int first, int last, int runs ) - { - std::vector workers; - - if( last == REST_OF_THE_WORLD ) - { - last = Node::comm().size() - 1; - } - - for(int i = first; i <= last; ++i) - { - workers.push_back( i ); - } - init( workers, runs ); - } + StaticAssignmentAlgorithm( int first, int last, int runs ); /** * @brief Uses all the hosts whose rank is higher than 1 (inclusive) as workers. @@ -258,16 +195,7 @@ namespace eo * @param runs Fixed amount of runs, strictly positive. If it's not set, you'll have to call reinit() * later. */ - StaticAssignmentAlgorithm( int runs = 0 ) - { - std::vector workers; - for(int i = 1; i < Node::comm().size(); ++i) - { - workers.push_back( i ); - } - - init( workers, runs ); - } + StaticAssignmentAlgorithm( int runs = 0 ); /** * @brief Uses an unique host as worker. @@ -275,12 +203,7 @@ namespace eo * @param unique The MPI rank of the host which will be the worker. * @param runs Fixed amount of runs, strictly positive. */ - StaticAssignmentAlgorithm( int unique, int runs ) - { - std::vector workers; - workers.push_back( unique ); - init( workers, runs ); - } + StaticAssignmentAlgorithm( int unique, int runs ); private: /** @@ -292,89 +215,18 @@ namespace eo * @param workers Vector of hosts' ranks * @param runs Fixed amount of runs, strictly positive. */ - void init( std::vector & workers, int runs ) - { - unsigned int nbWorkers = workers.size(); - freeWorkers = nbWorkers; - - busy.clear(); - busy.resize( nbWorkers, false ); - realRank = workers; - - if( runs <= 0 ) - { - return; - } - - attributions.clear(); - attributions.reserve( nbWorkers ); - - // Let be the euclidean division of runs by nbWorkers : - // runs == q * nbWorkers + r, 0 <= r < nbWorkers - // This one liner affects q requests to each worker - for (unsigned int i = 0; i < nbWorkers; attributions[i++] = runs / nbWorkers) ; - // The first line computes r and the one liner affects the remaining - // r requests to workers, in ascending order - unsigned int diff = runs - (runs / nbWorkers) * nbWorkers; - for (unsigned int i = 0; i < diff; ++attributions[i++]); - } + void init( std::vector & workers, int runs ); public: - int get( ) - { - int assignee = -1; - for( unsigned i = 0; i < busy.size(); ++i ) - { - if( !busy[i] && attributions[i] > 0 ) - { - busy[i] = true; - --freeWorkers; - assignee = realRank[ i ]; - break; - } - } - return assignee; - } + int get( ); - int availableWorkers( ) - { - return freeWorkers; - } + int availableWorkers( ); - std::vector idles() - { - std::vector ret; - for(unsigned int i = 0; i < busy.size(); ++i) - { - if( !busy[i] ) - { - ret.push_back( realRank[i] ); - } - } - return ret; - } + std::vector idles(); - void confirm( int rank ) - { - int i = -1; // i is the real index in table - for( unsigned int j = 0; j < realRank.size(); ++j ) - { - if( realRank[j] == rank ) - { - i = j; - break; - } - } + void confirm( int rank ); - --attributions[ i ]; - busy[ i ] = false; - ++freeWorkers; - } - - void reinit( int runs ) - { - init( realRank, runs ); - } + void reinit( int runs ); private: std::vector attributions; diff --git a/eo/src/mpi/eoMpiNode.cpp b/eo/src/mpi/eoMpiNode.cpp new file mode 100644 index 000000000..22e88da04 --- /dev/null +++ b/eo/src/mpi/eoMpiNode.cpp @@ -0,0 +1,19 @@ +# include "eoMpiNode.h" + +namespace eo +{ + namespace mpi + { + void Node::init( int argc, char** argv ) + { + static bmpi::environment env( argc, argv ); + } + + bmpi::communicator& Node::comm() + { + return _comm; + } + + bmpi::communicator Node::_comm; + } +} diff --git a/eo/src/mpi/eoMpiNode.h b/eo/src/mpi/eoMpiNode.h index f4ffa8a0f..c370e1c0b 100644 --- a/eo/src/mpi/eoMpiNode.h +++ b/eo/src/mpi/eoMpiNode.h @@ -22,7 +22,7 @@ Authors: # ifndef __MPI_NODE_H__ # define __MPI_NODE_H__ -# include "implMpi.hpp" +# include "implMpi.h" namespace bmpi = mpi; namespace eo @@ -30,9 +30,9 @@ namespace eo namespace mpi { /** - * @brief Global object used to reach boost::mpi::communicator everywhere. + * @brief Global object used to reach mpi::communicator everywhere. * - * boost::mpi::communicator is the main object used to send and receive messages between the different hosts of + * mpi::communicator is the main object used to send and receive messages between the different hosts of * a MPI algorithm. * * @ingroup MPI @@ -49,18 +49,12 @@ namespace eo * @param argc Main's argc * @param argv Main's argv */ - static void init( int argc, char** argv ) - { - static bmpi::environment env( argc, argv ); - } + static void init( int argc, char** argv ); /** - * @brief Returns the global boost::mpi::communicator + * @brief Returns the global mpi::communicator */ - static bmpi::communicator& comm() - { - return _comm; - } + static bmpi::communicator& comm(); protected: static bmpi::communicator _comm; diff --git a/eo/src/mpi/implMpi.cpp b/eo/src/mpi/implMpi.cpp new file mode 100644 index 000000000..ae0dd1c91 --- /dev/null +++ b/eo/src/mpi/implMpi.cpp @@ -0,0 +1,146 @@ +# include "implMpi.h" + +namespace mpi +{ + const int any_source = MPI_ANY_SOURCE; + const int any_tag = MPI_ANY_TAG; + + environment::environment(int argc, char**argv) + { + MPI_Init(&argc, &argv); + } + + environment::~environment() + { + MPI_Finalize(); + } + + status::status( const MPI_Status & s ) + { + _source = s.MPI_SOURCE; + _tag = s.MPI_TAG; + _error = s.MPI_ERROR; + } + + communicator::communicator( ) + { + _rank = -1; + _size = -1; + + _buf = 0; + _bufsize = -1; + } + + communicator::~communicator() + { + if( _buf ) + { + delete _buf; + _buf = 0; + } + } + + int communicator::rank() + { + if ( _rank == -1 ) + { + MPI_Comm_rank( MPI_COMM_WORLD, &_rank ); + } + return _rank; + } + + int communicator::size() + { + if ( _size == -1 ) + { + MPI_Comm_size( MPI_COMM_WORLD, &_size ); + } + return _size; + } + + /* + * SEND / RECV INT + */ + void communicator::send( int dest, int tag, int n ) + { + MPI_Send( &n, 1, MPI_INT, dest, tag, MPI_COMM_WORLD ); + } + + void communicator::recv( int src, int tag, int& n ) + { + MPI_Status stat; + MPI_Recv( &n, 1, MPI_INT, src, tag, MPI_COMM_WORLD , &stat ); + } + + /* + * SEND / RECV STRING + */ + void communicator::send( int dest, int tag, const std::string& str ) + { + int size = str.size() + 1; + send( dest, tag, size ); + MPI_Send( (char*)str.c_str(), size, MPI_CHAR, dest, tag, MPI_COMM_WORLD); + } + + void communicator::recv( int src, int tag, std::string& str ) + { + int size = -1; + MPI_Status stat; + recv( src, tag, size ); + + if( _buf == 0 ) + { + _buf = new char[ size ]; + _bufsize = size; + } else if( _bufsize < size ) + { + delete [] _buf; + _buf = new char[ size ]; + _bufsize = size; + } + MPI_Recv( _buf, size, MPI_CHAR, src, tag, MPI_COMM_WORLD, &stat ); + str.assign( _buf ); + } + + /* + * SEND / RECV Objects + */ + void communicator::send( int dest, int tag, const eoserial::Persistent & persistent ) + { + eoserial::Object* obj = persistent.pack(); + std::stringstream ss; + obj->print( ss ); + delete obj; + send( dest, tag, ss.str() ); + } + + void communicator::recv( int src, int tag, eoserial::Persistent & persistent ) + { + std::string asText; + recv( src, tag, asText ); + eoserial::Object* obj = eoserial::Parser::parse( asText ); + persistent.unpack( obj ); + delete obj; + } + + /* + * Other methods + */ + status communicator::probe( int src, int tag ) + { + MPI_Status stat; + MPI_Probe( src, tag, MPI_COMM_WORLD , &stat ); + return status( stat ); + } + + void communicator::barrier() + { + MPI_Barrier( MPI_COMM_WORLD ); + } + + void broadcast( communicator & comm, int value, int root ) + { + comm; // unused + MPI_Bcast( &value, 1, MPI_INT, root, MPI_COMM_WORLD ); + } +} diff --git a/eo/src/mpi/implMpi.h b/eo/src/mpi/implMpi.h new file mode 100644 index 000000000..b56ba2420 --- /dev/null +++ b/eo/src/mpi/implMpi.h @@ -0,0 +1,129 @@ +# ifndef __EO_MPI_HPP__ +# define __EO_MPI_HPP__ + +# include +# include + +namespace mpi +{ + extern const int any_source; + extern const int any_tag; + + class environment + { + public: + + environment(int argc, char**argv); + + ~environment(); + }; + + class status + { + public: + + status( const MPI_Status & s ); + + int tag() { return _tag; } + int error() { return _error; } + int source() { return _source; } + + private: + int _source; + int _tag; + int _error; + }; + + class communicator + { + public: + + communicator( ); + + ~communicator(); + + int rank(); + + int size(); + + /* + * SEND / RECV INT + */ + void send( int dest, int tag, int n ); + + void recv( int src, int tag, int& n ); + + /* + * SEND / RECV STRING + */ + void send( int dest, int tag, const std::string& str ); + + void recv( int src, int tag, std::string& str ); + + /* + * SEND / RECV Objects + */ + void send( int dest, int tag, const eoserial::Persistent & persistent ); + + template< class T > + void send( int dest, int tag, T* table, int size ) + { + // Puts all the values into an array + eoserial::Array* array = new eoserial::Array; + + for( int i = 0; i < size; ++i ) + { + array->push_back( table[i].pack() ); + } + + // Encapsulates the array into an object + eoserial::Object* obj = new eoserial::Object; + obj->add( "array", array ); + std::stringstream ss; + obj->print( ss ); + delete obj; + + // Sends the object as a string + send( dest, tag, ss.str() ); + } + + void recv( int src, int tag, eoserial::Persistent & persistent ); + + template< class T > + void recv( int src, int tag, T* table, int size ) + { + // Receives the string which contains the object + std::string asText; + recv( src, tag, asText ); + + // Parses the object and retrieves the table + eoserial::Object* obj = eoserial::Parser::parse( asText ); + eoserial::Array* array = static_cast( (*obj)["array"] ); + + // Retrieves all the values from the array + for( int i = 0; i < size; ++i ) + { + eoserial::unpackObject( *array, i, table[i] ); + } + delete obj; + } + + /* + * Other methods + */ + status probe( int src = any_source, int tag = any_tag ); + + void barrier(); + + private: + int _rank; + int _size; + + char* _buf; + int _bufsize; + }; + + void broadcast( communicator & comm, int value, int root ); +} // namespace mpi + +# endif //__EO_MPI_HPP__ diff --git a/eo/src/mpi/implMpi.hpp b/eo/src/mpi/implMpi.hpp deleted file mode 100644 index 4bf71dcc5..000000000 --- a/eo/src/mpi/implMpi.hpp +++ /dev/null @@ -1,283 +0,0 @@ -# ifndef __EO_MPI_HPP__ -# define __EO_MPI_HPP__ - -# include -# include - -# include - -namespace mpi -{ - const int any_source = MPI_ANY_SOURCE; - const int any_tag = MPI_ANY_TAG; - - class environment - { - public: - - environment(int argc, char**argv) - { - MPI_Init(&argc, &argv); - } - - ~environment() - { - MPI_Finalize(); - } - }; - - class status - { - public: - - status( const MPI_Status & s ) - { - _source = s.MPI_SOURCE; - _tag = s.MPI_TAG; - _error = s.MPI_ERROR; - } - - int tag() { return _tag; } - int error() { return _error; } - int source() { return _source; } - - private: - int _source; - int _tag; - int _error; - }; - - class communicator - { - public: - - communicator( ) - { - _rank = -1; - _size = -1; - - _buf = 0; - _bufsize = -1; - } - - ~communicator() - { - if( _buf ) - { - delete _buf; - _buf = 0; - } - } - - int rank() - { - if ( _rank == -1 ) - { - MPI_Comm_rank( MPI_COMM_WORLD, &_rank ); - } - return _rank; - } - - int size() - { - if ( _size == -1 ) - { - MPI_Comm_size( MPI_COMM_WORLD, &_size ); - } - return _size; - } - - /* - * SEND / RECV INT - */ - void send( int dest, int tag, int n ) - { - //send( dest, tag, &n, 1 ); - MPI_Send( &n, 1, MPI_INT, dest, tag, MPI_COMM_WORLD ); - } - - void recv( int src, int tag, int& n ) - { - MPI_Status stat; - MPI_Recv( &n, 1, MPI_INT, src, tag, MPI_COMM_WORLD , &stat ); - //recv( src, tag, &n, 1 ); - } - - /* - void send( int dest, int tag, int* table, int size ) - { - MPI_Send( table, size, MPI_INT, dest, tag, MPI_COMM_WORLD ); - } - - void recv( int src, int tag, int* table, int size ) - { - MPI_Status stat; - MPI_Recv( table, size, MPI_INT, src, tag, MPI_COMM_WORLD , &stat ); - } - */ - - /* - * SEND / RECV STRING - */ - void send( int dest, int tag, const std::string& str ) - { - int size = str.size() + 1; - send( dest, tag, size ); - MPI_Send( (char*)str.c_str(), size, MPI_CHAR, dest, tag, MPI_COMM_WORLD); - } - - void recv( int src, int tag, std::string& str ) - { - int size = -1; - MPI_Status stat; - recv( src, tag, size ); - - if( _buf == 0 ) - { - _buf = new char[ size ]; - _bufsize = size; - } else if( _bufsize < size ) - { - delete [] _buf; - _buf = new char[ size ]; - _bufsize = size; - } - MPI_Recv( _buf, size, MPI_CHAR, src, tag, MPI_COMM_WORLD, &stat ); - str.assign( _buf ); - } - - /* - * SEND / RECV Objects - */ - void send( int dest, int tag, const eoserial::Persistent & persistent ) - { - eoserial::Object* obj = persistent.pack(); - std::stringstream ss; - obj->print( ss ); - delete obj; - send( dest, tag, ss.str() ); - } - - /* - void send( int dest, int tag, eoserial::Persistent* table, int size ) - { - // Puts all the values into an array - eoserial::Array* array = new eoserial::Array; - - std::cout << "DEBUG EO: @ premier: " << table << " / @ second: " << table+1 << std::endl; - - for( int i = 0; i < size; ++i ) - { - array->push_back( table[i].pack() ); - } - - // Encapsulates the array into an object - eoserial::Object* obj = new eoserial::Object; - obj->add( "array", array ); - std::stringstream ss; - obj->print( ss ); - delete obj; - - // Sends the object as a string - send( dest, tag, ss.str() ); - } - */ - - template< class T > - void send( int dest, int tag, T* table, int size ) - { - // Puts all the values into an array - eoserial::Array* array = new eoserial::Array; - - for( int i = 0; i < size; ++i ) - { - array->push_back( table[i].pack() ); - } - - // Encapsulates the array into an object - eoserial::Object* obj = new eoserial::Object; - obj->add( "array", array ); - std::stringstream ss; - obj->print( ss ); - delete obj; - - // Sends the object as a string - send( dest, tag, ss.str() ); - } - - - void recv( int src, int tag, eoserial::Persistent & persistent ) - { - std::string asText; - recv( src, tag, asText ); - eoserial::Object* obj = eoserial::Parser::parse( asText ); - persistent.unpack( obj ); - delete obj; - } - - /* - void recv( int src, int tag, eoserial::Persistent* table, int size ) - { - // Receives the string which contains the object - std::string asText; - recv( src, tag, asText ); - - // Parses the object and retrieves the table - eoserial::Object* obj = eoserial::Parser::parse( asText ); - eoserial::Array* array = static_cast( (*obj)["array"] ); - - // Retrieves all the values from the array - for( int i = 0; i < size; ++i ) - { - eoserial::unpackObject( *array, i, table[i] ); - } - delete obj; - } - */ - - template< class T > - void recv( int src, int tag, T* table, int size ) - { - // Receives the string which contains the object - std::string asText; - recv( src, tag, asText ); - - // Parses the object and retrieves the table - eoserial::Object* obj = eoserial::Parser::parse( asText ); - eoserial::Array* array = static_cast( (*obj)["array"] ); - - // Retrieves all the values from the array - for( int i = 0; i < size; ++i ) - { - eoserial::unpackObject( *array, i, table[i] ); - } - delete obj; - } - - /* - * Other methods - */ - status probe( int src = any_source, int tag = any_tag ) - { - MPI_Status stat; - MPI_Probe( src, tag, MPI_COMM_WORLD , &stat ); - return status( stat ); - } - - void barrier() - { - MPI_Barrier( MPI_COMM_WORLD ); - } - - private: - int _rank; - int _size; - - char* _buf; - int _bufsize; - }; - - void broadcast( communicator & comm, int value, int root ); -} // namespace mpi - -# endif //__EO_MPI_HPP__ From dea09dea178056e4dabbca676cc8d3dd113e3efd Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 16:23:02 +0200 Subject: [PATCH 06/31] Removed warning at compilation of implMpi --- eo/src/mpi/implMpi.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/eo/src/mpi/implMpi.cpp b/eo/src/mpi/implMpi.cpp index ae0dd1c91..ee186c7cb 100644 --- a/eo/src/mpi/implMpi.cpp +++ b/eo/src/mpi/implMpi.cpp @@ -140,7 +140,6 @@ namespace mpi void broadcast( communicator & comm, int value, int root ) { - comm; // unused MPI_Bcast( &value, 1, MPI_INT, root, MPI_COMM_WORLD ); } } From 36136185443793d3a46f58cdf63ab53da0e24a67 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 16:36:10 +0200 Subject: [PATCH 07/31] Updated compilation system so as to remove dependance to Boost. --- eo/eo-conf.cmake | 1 - eo/src/CMakeLists.txt | 6 +----- eo/test/CMakeLists.txt | 2 -- eo/test/mpi/CMakeLists.txt | 5 +---- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/eo/eo-conf.cmake b/eo/eo-conf.cmake index 1f5ee8150..13fd720a9 100644 --- a/eo/eo-conf.cmake +++ b/eo/eo-conf.cmake @@ -13,5 +13,4 @@ SET(WITH_OMP FALSE CACHE BOOL "Use OpenMP ?" FORCE) SET(WITH_MPI FALSE CACHE BOOL "Use mpi ?" FORCE) SET(MPI_DIR "/mpi/directory" CACHE PATH "OpenMPI directory" FORCE) -SET(BOOST_DIR "/boost/directory" CACHE PATH "Boost directory" FORCE) diff --git a/eo/src/CMakeLists.txt b/eo/src/CMakeLists.txt index caeb673ad..6509908ae 100644 --- a/eo/src/CMakeLists.txt +++ b/eo/src/CMakeLists.txt @@ -9,23 +9,19 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) ###################################################################################### IF(WITH_MPI) - MESSAGE("[EO] Compilation with MPI and BoostMPI.") + MESSAGE("[EO] Compilation with MPI.") SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") # headers location INCLUDE_DIRECTORIES(${MPI_DIR}/include) - INCLUDE_DIRECTORIES(${BOOST_DIR}/include) # lib location LINK_DIRECTORIES(${MPI_DIR}/lib) - LINK_DIRECTORIES(${BOOST_DIR}/lib) # for conditional compilation in code ADD_DEFINITIONS(-DWITH_MPI) - LINK_LIBRARIES(boost_mpi boost_serialization) - ADD_SUBDIRECTORY(mpi) ENDIF() diff --git a/eo/test/CMakeLists.txt b/eo/test/CMakeLists.txt index 8bf8b500e..e3a5a624b 100644 --- a/eo/test/CMakeLists.txt +++ b/eo/test/CMakeLists.txt @@ -15,7 +15,6 @@ INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/contrib/MGE) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) IF(WITH_MPI) - INCLUDE_DIRECTORIES(${BOOST_DIR}/include) INCLUDE_DIRECTORIES(${MPI_DIR}/include) ENDIF() @@ -26,7 +25,6 @@ ENDIF() LINK_DIRECTORIES(${EO_BINARY_DIR}/lib) IF(WITH_MPI) - LINK_DIRECTORIES(${BOOST_DIR}/lib) LINK_DIRECTORIES(${MPI_DIR}/lib) SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") ENDIF() diff --git a/eo/test/mpi/CMakeLists.txt b/eo/test/mpi/CMakeLists.txt index 37b1227ff..f83ecd45d 100644 --- a/eo/test/mpi/CMakeLists.txt +++ b/eo/test/mpi/CMakeLists.txt @@ -10,10 +10,8 @@ MESSAGE("EO SOURCE DIR: ${EO_SOURCE_DIR}") MESSAGE("OMPI: ${MPI_DIR}") -MESSAGE("BOOST: ${BOOST_DIR}") INCLUDE_DIRECTORIES(${MPI_DIR}/include) -INCLUDE_DIRECTORIES(${BOOST_DIR}/include) INCLUDE_DIRECTORIES(${EO_SOURCE_DIR}/src) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) @@ -23,7 +21,6 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) LINK_DIRECTORIES(${EO_BINARY_DIR}/lib) LINK_DIRECTORIES(${MPI_DIR}/lib) -LINK_DIRECTORIES(${BOOST_DIR}/lib) ###################################################################################### ### 3) Define your targets and link the librairies @@ -47,7 +44,7 @@ IF(ENABLE_CMAKE_TESTING) FOREACH (test ${TEST_LIST}) ADD_EXECUTABLE(${test} ${T_${test}_SOURCES}) ADD_TEST(${test} ${test}) - TARGET_LINK_LIBRARIES(${test} boost_mpi boost_serialization eoutils eompi eoserial eo) + TARGET_LINK_LIBRARIES(${test} eoutils eompi eoserial eo) INSTALL(TARGETS ${test} RUNTIME DESTINATION share/eo/test COMPONENT test) ENDFOREACH (test) ENDIF() From 7c6e1f6200c5aca63b9d8f1cd5df0e8d07426f9f Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 17:01:41 +0200 Subject: [PATCH 08/31] Tutorial to install eoMpi module. --- eo/tutorial/Parallelization/INSTALL | 63 +++++++++++++++++++++++++++++ eo/tutorial/Parallelization/mpi.c | 35 ++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 eo/tutorial/Parallelization/INSTALL create mode 100644 eo/tutorial/Parallelization/mpi.c diff --git a/eo/tutorial/Parallelization/INSTALL b/eo/tutorial/Parallelization/INSTALL new file mode 100644 index 000000000..827216807 --- /dev/null +++ b/eo/tutorial/Parallelization/INSTALL @@ -0,0 +1,63 @@ +How to install EoMPI +==================== + +Install OpenMpi +--------------- + +1) Download OpenMPI on their website or with the following command: + + wget http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.tar.bz2 + +2) Untar the downloaded archive in a directory and change working directory to there: + + tar -xvjf openmpi*.tar.bz2 + cd openmpi-X.Y + +3) Use configuration script to indicate in which directory OpenMPI should be installed, and other options: + +Simplest configuration: + ./configure --prefix=/home/`whoami`/openmpi/ + +Only static libraries: + ./configure --enable-static --disable-shared + +Only static libraries, with prefix, disable build for Fortran77 and Fortran90, add support for SGE: + ./configure --enable-static --disable-shared --prefix=/home/`whoami`/openmpi/ --disable-mpi-f77 --disable-mpi-f90 --with-sge + +Other options are available in the README file. + +4) Make it and install: + +In sequential: + make all install + +Or in parallel: + make -j 2 all + make install + +5) Try to compile and run the sample program: + + ~/openmpi/bin/mpicxx -o sample mpi.c + ~/openmpi/bin/mpirun -np 3 ./sample + +Configure EO to use MPI +----------------------- + +You only need to configure eo-conf.cmake so as to use MPI : + +1) Put the WITH_MPI boolean to true: + + SET(WITH_MPI TRUE CACHE BOOL "Use mpi ?" FORCE) + +2) Indicate in which directories you have installed openmpi: + + SET(MPI_DIR "/where/did/you/install/openmpi" CACHE PATH "OpenMPI directory" FORCE) + +3) Recompile eo: + + ./distclean + ./build_gcc_linux_release.sh + +4) If you meet any issue, don't hesitate to contact the EO mailing list: + + eodev-main@lists.sourceforge.net diff --git a/eo/tutorial/Parallelization/mpi.c b/eo/tutorial/Parallelization/mpi.c new file mode 100644 index 000000000..23578437a --- /dev/null +++ b/eo/tutorial/Parallelization/mpi.c @@ -0,0 +1,35 @@ +# include +# include +# include + +int main(int argc, char **argv) +{ + int rank, size; + char someString[] = "Can haz cheezburgerz?"; + + MPI_Init(&argc, &argv); + + MPI_Comm_rank( MPI_COMM_WORLD, & rank ); + MPI_Comm_size( MPI_COMM_WORLD, & size ); + + if ( rank == 0 ) + { + int n = 42; + int i; + for( i = 1; i < size; ++i) + { + MPI_Send( &n, 1, MPI_INT, i, 0, MPI_COMM_WORLD ); + MPI_Send( &someString, strlen( someString )+1, MPI_CHAR, i, 0, MPI_COMM_WORLD ); + } + } else { + char buffer[ 128 ]; + int received; + MPI_Status stat; + MPI_Recv( &received, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &stat ); + printf( "[Worker] Number : %d\n", received ); + MPI_Recv( buffer, 128, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &stat ); + printf( "[Worker] String : %s\n", buffer ); + } + + MPI_Finalize(); +} From 09af6127498fac31cba54006d9718736b8f0a618 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 17:12:18 +0200 Subject: [PATCH 09/31] Updated examples' comments. --- eo/test/mpi/t-mpi-common.h | 8 +++++++ eo/test/mpi/t-mpi-eval.cpp | 37 +---------------------------- eo/test/mpi/t-mpi-multipleRoles.cpp | 11 ++++++--- eo/test/mpi/t-mpi-parallelApply.cpp | 5 +++- 4 files changed, 21 insertions(+), 40 deletions(-) diff --git a/eo/test/mpi/t-mpi-common.h b/eo/test/mpi/t-mpi-common.h index 4217af28b..c862a05c1 100644 --- a/eo/test/mpi/t-mpi-common.h +++ b/eo/test/mpi/t-mpi-common.h @@ -3,6 +3,14 @@ #include +/** + * @file t-mpi-common.h + * + * This file shows an example of serialization of a primitive type, so as to be used in a parallel algorithm. + * It is fully compatible with the basic type, by implementing conversion operator and constructor based on type. + * It can contain any simple type which can be written in a std::ostream. + */ + template< class T > struct SerializableBase : public eoserial::Persistent { diff --git a/eo/test/mpi/t-mpi-eval.cpp b/eo/test/mpi/t-mpi-eval.cpp index 4187c4e63..fb97701d2 100644 --- a/eo/test/mpi/t-mpi-eval.cpp +++ b/eo/test/mpi/t-mpi-eval.cpp @@ -34,8 +34,6 @@ Authors: #include -// #include - #include using namespace std; @@ -68,6 +66,7 @@ class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial void unpack( const eoserial::Object* obj ) { + this->clear(); eoserial::unpackArray< vector, eoserial::Array::UnpackAlgorithm > ( *obj, "array", *this ); @@ -81,40 +80,6 @@ class eoRealSerializable : public eoReal< eoMinimizingFitness >, public eoserial fitness( fitnessVal ); } } - - /* - // Gives access to boost serialization - friend class boost::serialization::access; - - - template - void save( Archive & ar, const unsigned int version ) const - { - std::stringstream ss; - printOn( ss ); - std::string asStr = ss.str(); - ar & asStr; - - (void) version; // avoid compilation warning - } - - - template - void load( Archive & ar, const unsigned int version ) - { - std::string asStr; - ar & asStr; - std::stringstream ss; - ss << asStr; - readFrom( ss ); - - (void) version; // avoid compilation warning - } - - // Indicates that boost save and load operations are not the same. - BOOST_SERIALIZATION_SPLIT_MEMBER() - */ - }; typedef eoRealSerializable EOT; diff --git a/eo/test/mpi/t-mpi-multipleRoles.cpp b/eo/test/mpi/t-mpi-multipleRoles.cpp index db4b1bb6c..07f65937e 100644 --- a/eo/test/mpi/t-mpi-multipleRoles.cpp +++ b/eo/test/mpi/t-mpi-multipleRoles.cpp @@ -43,8 +43,6 @@ Authors: # include "t-mpi-common.h" -// # include - # include # include @@ -52,6 +50,13 @@ using namespace std; using namespace eo::mpi; +/* + * This class allows the user to easily serialize a vector of elements which implement eoserial::Persistent too. + * + * T is the type of the contained element, which must implement eoserial::Persistent too. + * + * Here, it contains SerializableBase, which is a serializable integer that can be used as an integer. + */ template< class T > struct SerializableVector : public std::vector, public eoserial::Persistent { @@ -125,7 +130,7 @@ int main(int argc, char** argv) // eo::log << eo::setlevel( eo::debug ); Node::init( argc, argv ); if( Node::comm().size() != 7 ) { - // throw std::runtime_error("World size should be 7."); + throw std::runtime_error("World size should be 7."); } SerializableVector< SerializableBase > v; diff --git a/eo/test/mpi/t-mpi-parallelApply.cpp b/eo/test/mpi/t-mpi-parallelApply.cpp index df90f0f56..04985b3f7 100644 --- a/eo/test/mpi/t-mpi-parallelApply.cpp +++ b/eo/test/mpi/t-mpi-parallelApply.cpp @@ -25,6 +25,9 @@ Authors: * incremented... in a parallel fashion. While this operation is very easy to perform even on a single host, it's just * an example for parallel apply use. * + * The table of integers has to be serialized before it's sent. The wrapper object SerializableBase allows to serialize + * any type and manipulate it like this type: SerializableBase can be exactly be used as an integer. + * * Besides, this is also a test for assignment (scheduling) algorithms, in different cases. The test succeeds if and * only if the program terminates without any segfault ; otherwise, there could be a deadlock which prevents the end or * a segfault at any time. @@ -56,7 +59,7 @@ struct plusOne : public eoUF< SerializableBase&, void > { void operator() ( SerializableBase & x ) { - ++x; + ++x; // implicit conversion of SerializableBase in the integer it contains } }; From 8b1d376557d28cee1cbd116eba2da473201af318 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 17:38:54 +0200 Subject: [PATCH 10/31] Added comments and license for implMpi --- eo/src/mpi/implMpi.h | 203 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 198 insertions(+), 5 deletions(-) diff --git a/eo/src/mpi/implMpi.h b/eo/src/mpi/implMpi.h index b56ba2420..b7c1cd4ba 100644 --- a/eo/src/mpi/implMpi.h +++ b/eo/src/mpi/implMpi.h @@ -1,31 +1,115 @@ -# ifndef __EO_MPI_HPP__ -# define __EO_MPI_HPP__ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ +# ifndef __EO_IMPL_MPI_HPP__ +# define __EO_IMPL_MPI_HPP__ # include # include +/** + * This namespace contains reimplementations of some parts of the Boost::MPI API in C++, so as to be used in + * EO without any dependance to Boost. Historically, EO's parallelization module used the + * boost library to add a layer over MPI. After having noticed that just some functions + * were really used, we decided to reimplement our own C++-like implementation of MPI. + * + * Because the Boost::MPI API is really clean, we reused it in this module. However, all + * the functions of Boost::MPI were not used, hence a subset of the API is reused. For + * instance, users can just send integer, std::string or eoserial::Persistent objects; + * furthermore, only eoserial::Persistent objects can sent in a table. + * + * The documentation of the functions is exactly the same as the official Boost::MPI + * documentation. You can find it on www.boost.org/doc/libs/1_49_0/doc/html/mpi/ + * The entities are here shortly described, if you need further details, don't hesitate + * to visit the boost URL. + */ + namespace mpi { + /** + * @ingroup Parallel + * @{ + */ + + /** + * @brief Constant indicating that a message can come from any process. + */ extern const int any_source; + + /** + * @brief Constant indicating that a message can come from any tag (channel). + */ extern const int any_tag; + /** + * @brief Wrapper class to have a MPI environment. + * + * Instead of calling MPI_Init and MPI_Finalize, it is only necessary to instantiate + * this class once, in the global context. + */ class environment { public: + /** + * @brief Inits MPI context. + * + * @param argc Number of params in command line (same as one in main) + * @param argv Strings containing params (same as one in main) + */ environment(int argc, char**argv); + /** + * @brief Closes MPI context. + */ ~environment(); }; + /** + * @brief Wrapper class for MPI_Status + * + * Consists only in a C++ wrapper class, giving getters on status attributes. + */ class status { public: + /** + * @brief Converts a MPI_Status into a status. + */ status( const MPI_Status & s ); + /** + * @brief Returns the tag of the associated communication. + */ int tag() { return _tag; } + + /** + * @brief Indicates which error number we obtained in the associated communication. + */ int error() { return _error; } + + /** + * @brief Returns the MPI rank of the source of the associated communication. + */ int source() { return _source; } private: @@ -34,37 +118,99 @@ namespace mpi int _error; }; + /** + * @brief Main object, used to send / receive messages, get informations about the rank and the size of the world, + * etc. + */ class communicator { public: + /** + * Creates the communicator, using the whole world as a MPI_Comm. + * + * @todo Allow the user to precise which MPI_Comm to use + */ communicator( ); ~communicator(); + /** + * @brief Returns the MPI rank of the current process. + */ int rank(); + /** + * @brief Returns the size of the MPI cluster. + */ int size(); /* * SEND / RECV INT */ + + /** + * @brief Sends an integer to dest on channel "tag". + * + * @param dest MPI rank of the receiver + * @param tag MPI tag of message + * @param n The integer to send + */ void send( int dest, int tag, int n ); + /* + * @brief Receives an integer from src on channel "tag". + * + * @param src MPI rank of the sender + * @param tag MPI tag of message + * @param n Where to save the received integer + */ void recv( int src, int tag, int& n ); /* * SEND / RECV STRING */ + + /** + * @brief Sends a string to dest on channel "tag". + * + * @param dest MPI rank of the receiver + * @param tag MPI tag of message + * @param str The std::string to send + */ void send( int dest, int tag, const std::string& str ); + /* + * @brief Receives a string from src on channel "tag". + * + * @param src MPI rank of the sender + * @param tag MPI tag of message + * @param std::string Where to save the received string + */ void recv( int src, int tag, std::string& str ); /* * SEND / RECV Objects */ + + /** + * @brief Sends an eoserial::Persistent to dest on channel "tag". + * + * @param dest MPI rank of the receiver + * @param tag MPI tag of message + * @param persistent The object to send (it must absolutely implement eoserial::Persistent) + */ void send( int dest, int tag, const eoserial::Persistent & persistent ); + /** + * @brief Sends an array of eoserial::Persistent to dest on channel "tag". + * + * @param dest MPI rank of the receiver + * @param tag MPI tag of message + * @param table The array of eoserial::Persistent objects + * @param size The number of elements to send (no check is done, the user has to be sure that the size won't + * overflow!) + */ template< class T > void send( int dest, int tag, T* table, int size ) { @@ -87,8 +233,25 @@ namespace mpi send( dest, tag, ss.str() ); } + /* + * @brief Receives an eoserial::Persistent object from src on channel "tag". + * + * @param src MPI rank of the sender + * @param tag MPI tag of message + * @param persistent Where to unpack the serialized object? + */ void recv( int src, int tag, eoserial::Persistent & persistent ); + /* + * @brief Receives an array of eoserial::Persistent from src on channel "tag". + * + * @param src MPI rank of the sender + * @param tag MPI tag of message + * @param table The table in which we're saving the received objects. It must have been allocated by the user, + * as no allocation is performed here. + * @param size The number of elements to receive (no check is done, the user has to be sure that the size won't + * overflow!) + */ template< class T > void recv( int src, int tag, T* table, int size ) { @@ -111,19 +274,49 @@ namespace mpi /* * Other methods */ + + /** + * @brief Wrapper for MPI_Probe + * + * Waits for a message to come from process having rank src, on the channel + * tag. + * + * @param src MPI rank of the sender (any_source if it can be any sender) + * @param tag MPI tag of the expected message (any_tag if it can be any tag) + */ status probe( int src = any_source, int tag = any_tag ); + /** + * @brief Wrapper for MPI_Barrier + * + * + */ void barrier(); private: int _rank; int _size; - char* _buf; - int _bufsize; + char* _buf; // temporary buffer for sending and receiving strings. Avoids reallocations + int _bufsize; // size of the above temporary buffer }; + /** + * @brief Wrapper for MPI_Bcast + * + * Broadcasts an integer value on the communicator comm, from the process having the MPI rank root. + * + * @param comm The communicator on which to broadcast + * @param value The integer value to send + * @param root The MPI rank of the broadcaster + * + * @todo Actually comm isn't used and broadcast is performed on the whole MPI_COMM_WORLD. TODO: Use comm instead + */ void broadcast( communicator & comm, int value, int root ); + + /** + * @} + */ } // namespace mpi -# endif //__EO_MPI_HPP__ +# endif //__EO_IMPL_MPI_HPP__ From 72e26513f62e8417570e64aa0828754fbd4b07c6 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 17:39:24 +0200 Subject: [PATCH 11/31] Added licences for MPI cpp files --- eo/src/mpi/eoMpi.cpp | 21 +++++++++++++++++++++ eo/src/mpi/eoMpiAssignmentAlgorithm.cpp | 20 ++++++++++++++++++++ eo/src/mpi/eoMpiNode.cpp | 21 +++++++++++++++++++++ eo/src/mpi/implMpi.cpp | 21 +++++++++++++++++++++ 4 files changed, 83 insertions(+) diff --git a/eo/src/mpi/eoMpi.cpp b/eo/src/mpi/eoMpi.cpp index 56422c13d..cceb403b5 100644 --- a/eo/src/mpi/eoMpi.cpp +++ b/eo/src/mpi/eoMpi.cpp @@ -1,3 +1,24 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ # include "eoMpi.h" namespace eo diff --git a/eo/src/mpi/eoMpiAssignmentAlgorithm.cpp b/eo/src/mpi/eoMpiAssignmentAlgorithm.cpp index e51fc2b51..3e7ff0251 100644 --- a/eo/src/mpi/eoMpiAssignmentAlgorithm.cpp +++ b/eo/src/mpi/eoMpiAssignmentAlgorithm.cpp @@ -1,5 +1,25 @@ # include "eoMpiAssignmentAlgorithm.h" +/* +(c) Thales group, 2012 + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ # include "eoMpiNode.h" namespace eo diff --git a/eo/src/mpi/eoMpiNode.cpp b/eo/src/mpi/eoMpiNode.cpp index 22e88da04..69f06576b 100644 --- a/eo/src/mpi/eoMpiNode.cpp +++ b/eo/src/mpi/eoMpiNode.cpp @@ -1,3 +1,24 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ # include "eoMpiNode.h" namespace eo diff --git a/eo/src/mpi/implMpi.cpp b/eo/src/mpi/implMpi.cpp index ee186c7cb..6ae0c0996 100644 --- a/eo/src/mpi/implMpi.cpp +++ b/eo/src/mpi/implMpi.cpp @@ -1,3 +1,24 @@ +/* +(c) Thales group, 2012 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; + version 2 of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +Contact: http://eodev.sourceforge.net + +Authors: + Benjamin Bouvier +*/ # include "implMpi.h" namespace mpi From 66d56bd1d89b798b982f44db18282c834aaaf012 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 18:03:50 +0200 Subject: [PATCH 12/31] Serialized eoTimer with eoserial module. --- eo/src/utils/eoTimer.h | 78 ++++++++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/eo/src/utils/eoTimer.h b/eo/src/utils/eoTimer.h index 211313778..4ff451b48 100644 --- a/eo/src/utils/eoTimer.h +++ b/eo/src/utils/eoTimer.h @@ -30,12 +30,7 @@ Authors: # include "utils/eoParallel.h" // eo::parallel -# ifdef WITH_MPI -// For serialization purposes -# include -# include -# include -# endif +# include "serial/eoSerial.h" // eo::Persistent /** * @brief Timer allowing to measure time between a start point and a stop point. @@ -202,6 +197,9 @@ class eoTimer * @ingroup Utilities */ class eoTimerStat +# ifdef WITH_MPI + : public eoserial::Persistent +# endif { public: @@ -215,41 +213,63 @@ class eoTimerStat * It can readily be serialized with boost when compiling with mpi. */ struct Stat +# ifdef WITH_MPI + : public eoserial::Persistent +# endif { std::vector utime; std::vector stime; std::vector wtime; #ifdef WITH_MPI - // Gives access to boost serialization - friend class boost::serialization::access; + void unpack( const eoserial::Object* obj ) + { + utime.clear(); + static_cast< eoserial::Array* >(obj->find("utime")->second) + ->deserialize< std::vector, eoserial::Array::UnpackAlgorithm >( utime ); - /** - * Serializes the single statistic in a boost archive (useful for boost::mpi). - * Just serializes the 3 vectors. - */ - template - void serialize( Archive & ar, const unsigned int version ) - { - ar & utime & stime & wtime; - (void) version; // avoid compilation warning - } + stime.clear(); + static_cast< eoserial::Array* >(obj->find("stime")->second) + ->deserialize< std::vector, eoserial::Array::UnpackAlgorithm >( stime ); + + wtime.clear(); + static_cast< eoserial::Array* >(obj->find("wtime")->second) + ->deserialize< std::vector, eoserial::Array::UnpackAlgorithm >( wtime ); + } + + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add("utime", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( utime ) ); + obj->add("stime", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( stime ) ); + obj->add("wtime", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( wtime ) ); + return obj; + } # endif }; #ifdef WITH_MPI - // Gives access to boost serialization - friend class boost::serialization::access; - - /** - * Serializes the timerStat object in a boost archive (useful for boost::mpi). - * Just serializes the map. - */ - template - void serialize( Archive & ar, const unsigned int version ) + void unpack( const eoserial::Object* obj ) + { + _stats.clear(); + for( eoserial::Object::const_iterator it = obj->begin(), final = obj->end(); + it != final; + ++it) { - ar & _stats; - (void) version; // avoid compilation warning + eoserial::unpackObject( *obj, it->first, _stats[ it->first ] ); } + } + + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + for( std::map::const_iterator it = _stats.begin(), final = _stats.end(); + it != final; + ++it) + { + obj->add( it->first, it->second.pack() ); + } + return obj; + } # endif /** From e48782f22d2ad5c7a5f21defdd92ea25b2cb896d Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Tue, 24 Jul 2012 18:22:35 +0200 Subject: [PATCH 13/31] Added a template for eo::mpi::Job in test/mpi directory. --- eo/test/mpi/template-job.cpp | 104 +++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 eo/test/mpi/template-job.cpp diff --git a/eo/test/mpi/template-job.cpp b/eo/test/mpi/template-job.cpp new file mode 100644 index 000000000..3d25e9736 --- /dev/null +++ b/eo/test/mpi/template-job.cpp @@ -0,0 +1,104 @@ +# include + +using namespace eo::mpi; + +/* + * This file is a template for a new eo::mpi::Job. You have everything that should be necessary to implement a new + * parallelized algorithm. + * + * Replace __TEMPLATE__ by the name of your algorithm (for instance: MultiStart, ParallelApply, etc.). + */ + +template< class EOT > +struct __TEMPLATE__Data +{ + +}; + +template< class EOT > +class SendTask__TEMPLATE__ : public SendTaskFunction< __TEMPLATE__Data< EOT > > +{ + public: + + using SendTaskFunction< __TEMPLATE__Data< EOT > >::_data; + + void operator()( int wrkRank ) + { + // TODO implement me + } +}; + +template< class EOT > +class HandleResponse__TEMPLATE__ : public HandleResponseFunction< __TEMPLATE__Data< EOT > > +{ + public: + + using HandleResponseFunction< __TEMPLATE__Data< EOT > >::_data; + + void operator()( int wrkRank ) + { + // TODO implement me + } +}; + +template< class EOT > +class ProcessTask__TEMPLATE__ : public ProcessTaskFunction< __TEMPLATE__Data< EOT > > +{ + public: + using ProcessTaskFunction< __TEMPLATE__Data >::_data; + + void operator()() + { + // TODO implement me + } +}; + +template< class EOT > +class IsFinished__TEMPLATE__ : public IsFinishedFunction< __TEMPLATE__Data< EOT > > +{ + public: + + using IsFinishedFunction< __TEMPLATE__Data< EOT > >::_data; + + bool operator()() + { + // TODO implement me + } +}; + +template< class EOT > +class __TEMPLATE__Store : public JobStore< __TEMPLATE__Data< EOT > > +{ + public: + + __TEMPLATE__Data* data() + { + // TODO implement me + return 0; + } +}; + +template< class EOT > +class __TEMPLATE__ : public MultiJob< __TEMPLATE__Data< EOT > > +{ + public: + + __TEMPLATE__( AssignmentAlgorithm & algo, + int masterRank, + __TEMPLATE__Store< EOT > & store ) : + MultiJob< __TEMPLATE__Data< EOT > >( algo, masterRank, store ) + { + // TODO implement me + } +}; + +/* +int main(int argc, char **argv) +{ + Node::init( argc, argv ); + + DynamicAssignmentAlgorithm assignmentAlgo; + __TEMPLATE__Store store; + __TEMPLATE__ job( assignmentAlgo, DEFAULT_MASTER, store ); +} +*/ From 73af5d4f7b4d22ff2f430e2f83be5bfc65dc269b Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 25 Jul 2012 14:55:45 +0200 Subject: [PATCH 14/31] Fixed compilation bug (use of stdlib constantes) in MPI tests, when cmaking locally. --- eo/test/mpi/t-mpi-parallelApply.cpp | 1 + eo/test/mpi/t-mpi-wrapper.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/eo/test/mpi/t-mpi-parallelApply.cpp b/eo/test/mpi/t-mpi-parallelApply.cpp index 04985b3f7..6cda89907 100644 --- a/eo/test/mpi/t-mpi-parallelApply.cpp +++ b/eo/test/mpi/t-mpi-parallelApply.cpp @@ -46,6 +46,7 @@ Authors: # include "t-mpi-common.h" # include +# include # include using namespace std; diff --git a/eo/test/mpi/t-mpi-wrapper.cpp b/eo/test/mpi/t-mpi-wrapper.cpp index 7da45ad4e..cedc717fe 100644 --- a/eo/test/mpi/t-mpi-wrapper.cpp +++ b/eo/test/mpi/t-mpi-wrapper.cpp @@ -36,6 +36,7 @@ Authors: # include "t-mpi-common.h" # include +# include # include using namespace std; From 355541ae8843d6c814978e2eb67f02e2ec6ac943 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 25 Jul 2012 14:56:02 +0200 Subject: [PATCH 15/31] MPI example: Multistart. --- eo/test/mpi/t-mpi-multistart.cpp | 384 +++++++++++++++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 eo/test/mpi/t-mpi-multistart.cpp diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp new file mode 100644 index 000000000..451bd4d37 --- /dev/null +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -0,0 +1,384 @@ +# include +using namespace eo::mpi; + +#include + +/***************************** EASY PSO STUFF ********************************/ +//----------------------------------------------------------------------------- +typedef eoMinimizingFitness ParticleFitness; +//----------------------------------------------------------------------------- +class SerializableParticle : public eoRealParticle< ParticleFitness >, public eoserial::Persistent +{ + public: + + SerializableParticle(unsigned size = 0, double positions = 0.0,double velocities = 0.0,double bestPositions = 0.0): eoRealParticle< ParticleFitness > (size, positions,velocities,bestPositions) {} + + void unpack( const eoserial::Object* obj ) + { + this->clear(); + eoserial::unpackArray + < std::vector, eoserial::Array::UnpackAlgorithm > + ( *obj, "vector", *this ); + + this->bestPositions.clear(); + eoserial::unpackArray + < std::vector, eoserial::Array::UnpackAlgorithm > + ( *obj, "best_positions", this->bestPositions ); + + this->velocities.clear(); + eoserial::unpackArray + < std::vector, eoserial::Array::UnpackAlgorithm > + ( *obj, "velocities", this->velocities ); + + bool invalidFitness; + eoserial::unpack( *obj, "invalid_fitness", invalidFitness ); + if( invalidFitness ) + { + this->invalidate(); + } else + { + ParticleFitness f; + eoserial::unpack( *obj, "fitness", f ); + this->fitness( f ); + } + } + + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "vector", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( *this ) ); + obj->add( "best_positions", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( this->bestPositions ) ); + obj->add( "velocities", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm>( this->velocities ) ); + + bool invalidFitness = this->invalid(); + obj->add( "invalid_fitness", eoserial::make( invalidFitness ) ); + if( !invalidFitness ) + { + obj->add( "fitness", eoserial::make( this->fitness() ) ); + } + + return obj; + } + +}; +typedef SerializableParticle Particle; +//----------------------------------------------------------------------------- + +// the objective function +double real_value (const Particle & _particle) +{ + double sum = 0; + for (unsigned i = 0; i < _particle.size ()-1; i++) + sum += pow(_particle[i],2); + return (sum); +} + +/************************** PARALLELIZATION JOB *******************************/ +/* + * This file is a template for a new eo::mpi::Job. You have everything that should be necessary to implement a new + * parallelized algorithm. + * + * Replace MultiStart by the name of your algorithm (for instance: MultiStart, ParallelApply, etc.). + */ + +template< class T > +struct SerializableBasicType : public eoserial::Persistent +{ + public: + SerializableBasicType( T & value ) + { + _value = value; + } + + operator T&() + { + return _value; + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpack( *obj, "value", _value ); + } + + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "value", eoserial::make( _value ) ); + } + + private: + T _value; +}; + +template< class EOT, class FitT > +struct MultiStartData +{ + MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, const eoPop< EOT >& _originalPop, int _masterRank ) + : + runs( 0 ), firstIndividual( true ), bestFitness(), pop(), + comm( _comm ), algo( _algo ), originalPop( _originalPop ), masterRank( _masterRank ) + { + // empty + } + + // dynamic parameters + int runs; + bool firstIndividual; + FitT bestFitness; + EOT bestIndividual; + eoPop< EOT > pop; + + // static parameters + mpi::communicator& comm; + eoAlgo& algo; + const eoPop< EOT >& originalPop; + int masterRank; +}; + +template< class EOT, class FitT > +class SendTaskMultiStart : public SendTaskFunction< MultiStartData< EOT, FitT > > +{ + public: + using SendTaskFunction< MultiStartData< EOT, FitT > >::_data; + + void operator()( int wrkRank ) + { + --(_data->runs); + } +}; + +template< class EOT, class FitT > +class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< EOT, FitT > > +{ + public: + using HandleResponseFunction< MultiStartData< EOT, FitT > >::_data; + + void operator()( int wrkRank ) + { + std::cout << "Response received." << std::endl; + + EOT individual; + MultiStartData< EOT, FitT >& d = *_data; + d.comm.recv( wrkRank, 1, individual ); + + std::cout << "Fitness of individual: " << individual.fitness() << std::endl; + if ( ! d.firstIndividual ) { + std::cout << "Best fitness: " << d.bestFitness << std::endl; + } + + if( d.firstIndividual || individual.fitness() > d.bestFitness ) + { + d.bestFitness = individual.fitness(); + d.bestIndividual = individual; + d.firstIndividual = false; + } + } +}; + +template< class EOT, class FitT > +class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT, FitT > > +{ + public: + using ProcessTaskFunction< MultiStartData >::_data; + + void operator()() + { + _data->algo( _data->pop ); + _data->comm.send( _data->masterRank, 1, _data->pop.best_element() ); + } +}; + +template< class EOT, class FitT > +class IsFinishedMultiStart : public IsFinishedFunction< MultiStartData< EOT, FitT > > +{ + public: + using IsFinishedFunction< MultiStartData< EOT, FitT > >::_data; + + bool operator()() + { + return _data->runs <= 0; + } +}; + +template< class EOT, class FitT > +class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > +{ + public: + + MultiStartStore( eoAlgo & algo, const eoPop< EOT >& pop, int masterRank ) + : _data( Node::comm(), algo, pop, masterRank ) + { + this->_iff = new IsFinishedMultiStart< EOT, FitT >; + this->_iff->needDelete(true); + this->_stf = new SendTaskMultiStart< EOT, FitT >; + this->_stf->needDelete(true); + this->_hrf = new HandleResponseMultiStart< EOT, FitT >; + this->_hrf->needDelete(true); + this->_ptf = new ProcessTaskMultiStart< EOT, FitT >; + this->_ptf->needDelete(true); + } + + void init( int runs ) + { + _data.runs = runs; + _data.pop = _data.originalPop; // FIXME ? + _data.firstIndividual = true; + } + + MultiStartData* data() + { + return &_data; + } + + private: + MultiStartData< EOT, FitT > _data; +}; + +template< class EOT, class FitT > +class MultiStart : public MultiJob< MultiStartData< EOT, FitT > > +{ + public: + + MultiStart( AssignmentAlgorithm & algo, + int masterRank, + MultiStartStore< EOT, FitT > & store, + // dynamic parameters + int runs, + const std::vector& seeds = std::vector() ) : + MultiJob< MultiStartData< EOT, FitT > >( algo, masterRank, store ) + { + store.init( runs ); + + if( this->isMaster() ) + { + int nbWorkers = algo.availableWorkers(); + std::vector realSeeds = seeds; + if( realSeeds.size() < nbWorkers ) + { + // TODO + // get multiples of the current seed? + // generate seeds? + for( int i = 1; realSeeds.size() < nbWorkers ; ++i ) + { + realSeeds.push_back( i ); + } + } + + std::vector idles = algo.idles(); + for( int i = 0 ; i < nbWorkers ; ++i ) + { + int wrkRank = idles[i]; + Node::comm().send( wrkRank, 1, realSeeds[ i ] ); + } + } else + { + int seed; + Node::comm().recv( masterRank, 1, seed ); + std::cout << Node::comm().rank() << "- Seed: " << seed << std::endl; + eo::rng.reseed( seed ); + } + } + + EOT& best_individual() + { + return this->store.data()->bestIndividual; + } + + FitT best_fitness() + { + return this->store.data()->bestFitness; + } +}; + +int main(int argc, char **argv) +{ + Node::init( argc, argv ); + + const unsigned int VEC_SIZE = 2; + const unsigned int POP_SIZE = 20; + const unsigned int NEIGHBORHOOD_SIZE= 5; + unsigned i; + + eo::rng.reseed(1); + + // the population: + eoPop pop; + + // Evaluation + eoEvalFuncPtr eval( real_value ); + + // position init + eoUniformGenerator < double >uGen (-3, 3); + eoInitFixedLength < Particle > random (VEC_SIZE, uGen); + + // velocity init + eoUniformGenerator < double >sGen (-2, 2); + eoVelocityInitFixedLength < Particle > veloRandom (VEC_SIZE, sGen); + + // local best init + eoFirstIsBestInit < Particle > localInit; + + // perform position initialization + pop.append (POP_SIZE, random); + + // topology + eoLinearTopology topology(NEIGHBORHOOD_SIZE); + + // the full initializer + eoInitializer init(eval,veloRandom,localInit,topology,pop); + init(); + + // bounds + eoRealVectorBounds bnds(VEC_SIZE,-1.5,1.5); + + // velocity + eoStandardVelocity velocity (topology,1,1.6,2,bnds); + + // flight + eoStandardFlight flight; + + // Terminators + eoGenContinue genCont1 (50); + eoGenContinue genCont2 (50); + + // PS flight + eoEasyPSO pso1(genCont1, eval, velocity, flight); + + // eoEasyPSO pso2(init,genCont2, eval, velocity, flight); + + DynamicAssignmentAlgorithm assignmentAlgo; + MultiStartStore< Particle, ParticleFitness > store( pso1, pop, DEFAULT_MASTER ); + + MultiStart< Particle, ParticleFitness > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); + msjob.run(); + + if( msjob.isMaster() ) + { + eo::mpi::EmptyJob tjob( assignmentAlgo, DEFAULT_MASTER ); + std::cout << "Global best individual has fitness " << msjob.best_fitness() << std::endl; + } + + // flight + /* + try + { + pso1(pop); + std::cout << "FINAL POPULATION AFTER PSO n°1:" << std::endl; + for (i = 0; i < pop.size(); ++i) + std::cout << "\t" << pop[i] << " " << pop[i].fitness() << std::endl; + + pso2(pop); + std::cout << "FINAL POPULATION AFTER PSO n°2:" << std::endl; + for (i = 0; i < pop.size(); ++i) + std::cout << "\t" << pop[i] << " " << pop[i].fitness() << std::endl; + } + catch (std::exception& e) + { + std::cout << "exception: " << e.what() << std::endl;; + exit(EXIT_FAILURE); + } + */ + + return 0; + +} From 74bdb0fa916dd7585471252bbdc36b2a0a703fcb Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 25 Jul 2012 15:24:25 +0200 Subject: [PATCH 16/31] Multistart: added possibility to reinit population on each MultiStartJob, or use the same pop. --- eo/test/mpi/t-mpi-multistart.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index 451bd4d37..23d3d962d 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -113,10 +113,10 @@ struct SerializableBasicType : public eoserial::Persistent template< class EOT, class FitT > struct MultiStartData { - MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, const eoPop< EOT >& _originalPop, int _masterRank ) + MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, int _masterRank, eoInit* _init = 0 ) : runs( 0 ), firstIndividual( true ), bestFitness(), pop(), - comm( _comm ), algo( _algo ), originalPop( _originalPop ), masterRank( _masterRank ) + comm( _comm ), algo( _algo ), masterRank( _masterRank ), init( _init ) { // empty } @@ -131,7 +131,7 @@ struct MultiStartData // static parameters mpi::communicator& comm; eoAlgo& algo; - const eoPop< EOT >& originalPop; + eoInit* init; int masterRank; }; @@ -205,8 +205,10 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > { public: - MultiStartStore( eoAlgo & algo, const eoPop< EOT >& pop, int masterRank ) - : _data( Node::comm(), algo, pop, masterRank ) + MultiStartStore( eoAlgo & algo, int masterRank, const eoPop< EOT > & pop, eoInit* init = 0 ) + : _data( Node::comm(), algo, masterRank, init ), + _pop( pop ), + _firstPopInit( true ) { this->_iff = new IsFinishedMultiStart< EOT, FitT >; this->_iff->needDelete(true); @@ -221,7 +223,16 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > void init( int runs ) { _data.runs = runs; - _data.pop = _data.originalPop; // FIXME ? + + if( _data.init ) + { + _data.pop = eoPop( _pop.size(), *_data.init ); + } else if( _firstPopInit ) + { + _data.pop = _pop; + } + _firstPopInit = false; + _data.firstIndividual = true; } @@ -232,6 +243,8 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > private: MultiStartData< EOT, FitT > _data; + const eoPop< EOT >& _pop; + bool _firstPopInit; }; template< class EOT, class FitT > @@ -347,7 +360,7 @@ int main(int argc, char **argv) // eoEasyPSO pso2(init,genCont2, eval, velocity, flight); DynamicAssignmentAlgorithm assignmentAlgo; - MultiStartStore< Particle, ParticleFitness > store( pso1, pop, DEFAULT_MASTER ); + MultiStartStore< Particle, ParticleFitness > store( pso1, DEFAULT_MASTER, pop ); MultiStart< Particle, ParticleFitness > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); msjob.run(); From 0c1fc2ce99e63e8e77043c686a8683f840906f34 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 11:58:42 +0200 Subject: [PATCH 17/31] MPI MultiStart: using SGA as example and functors for seed generation, reinitialization of pop, algorithm reset. --- eo/test/mpi/t-mpi-multistart.cpp | 470 ++++++++++++++++++++----------- 1 file changed, 298 insertions(+), 172 deletions(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index 23d3d962d..dc070af6c 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -1,76 +1,79 @@ # include using namespace eo::mpi; +#include +#include +#include + #include +#include -/***************************** EASY PSO STUFF ********************************/ -//----------------------------------------------------------------------------- -typedef eoMinimizingFitness ParticleFitness; -//----------------------------------------------------------------------------- -class SerializableParticle : public eoRealParticle< ParticleFitness >, public eoserial::Persistent +// Use functions from namespace std +using namespace std; + + +class SerializableEOReal: public eoReal, public eoserial::Persistent { - public: +public: - SerializableParticle(unsigned size = 0, double positions = 0.0,double velocities = 0.0,double bestPositions = 0.0): eoRealParticle< ParticleFitness > (size, positions,velocities,bestPositions) {} + SerializableEOReal(unsigned size = 0, double value = 0.0) : + eoReal(size, value) + { + } - void unpack( const eoserial::Object* obj ) + void unpack( const eoserial::Object* obj ) + { + this->clear(); + eoserial::unpackArray + < std::vector, eoserial::Array::UnpackAlgorithm > + ( *obj, "vector", *this ); + + bool invalidFitness; + eoserial::unpack( *obj, "invalid_fitness", invalidFitness ); + if( invalidFitness ) { - this->clear(); - eoserial::unpackArray - < std::vector, eoserial::Array::UnpackAlgorithm > - ( *obj, "vector", *this ); + this->invalidate(); + } else + { + double f; + eoserial::unpack( *obj, "fitness", f ); + this->fitness( f ); + } + } - this->bestPositions.clear(); - eoserial::unpackArray - < std::vector, eoserial::Array::UnpackAlgorithm > - ( *obj, "best_positions", this->bestPositions ); + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "vector", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( *this ) ); - this->velocities.clear(); - eoserial::unpackArray - < std::vector, eoserial::Array::UnpackAlgorithm > - ( *obj, "velocities", this->velocities ); - - bool invalidFitness; - eoserial::unpack( *obj, "invalid_fitness", invalidFitness ); - if( invalidFitness ) - { - this->invalidate(); - } else - { - ParticleFitness f; - eoserial::unpack( *obj, "fitness", f ); - this->fitness( f ); - } - } - - eoserial::Object* pack( void ) const - { - eoserial::Object* obj = new eoserial::Object; - obj->add( "vector", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( *this ) ); - obj->add( "best_positions", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm >( this->bestPositions ) ); - obj->add( "velocities", eoserial::makeArray< std::vector, eoserial::MakeAlgorithm>( this->velocities ) ); - - bool invalidFitness = this->invalid(); - obj->add( "invalid_fitness", eoserial::make( invalidFitness ) ); - if( !invalidFitness ) - { - obj->add( "fitness", eoserial::make( this->fitness() ) ); - } - - return obj; + bool invalidFitness = this->invalid(); + obj->add( "invalid_fitness", eoserial::make( invalidFitness ) ); + if( !invalidFitness ) + { + obj->add( "fitness", eoserial::make( this->fitness() ) ); } + return obj; + } }; -typedef SerializableParticle Particle; -//----------------------------------------------------------------------------- -// the objective function -double real_value (const Particle & _particle) +// REPRESENTATION +//----------------------------------------------------------------------------- +// define your individuals +typedef SerializableEOReal Indi; +typedef double IndiFitness; + +// EVAL +//----------------------------------------------------------------------------- +// a simple fitness function that computes the euclidian norm of a real vector +// @param _indi A real-valued individual + +double real_value(const Indi & _indi) { - double sum = 0; - for (unsigned i = 0; i < _particle.size ()-1; i++) - sum += pow(_particle[i],2); - return (sum); + double sum = 0; + for (unsigned i = 0; i < _indi.size(); i++) + sum += _indi[i]*_indi[i]; + return (-sum); // maximizing only } /************************** PARALLELIZATION JOB *******************************/ @@ -113,10 +116,12 @@ struct SerializableBasicType : public eoserial::Persistent template< class EOT, class FitT > struct MultiStartData { - MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, int _masterRank, eoInit* _init = 0 ) + typedef eoF ResetAlgo; + + MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, int _masterRank, ResetAlgo & _resetAlgo ) : runs( 0 ), firstIndividual( true ), bestFitness(), pop(), - comm( _comm ), algo( _algo ), masterRank( _masterRank ), init( _init ) + comm( _comm ), algo( _algo ), masterRank( _masterRank ), resetAlgo( _resetAlgo ) { // empty } @@ -131,7 +136,7 @@ struct MultiStartData // static parameters mpi::communicator& comm; eoAlgo& algo; - eoInit* init; + ResetAlgo& resetAlgo; int masterRank; }; @@ -183,6 +188,13 @@ class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT, F void operator()() { + // DEBUG + //static int i = 0; + //std::cout << Node::comm().rank() << "-" << i++ << " random: " << eo::rng.rand() << std::endl; + + // std::cout << "POP(" << _data->pop.size() << ") : " << _data->pop << std::endl; + + _data->resetAlgo(); _data->algo( _data->pop ); _data->comm.send( _data->masterRank, 1, _data->pop.best_element() ); } @@ -205,10 +217,22 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > { public: - MultiStartStore( eoAlgo & algo, int masterRank, const eoPop< EOT > & pop, eoInit* init = 0 ) - : _data( Node::comm(), algo, masterRank, init ), - _pop( pop ), - _firstPopInit( true ) + typedef typename MultiStartData::ResetAlgo ResetAlgo; + typedef eoUF< eoPop&, void > ReinitJob; + typedef eoUF< int, std::vector > GetSeeds; + + MultiStartStore( + eoAlgo & algo, + int masterRank, + // eoInit* init = 0 + ReinitJob & reinitJob, + ResetAlgo & resetAlgo, + GetSeeds & getSeeds + ) + : _data( Node::comm(), algo, masterRank, resetAlgo ), + _masterRank( masterRank ), + _getSeeds( getSeeds ), + _reinitJob( reinitJob ) { this->_iff = new IsFinishedMultiStart< EOT, FitT >; this->_iff->needDelete(true); @@ -220,20 +244,39 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > this->_ptf->needDelete(true); } - void init( int runs ) + void init( const std::vector& workers, int runs ) { + int nbWorkers = workers.size(); + + _reinitJob( _data.pop ); _data.runs = runs; - if( _data.init ) + std::vector< int > seeds = _getSeeds( nbWorkers ); + if( Node::comm().rank() == _masterRank ) { - _data.pop = eoPop( _pop.size(), *_data.init ); - } else if( _firstPopInit ) - { - _data.pop = _pop; - } - _firstPopInit = false; + if( seeds.size() < nbWorkers ) + { + // TODO + // get multiples of the current seed? + // generate seeds? + for( int i = 1; seeds.size() < nbWorkers ; ++i ) + { + seeds.push_back( i ); + } + } - _data.firstIndividual = true; + for( int i = 0 ; i < nbWorkers ; ++i ) + { + int wrkRank = workers[i]; + Node::comm().send( wrkRank, 1, seeds[ i ] ); + } + } else + { + int seed; + Node::comm().recv( _masterRank, 1, seed ); + std::cout << Node::comm().rank() << "- Seed: " << seed << std::endl; + eo::rng.reseed( seed ); + } } MultiStartData* data() @@ -243,12 +286,14 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > private: MultiStartData< EOT, FitT > _data; - const eoPop< EOT >& _pop; - bool _firstPopInit; + + GetSeeds & _getSeeds; + ReinitJob & _reinitJob; + int _masterRank; }; template< class EOT, class FitT > -class MultiStart : public MultiJob< MultiStartData< EOT, FitT > > +class MultiStart : public OneShotJob< MultiStartData< EOT, FitT > > { public: @@ -258,38 +303,9 @@ class MultiStart : public MultiJob< MultiStartData< EOT, FitT > > // dynamic parameters int runs, const std::vector& seeds = std::vector() ) : - MultiJob< MultiStartData< EOT, FitT > >( algo, masterRank, store ) + OneShotJob< MultiStartData< EOT, FitT > >( algo, masterRank, store ) { - store.init( runs ); - - if( this->isMaster() ) - { - int nbWorkers = algo.availableWorkers(); - std::vector realSeeds = seeds; - if( realSeeds.size() < nbWorkers ) - { - // TODO - // get multiples of the current seed? - // generate seeds? - for( int i = 1; realSeeds.size() < nbWorkers ; ++i ) - { - realSeeds.push_back( i ); - } - } - - std::vector idles = algo.idles(); - for( int i = 0 ; i < nbWorkers ; ++i ) - { - int wrkRank = idles[i]; - Node::comm().send( wrkRank, 1, realSeeds[ i ] ); - } - } else - { - int seed; - Node::comm().recv( masterRank, 1, seed ); - std::cout << Node::comm().rank() << "- Seed: " << seed << std::endl; - eo::rng.reseed( seed ); - } + store.init( algo.idles(), runs ); } EOT& best_individual() @@ -303,95 +319,205 @@ class MultiStart : public MultiJob< MultiStartData< EOT, FitT > > } }; +template +struct DummyGetSeeds : public MultiStartStore::GetSeeds +{ + std::vector operator()( int n ) + { + return std::vector(); + } +}; + +template +struct GetRandomSeeds : public MultiStartStore::GetSeeds +{ + std::vector operator()( int n ) + { + std::vector ret; + for(int i = 0; i < n; ++i) + { + ret.push_back( eo::rng.rand() ); + } + } +}; + +template +struct ReinitMultiEA : public MultiStartStore::ReinitJob +{ + ReinitMultiEA( const eoPop& pop, eoEvalFunc& eval ) : _originalPop( pop ), _eval( eval ) + { + // empty + } + + void operator()( eoPop& pop ) + { + pop = _originalPop; + for(unsigned i = 0, size = pop.size(); i < size; ++i) + { + _eval( pop[i] ); + } + } + + private: + const eoPop& _originalPop; + eoEvalFunc& _eval; +}; + +template +struct ResetAlgoEA : public MultiStartStore::ResetAlgo +{ + ResetAlgoEA( eoGenContinue & continuator ) : + _continuator( continuator ), + _initial( continuator.totalGenerations() ) + { + // empty + } + + void operator()() + { + _continuator.totalGenerations( _initial ); + } + + private: + unsigned int _initial; + eoGenContinue & _continuator; +}; + +template< class EOT > +struct eoInitAndEval : public eoInit +{ + eoInitAndEval( eoInit& init, eoEvalFunc& eval ) : _init( init ), _eval( eval ) + { + // empty + } + + void operator()( EOT & indi ) + { + _init( indi ); + _eval( indi ); + } + + private: + eoInit& _init; + eoEvalFunc& _eval; +}; + int main(int argc, char **argv) { Node::init( argc, argv ); - const unsigned int VEC_SIZE = 2; - const unsigned int POP_SIZE = 20; - const unsigned int NEIGHBORHOOD_SIZE= 5; - unsigned i; + // PARAMETRES + // all parameters are hard-coded! + const unsigned int SEED = 133742; // seed for random number generator + const unsigned int VEC_SIZE = 8; // Number of object variables in genotypes + const unsigned int POP_SIZE = 20; // Size of population + const unsigned int T_SIZE = 3; // size for tournament selection + const unsigned int MAX_GEN = 20; // Maximum number of generation before STOP + const float CROSS_RATE = 0.8; // Crossover rate + const double EPSILON = 0.01; // range for real uniform mutation + const float MUT_RATE = 0.5; // mutation rate - eo::rng.reseed(1); + // GENERAL + ////////////////////////// + // Random seed + ////////////////////////// + //reproducible random seed: if you don't change SEED above, + // you'll aways get the same result, NOT a random run + rng.reseed(SEED); - // the population: - eoPop pop; + // EVAL + ///////////////////////////// + // Fitness function + //////////////////////////// + // Evaluation: from a plain C++ fn to an EvalFunc Object + eoEvalFuncPtr eval( real_value ); - // Evaluation - eoEvalFuncPtr eval( real_value ); + // INIT + //////////////////////////////// + // Initilisation of population + //////////////////////////////// - // position init - eoUniformGenerator < double >uGen (-3, 3); - eoInitFixedLength < Particle > random (VEC_SIZE, uGen); + // declare the population + eoPop pop; + // fill it! + /* + for (unsigned int igeno=0; igeno generator; + eoInitFixedLength< Indi > init( VEC_SIZE, generator ); + // eoInitAndEval< Indi > init( real_init, eval, continuator ); + pop = eoPop( POP_SIZE, init ); - // velocity init - eoUniformGenerator < double >sGen (-2, 2); - eoVelocityInitFixedLength < Particle > veloRandom (VEC_SIZE, sGen); + // ENGINE + ///////////////////////////////////// + // selection and replacement + //////////////////////////////////// + // SELECT + // The robust tournament selection + eoDetTournamentSelect select(T_SIZE); // T_SIZE in [2,POP_SIZE] - // local best init - eoFirstIsBestInit < Particle > localInit; + // REPLACE + // eoSGA uses generational replacement by default + // so no replacement procedure has to be given - // perform position initialization - pop.append (POP_SIZE, random); + // OPERATORS + ////////////////////////////////////// + // The variation operators + ////////////////////////////////////// + // CROSSOVER + // offspring(i) is a linear combination of parent(i) + eoSegmentCrossover xover; + // MUTATION + // offspring(i) uniformly chosen in [parent(i)-epsilon, parent(i)+epsilon] + eoUniformMutation mutation(EPSILON); - // topology - eoLinearTopology topology(NEIGHBORHOOD_SIZE); + // STOP + // CHECKPOINT + ////////////////////////////////////// + // termination condition + ///////////////////////////////////// + // stop after MAX_GEN generations + eoGenContinue continuator(MAX_GEN); /** TODO FIXME FIXME BUG HERE! + Continuator thinks it's done! */ - // the full initializer - eoInitializer init(eval,veloRandom,localInit,topology,pop); - init(); + // GENERATION + ///////////////////////////////////////// + // the algorithm + //////////////////////////////////////// + // standard Generational GA requires + // selection, evaluation, crossover and mutation, stopping criterion - // bounds - eoRealVectorBounds bnds(VEC_SIZE,-1.5,1.5); - - // velocity - eoStandardVelocity velocity (topology,1,1.6,2,bnds); - - // flight - eoStandardFlight flight; - - // Terminators - eoGenContinue genCont1 (50); - eoGenContinue genCont2 (50); - - // PS flight - eoEasyPSO pso1(genCont1, eval, velocity, flight); - - // eoEasyPSO pso2(init,genCont2, eval, velocity, flight); + eoSGA gga(select, xover, CROSS_RATE, mutation, MUT_RATE, + eval, continuator); DynamicAssignmentAlgorithm assignmentAlgo; - MultiStartStore< Particle, ParticleFitness > store( pso1, DEFAULT_MASTER, pop ); + MultiStartStore< Indi, IndiFitness > store( + gga, + DEFAULT_MASTER, + *new ReinitMultiEA< Indi, IndiFitness >( pop, eval ), + *new ResetAlgoEA< Indi, IndiFitness >( continuator ), + *new DummyGetSeeds< Indi, IndiFitness >()); - MultiStart< Particle, ParticleFitness > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); + MultiStart< Indi, IndiFitness > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); msjob.run(); if( msjob.isMaster() ) { - eo::mpi::EmptyJob tjob( assignmentAlgo, DEFAULT_MASTER ); std::cout << "Global best individual has fitness " << msjob.best_fitness() << std::endl; } - // flight - /* - try - { - pso1(pop); - std::cout << "FINAL POPULATION AFTER PSO n°1:" << std::endl; - for (i = 0; i < pop.size(); ++i) - std::cout << "\t" << pop[i] << " " << pop[i].fitness() << std::endl; - - pso2(pop); - std::cout << "FINAL POPULATION AFTER PSO n°2:" << std::endl; - for (i = 0; i < pop.size(); ++i) - std::cout << "\t" << pop[i] << " " << pop[i].fitness() << std::endl; - } - catch (std::exception& e) - { - std::cout << "exception: " << e.what() << std::endl;; - exit(EXIT_FAILURE); - } - */ + MultiStart< Indi, IndiFitness > msjob10( assignmentAlgo, DEFAULT_MASTER, store, 10 ); + msjob10.run(); return 0; - } From e58b3e5cb258c52155d9d557f3e964721f8a0726 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 12:06:23 +0200 Subject: [PATCH 18/31] MPI Multistart: removed second template parameter Fitness, as it can be deduced from Individual. --- eo/test/mpi/t-mpi-multistart.cpp | 87 ++++++++++++++++---------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index dc070af6c..da736fab5 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -61,7 +61,6 @@ public: //----------------------------------------------------------------------------- // define your individuals typedef SerializableEOReal Indi; -typedef double IndiFitness; // EVAL //----------------------------------------------------------------------------- @@ -113,7 +112,7 @@ struct SerializableBasicType : public eoserial::Persistent T _value; }; -template< class EOT, class FitT > +template< class EOT > struct MultiStartData { typedef eoF ResetAlgo; @@ -129,7 +128,7 @@ struct MultiStartData // dynamic parameters int runs; bool firstIndividual; - FitT bestFitness; + typename EOT::Fitness bestFitness; EOT bestIndividual; eoPop< EOT > pop; @@ -140,11 +139,11 @@ struct MultiStartData int masterRank; }; -template< class EOT, class FitT > -class SendTaskMultiStart : public SendTaskFunction< MultiStartData< EOT, FitT > > +template< class EOT > +class SendTaskMultiStart : public SendTaskFunction< MultiStartData< EOT > > { public: - using SendTaskFunction< MultiStartData< EOT, FitT > >::_data; + using SendTaskFunction< MultiStartData< EOT > >::_data; void operator()( int wrkRank ) { @@ -152,18 +151,18 @@ class SendTaskMultiStart : public SendTaskFunction< MultiStartData< EOT, FitT > } }; -template< class EOT, class FitT > -class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< EOT, FitT > > +template< class EOT > +class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< EOT > > { public: - using HandleResponseFunction< MultiStartData< EOT, FitT > >::_data; + using HandleResponseFunction< MultiStartData< EOT > >::_data; void operator()( int wrkRank ) { std::cout << "Response received." << std::endl; EOT individual; - MultiStartData< EOT, FitT >& d = *_data; + MultiStartData< EOT >& d = *_data; d.comm.recv( wrkRank, 1, individual ); std::cout << "Fitness of individual: " << individual.fitness() << std::endl; @@ -180,11 +179,11 @@ class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< } }; -template< class EOT, class FitT > -class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT, FitT > > +template< class EOT > +class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT > > { public: - using ProcessTaskFunction< MultiStartData >::_data; + using ProcessTaskFunction< MultiStartData >::_data; void operator()() { @@ -200,11 +199,11 @@ class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT, F } }; -template< class EOT, class FitT > -class IsFinishedMultiStart : public IsFinishedFunction< MultiStartData< EOT, FitT > > +template< class EOT > +class IsFinishedMultiStart : public IsFinishedFunction< MultiStartData< EOT > > { public: - using IsFinishedFunction< MultiStartData< EOT, FitT > >::_data; + using IsFinishedFunction< MultiStartData< EOT > >::_data; bool operator()() { @@ -212,12 +211,12 @@ class IsFinishedMultiStart : public IsFinishedFunction< MultiStartData< EOT, Fit } }; -template< class EOT, class FitT > -class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > +template< class EOT > +class MultiStartStore : public JobStore< MultiStartData< EOT > > { public: - typedef typename MultiStartData::ResetAlgo ResetAlgo; + typedef typename MultiStartData::ResetAlgo ResetAlgo; typedef eoUF< eoPop&, void > ReinitJob; typedef eoUF< int, std::vector > GetSeeds; @@ -234,13 +233,13 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > _getSeeds( getSeeds ), _reinitJob( reinitJob ) { - this->_iff = new IsFinishedMultiStart< EOT, FitT >; + this->_iff = new IsFinishedMultiStart< EOT >; this->_iff->needDelete(true); - this->_stf = new SendTaskMultiStart< EOT, FitT >; + this->_stf = new SendTaskMultiStart< EOT >; this->_stf->needDelete(true); - this->_hrf = new HandleResponseMultiStart< EOT, FitT >; + this->_hrf = new HandleResponseMultiStart< EOT >; this->_hrf->needDelete(true); - this->_ptf = new ProcessTaskMultiStart< EOT, FitT >; + this->_ptf = new ProcessTaskMultiStart< EOT >; this->_ptf->needDelete(true); } @@ -279,31 +278,31 @@ class MultiStartStore : public JobStore< MultiStartData< EOT, FitT > > } } - MultiStartData* data() + MultiStartData* data() { return &_data; } private: - MultiStartData< EOT, FitT > _data; + MultiStartData< EOT > _data; GetSeeds & _getSeeds; ReinitJob & _reinitJob; int _masterRank; }; -template< class EOT, class FitT > -class MultiStart : public OneShotJob< MultiStartData< EOT, FitT > > +template< class EOT > +class MultiStart : public OneShotJob< MultiStartData< EOT > > { public: MultiStart( AssignmentAlgorithm & algo, int masterRank, - MultiStartStore< EOT, FitT > & store, + MultiStartStore< EOT > & store, // dynamic parameters int runs, const std::vector& seeds = std::vector() ) : - OneShotJob< MultiStartData< EOT, FitT > >( algo, masterRank, store ) + OneShotJob< MultiStartData< EOT > >( algo, masterRank, store ) { store.init( algo.idles(), runs ); } @@ -313,14 +312,14 @@ class MultiStart : public OneShotJob< MultiStartData< EOT, FitT > > return this->store.data()->bestIndividual; } - FitT best_fitness() + typename EOT::Fitness best_fitness() { return this->store.data()->bestFitness; } }; -template -struct DummyGetSeeds : public MultiStartStore::GetSeeds +template +struct DummyGetSeeds : public MultiStartStore::GetSeeds { std::vector operator()( int n ) { @@ -328,8 +327,8 @@ struct DummyGetSeeds : public MultiStartStore::GetSeeds } }; -template -struct GetRandomSeeds : public MultiStartStore::GetSeeds +template +struct GetRandomSeeds : public MultiStartStore::GetSeeds { std::vector operator()( int n ) { @@ -341,8 +340,8 @@ struct GetRandomSeeds : public MultiStartStore::GetSeeds } }; -template -struct ReinitMultiEA : public MultiStartStore::ReinitJob +template +struct ReinitMultiEA : public MultiStartStore::ReinitJob { ReinitMultiEA( const eoPop& pop, eoEvalFunc& eval ) : _originalPop( pop ), _eval( eval ) { @@ -363,8 +362,8 @@ struct ReinitMultiEA : public MultiStartStore::ReinitJob eoEvalFunc& _eval; }; -template -struct ResetAlgoEA : public MultiStartStore::ResetAlgo +template +struct ResetAlgoEA : public MultiStartStore::ResetAlgo { ResetAlgoEA( eoGenContinue & continuator ) : _continuator( continuator ), @@ -501,14 +500,14 @@ int main(int argc, char **argv) eval, continuator); DynamicAssignmentAlgorithm assignmentAlgo; - MultiStartStore< Indi, IndiFitness > store( + MultiStartStore< Indi > store( gga, DEFAULT_MASTER, - *new ReinitMultiEA< Indi, IndiFitness >( pop, eval ), - *new ResetAlgoEA< Indi, IndiFitness >( continuator ), - *new DummyGetSeeds< Indi, IndiFitness >()); + *new ReinitMultiEA< Indi >( pop, eval ), + *new ResetAlgoEA< Indi >( continuator ), + *new DummyGetSeeds< Indi >()); - MultiStart< Indi, IndiFitness > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); + MultiStart< Indi > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); msjob.run(); if( msjob.isMaster() ) @@ -516,7 +515,7 @@ int main(int argc, char **argv) std::cout << "Global best individual has fitness " << msjob.best_fitness() << std::endl; } - MultiStart< Indi, IndiFitness > msjob10( assignmentAlgo, DEFAULT_MASTER, store, 10 ); + MultiStart< Indi > msjob10( assignmentAlgo, DEFAULT_MASTER, store, 10 ); msjob10.run(); return 0; From e2e6ad8c9999edbf4ad3af70dcbd41fe571ac2a0 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 13:48:40 +0200 Subject: [PATCH 19/31] MPI Multistart: renamed default functors and added a few default functors. --- eo/test/mpi/t-mpi-multistart.cpp | 47 ++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index da736fab5..cd9a28f40 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -319,6 +319,7 @@ class MultiStart : public OneShotJob< MultiStartData< EOT > > }; template +// No seeds! Use default generator struct DummyGetSeeds : public MultiStartStore::GetSeeds { std::vector operator()( int n ) @@ -327,9 +328,43 @@ struct DummyGetSeeds : public MultiStartStore::GetSeeds } }; +template +// Multiple of a seed +struct MultiplesOfNumber : public MultiStartStore::GetSeeds +{ + MultiplesOfNumber ( int n = 0 ) + { + while( n == 0 ) + { + n = eo::rng.rand(); + } + _seed = n; + _i = 0; + } + + std::vector operator()( int n ) + { + std::vector ret; + for( unsigned int i = 0; i < n; ++i ) + { + ret.push_back( (++_i) * _seed ); + } + } + + private: + + unsigned int _seed; + unsigned int _i; +}; + template struct GetRandomSeeds : public MultiStartStore::GetSeeds { + GetRandomSeeds( int seed ) + { + eo::rng.reseed( seed ); + } + std::vector operator()( int n ) { std::vector ret; @@ -341,9 +376,9 @@ struct GetRandomSeeds : public MultiStartStore::GetSeeds }; template -struct ReinitMultiEA : public MultiStartStore::ReinitJob +struct RecopyPopEA : public MultiStartStore::ReinitJob { - ReinitMultiEA( const eoPop& pop, eoEvalFunc& eval ) : _originalPop( pop ), _eval( eval ) + RecopyPopEA( const eoPop& pop, eoEvalFunc& eval ) : _originalPop( pop ), _eval( eval ) { // empty } @@ -363,9 +398,9 @@ struct ReinitMultiEA : public MultiStartStore::ReinitJob }; template -struct ResetAlgoEA : public MultiStartStore::ResetAlgo +struct ResetGenContinueEA: public MultiStartStore::ResetAlgo { - ResetAlgoEA( eoGenContinue & continuator ) : + ResetGenContinueEA( eoGenContinue & continuator ) : _continuator( continuator ), _initial( continuator.totalGenerations() ) { @@ -503,8 +538,8 @@ int main(int argc, char **argv) MultiStartStore< Indi > store( gga, DEFAULT_MASTER, - *new ReinitMultiEA< Indi >( pop, eval ), - *new ResetAlgoEA< Indi >( continuator ), + *new RecopyPopEA< Indi >( pop, eval ), + *new ResetGenContinueEA< Indi >( continuator ), *new DummyGetSeeds< Indi >()); MultiStart< Indi > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); From db93ac635bbbeac2e8d76c1bd7d9e641561cf1ec Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 13:58:36 +0200 Subject: [PATCH 20/31] MPI Multistart: merged functors ReinitJob and ResetAlgo, as it should in fact be done in the mean while. --- eo/test/mpi/t-mpi-multistart.cpp | 70 ++++++++------------------------ 1 file changed, 17 insertions(+), 53 deletions(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index cd9a28f40..e704ab613 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -115,7 +115,7 @@ struct SerializableBasicType : public eoserial::Persistent template< class EOT > struct MultiStartData { - typedef eoF ResetAlgo; + typedef eoUF< eoPop&, void> ResetAlgo; MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, int _masterRank, ResetAlgo & _resetAlgo ) : @@ -193,7 +193,7 @@ class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT > // std::cout << "POP(" << _data->pop.size() << ") : " << _data->pop << std::endl; - _data->resetAlgo(); + _data->resetAlgo( _data->pop ); _data->algo( _data->pop ); _data->comm.send( _data->masterRank, 1, _data->pop.best_element() ); } @@ -217,21 +217,18 @@ class MultiStartStore : public JobStore< MultiStartData< EOT > > public: typedef typename MultiStartData::ResetAlgo ResetAlgo; - typedef eoUF< eoPop&, void > ReinitJob; typedef eoUF< int, std::vector > GetSeeds; MultiStartStore( eoAlgo & algo, int masterRank, // eoInit* init = 0 - ReinitJob & reinitJob, ResetAlgo & resetAlgo, GetSeeds & getSeeds ) : _data( Node::comm(), algo, masterRank, resetAlgo ), _masterRank( masterRank ), - _getSeeds( getSeeds ), - _reinitJob( reinitJob ) + _getSeeds( getSeeds ) { this->_iff = new IsFinishedMultiStart< EOT >; this->_iff->needDelete(true); @@ -245,11 +242,9 @@ class MultiStartStore : public JobStore< MultiStartData< EOT > > void init( const std::vector& workers, int runs ) { - int nbWorkers = workers.size(); - - _reinitJob( _data.pop ); _data.runs = runs; + int nbWorkers = workers.size(); std::vector< int > seeds = _getSeeds( nbWorkers ); if( Node::comm().rank() == _masterRank ) { @@ -287,7 +282,6 @@ class MultiStartStore : public JobStore< MultiStartData< EOT > > MultiStartData< EOT > _data; GetSeeds & _getSeeds; - ReinitJob & _reinitJob; int _masterRank; }; @@ -376,9 +370,16 @@ struct GetRandomSeeds : public MultiStartStore::GetSeeds }; template -struct RecopyPopEA : public MultiStartStore::ReinitJob +struct ReusePopEA: public MultiStartStore::ResetAlgo { - RecopyPopEA( const eoPop& pop, eoEvalFunc& eval ) : _originalPop( pop ), _eval( eval ) + ReusePopEA( + eoGenContinue & continuator, + const eoPop& originalPop, + eoEvalFunc& eval) : + _initial( continuator.totalGenerations() ), + _continuator( continuator ), + _originalPop( originalPop ), + _eval( eval ) { // empty } @@ -390,49 +391,13 @@ struct RecopyPopEA : public MultiStartStore::ReinitJob { _eval( pop[i] ); } - } - - private: - const eoPop& _originalPop; - eoEvalFunc& _eval; -}; - -template -struct ResetGenContinueEA: public MultiStartStore::ResetAlgo -{ - ResetGenContinueEA( eoGenContinue & continuator ) : - _continuator( continuator ), - _initial( continuator.totalGenerations() ) - { - // empty - } - - void operator()() - { _continuator.totalGenerations( _initial ); } private: unsigned int _initial; eoGenContinue & _continuator; -}; - -template< class EOT > -struct eoInitAndEval : public eoInit -{ - eoInitAndEval( eoInit& init, eoEvalFunc& eval ) : _init( init ), _eval( eval ) - { - // empty - } - - void operator()( EOT & indi ) - { - _init( indi ); - _eval( indi ); - } - - private: - eoInit& _init; + const eoPop& _originalPop; eoEvalFunc& _eval; }; @@ -444,9 +409,9 @@ int main(int argc, char **argv) // all parameters are hard-coded! const unsigned int SEED = 133742; // seed for random number generator const unsigned int VEC_SIZE = 8; // Number of object variables in genotypes - const unsigned int POP_SIZE = 20; // Size of population + const unsigned int POP_SIZE = 100; // Size of population const unsigned int T_SIZE = 3; // size for tournament selection - const unsigned int MAX_GEN = 20; // Maximum number of generation before STOP + const unsigned int MAX_GEN = 100; // Maximum number of generation before STOP const float CROSS_RATE = 0.8; // Crossover rate const double EPSILON = 0.01; // range for real uniform mutation const float MUT_RATE = 0.5; // mutation rate @@ -538,8 +503,7 @@ int main(int argc, char **argv) MultiStartStore< Indi > store( gga, DEFAULT_MASTER, - *new RecopyPopEA< Indi >( pop, eval ), - *new ResetGenContinueEA< Indi >( continuator ), + *new ReusePopEA< Indi >( continuator, pop, eval ), *new DummyGetSeeds< Indi >()); MultiStart< Indi > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); From 9a40d6ed0417780a386e2e740d349b5c6d3143d4 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 14:12:32 +0200 Subject: [PATCH 21/31] MPI Multistart: removed traces and useless comments. --- eo/test/mpi/t-mpi-multistart.cpp | 79 ++++++-------------------------- 1 file changed, 14 insertions(+), 65 deletions(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index e704ab613..8de75c6e7 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -11,7 +11,6 @@ using namespace eo::mpi; // Use functions from namespace std using namespace std; - class SerializableEOReal: public eoReal, public eoserial::Persistent { public: @@ -19,6 +18,7 @@ public: SerializableEOReal(unsigned size = 0, double value = 0.0) : eoReal(size, value) { + // empty } void unpack( const eoserial::Object* obj ) @@ -69,48 +69,13 @@ typedef SerializableEOReal Indi; double real_value(const Indi & _indi) { - double sum = 0; - for (unsigned i = 0; i < _indi.size(); i++) - sum += _indi[i]*_indi[i]; - return (-sum); // maximizing only + double sum = 0; + for (unsigned i = 0; i < _indi.size(); i++) + sum += _indi[i]*_indi[i]; + return (-sum); // maximizing only } /************************** PARALLELIZATION JOB *******************************/ -/* - * This file is a template for a new eo::mpi::Job. You have everything that should be necessary to implement a new - * parallelized algorithm. - * - * Replace MultiStart by the name of your algorithm (for instance: MultiStart, ParallelApply, etc.). - */ - -template< class T > -struct SerializableBasicType : public eoserial::Persistent -{ - public: - SerializableBasicType( T & value ) - { - _value = value; - } - - operator T&() - { - return _value; - } - - void unpack( const eoserial::Object* obj ) - { - eoserial::unpack( *obj, "value", _value ); - } - - eoserial::Object* pack( void ) const - { - eoserial::Object* obj = new eoserial::Object; - obj->add( "value", eoserial::make( _value ) ); - } - - private: - T _value; -}; template< class EOT > struct MultiStartData @@ -159,17 +124,10 @@ class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< void operator()( int wrkRank ) { - std::cout << "Response received." << std::endl; - EOT individual; MultiStartData< EOT >& d = *_data; d.comm.recv( wrkRank, 1, individual ); - std::cout << "Fitness of individual: " << individual.fitness() << std::endl; - if ( ! d.firstIndividual ) { - std::cout << "Best fitness: " << d.bestFitness << std::endl; - } - if( d.firstIndividual || individual.fitness() > d.bestFitness ) { d.bestFitness = individual.fitness(); @@ -187,12 +145,6 @@ class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT > void operator()() { - // DEBUG - //static int i = 0; - //std::cout << Node::comm().rank() << "-" << i++ << " random: " << eo::rng.rand() << std::endl; - - // std::cout << "POP(" << _data->pop.size() << ") : " << _data->pop << std::endl; - _data->resetAlgo( _data->pop ); _data->algo( _data->pop ); _data->comm.send( _data->masterRank, 1, _data->pop.best_element() ); @@ -222,13 +174,12 @@ class MultiStartStore : public JobStore< MultiStartData< EOT > > MultiStartStore( eoAlgo & algo, int masterRank, - // eoInit* init = 0 ResetAlgo & resetAlgo, GetSeeds & getSeeds ) : _data( Node::comm(), algo, masterRank, resetAlgo ), - _masterRank( masterRank ), - _getSeeds( getSeeds ) + _getSeeds( getSeeds ), + _masterRank( masterRank ) { this->_iff = new IsFinishedMultiStart< EOT >; this->_iff->needDelete(true); @@ -250,12 +201,10 @@ class MultiStartStore : public JobStore< MultiStartData< EOT > > { if( seeds.size() < nbWorkers ) { - // TODO - // get multiples of the current seed? - // generate seeds? - for( int i = 1; seeds.size() < nbWorkers ; ++i ) + // Random seeds + for( int i = seeds.size(); i < nbWorkers; ++i ) { - seeds.push_back( i ); + seeds.push_back( eo::rng.rand() ); } } @@ -268,7 +217,7 @@ class MultiStartStore : public JobStore< MultiStartData< EOT > > { int seed; Node::comm().recv( _masterRank, 1, seed ); - std::cout << Node::comm().rank() << "- Seed: " << seed << std::endl; + eo::log << eo::debug << Node::comm().rank() << "- Seed: " << seed << std::endl; eo::rng.reseed( seed ); } } @@ -280,7 +229,6 @@ class MultiStartStore : public JobStore< MultiStartData< EOT > > private: MultiStartData< EOT > _data; - GetSeeds & _getSeeds; int _masterRank; }; @@ -343,6 +291,7 @@ struct MultiplesOfNumber : public MultiStartStore::GetSeeds { ret.push_back( (++_i) * _seed ); } + return ret; } private: @@ -366,6 +315,7 @@ struct GetRandomSeeds : public MultiStartStore::GetSeeds { ret.push_back( eo::rng.rand() ); } + return ret; } }; @@ -486,8 +436,7 @@ int main(int argc, char **argv) // termination condition ///////////////////////////////////// // stop after MAX_GEN generations - eoGenContinue continuator(MAX_GEN); /** TODO FIXME FIXME BUG HERE! - Continuator thinks it's done! */ + eoGenContinue continuator(MAX_GEN); // GENERATION ///////////////////////////////////////// From 12614faee1986cec3e6f02c1cc6370add7c3ec41 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 14:23:07 +0200 Subject: [PATCH 22/31] MPI Multistart: saving solutions sent by workers in a eoPop instead of keeping only the best solution. --- eo/test/mpi/t-mpi-multistart.cpp | 45 +++++++++++++++----------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index 8de75c6e7..d08d75f6d 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -84,7 +84,7 @@ struct MultiStartData MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, int _masterRank, ResetAlgo & _resetAlgo ) : - runs( 0 ), firstIndividual( true ), bestFitness(), pop(), + runs( 0 ), pop(), bests(), comm( _comm ), algo( _algo ), masterRank( _masterRank ), resetAlgo( _resetAlgo ) { // empty @@ -92,9 +92,7 @@ struct MultiStartData // dynamic parameters int runs; - bool firstIndividual; - typename EOT::Fitness bestFitness; - EOT bestIndividual; + eoPop< EOT > bests; eoPop< EOT > pop; // static parameters @@ -127,13 +125,7 @@ class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< EOT individual; MultiStartData< EOT >& d = *_data; d.comm.recv( wrkRank, 1, individual ); - - if( d.firstIndividual || individual.fitness() > d.bestFitness ) - { - d.bestFitness = individual.fitness(); - d.bestIndividual = individual; - d.firstIndividual = false; - } + d.bests.push_back( individual ); } }; @@ -249,14 +241,9 @@ class MultiStart : public OneShotJob< MultiStartData< EOT > > store.init( algo.idles(), runs ); } - EOT& best_individual() + eoPop& best_individuals() { - return this->store.data()->bestIndividual; - } - - typename EOT::Fitness best_fitness() - { - return this->store.data()->bestFitness; + return this->store.data()->bests; } }; @@ -320,9 +307,9 @@ struct GetRandomSeeds : public MultiStartStore::GetSeeds }; template -struct ReusePopEA: public MultiStartStore::ResetAlgo +struct ReuseOriginalPopEA: public MultiStartStore::ResetAlgo { - ReusePopEA( + ReuseOriginalPopEA( eoGenContinue & continuator, const eoPop& originalPop, eoEvalFunc& eval) : @@ -372,7 +359,7 @@ int main(int argc, char **argv) ////////////////////////// //reproducible random seed: if you don't change SEED above, // you'll aways get the same result, NOT a random run - rng.reseed(SEED); + // rng.reseed(SEED); // EVAL ///////////////////////////// @@ -449,22 +436,32 @@ int main(int argc, char **argv) eval, continuator); DynamicAssignmentAlgorithm assignmentAlgo; + ReuseOriginalPopEA< Indi > resetAlgo( continuator, pop, eval ); + GetRandomSeeds< Indi > getSeeds( SEED ); + MultiStartStore< Indi > store( gga, DEFAULT_MASTER, - *new ReusePopEA< Indi >( continuator, pop, eval ), - *new DummyGetSeeds< Indi >()); + resetAlgo, + getSeeds); MultiStart< Indi > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); msjob.run(); if( msjob.isMaster() ) { - std::cout << "Global best individual has fitness " << msjob.best_fitness() << std::endl; + msjob.best_individuals().sort(); + std::cout << "Global best individual has fitness " << msjob.best_individuals().best_element().fitness() << std::endl; } MultiStart< Indi > msjob10( assignmentAlgo, DEFAULT_MASTER, store, 10 ); msjob10.run(); + if( msjob10.isMaster() ) + { + msjob10.best_individuals().sort(); + std::cout << "Global best individual has fitness " << msjob10.best_individuals().best_element().fitness() << std::endl; + } return 0; + } From 1885517dc1434fc7255847da47715c53fc7eff70 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 14:31:57 +0200 Subject: [PATCH 23/31] MPI Multistart: new default ResetAlgo functor: reuse the same pop. --- eo/test/mpi/t-mpi-multistart.cpp | 40 +++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index d08d75f6d..91db72e3a 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -338,6 +338,44 @@ struct ReuseOriginalPopEA: public MultiStartStore::ResetAlgo eoEvalFunc& _eval; }; +template< class EOT > +struct ReuseSamePopEA : public MultiStartStore::ResetAlgo +{ + ReuseSamePopEA( + eoGenContinue& continuator, + const eoPop& originalPop, + eoEvalFunc& eval + ) : + _continuator( continuator ), + _originalPop( originalPop ), + _firstTime( true ), + _initial( continuator.totalGenerations() ) + { + for( unsigned i = 0, size = originalPop.size(); + i < size; ++i ) + { + eval(_originalPop[i]); + } + } + + void operator()( eoPop& pop ) + { + if( _firstTime ) + { + pop = _originalPop; + _firstTime = false; + } + _continuator.totalGenerations( _initial ); + } + + protected: + + eoGenContinue& _continuator; + eoPop _originalPop; + bool _firstTime; + int _initial; +}; + int main(int argc, char **argv) { Node::init( argc, argv ); @@ -436,7 +474,7 @@ int main(int argc, char **argv) eval, continuator); DynamicAssignmentAlgorithm assignmentAlgo; - ReuseOriginalPopEA< Indi > resetAlgo( continuator, pop, eval ); + ReuseSamePopEA< Indi > resetAlgo( continuator, pop, eval ); GetRandomSeeds< Indi > getSeeds( SEED ); MultiStartStore< Indi > store( From dfd653d95b36228fbda2679686a1d6498b0111b0 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 14:45:31 +0200 Subject: [PATCH 24/31] MPI Multistart: moved MultiStart job into a standalone header, eoMultiStart.h. t-mpi-multistart.cpp becomes an example. --- eo/src/mpi/eoMultiStart.h | 314 +++++++++++++++++++++++++++++++ eo/test/mpi/t-mpi-multistart.cpp | 302 +---------------------------- 2 files changed, 315 insertions(+), 301 deletions(-) create mode 100644 eo/src/mpi/eoMultiStart.h diff --git a/eo/src/mpi/eoMultiStart.h b/eo/src/mpi/eoMultiStart.h new file mode 100644 index 000000000..456124f11 --- /dev/null +++ b/eo/src/mpi/eoMultiStart.h @@ -0,0 +1,314 @@ +# ifndef __EO_MULTISTART_H__ +# define __EO_MULTISTART_H__ + +# include +# include "eoMpi.h" + +namespace eo +{ + namespace mpi + { + template< class EOT > + struct MultiStartData + { + typedef eoUF< eoPop&, void> ResetAlgo; + + MultiStartData( bmpi::communicator& _comm, eoAlgo& _algo, int _masterRank, ResetAlgo & _resetAlgo ) + : + runs( 0 ), pop(), bests(), + comm( _comm ), algo( _algo ), masterRank( _masterRank ), resetAlgo( _resetAlgo ) + { + // empty + } + + // dynamic parameters + int runs; + eoPop< EOT > bests; + eoPop< EOT > pop; + + // static parameters + bmpi::communicator& comm; + eoAlgo& algo; + ResetAlgo& resetAlgo; + int masterRank; + }; + + template< class EOT > + class SendTaskMultiStart : public SendTaskFunction< MultiStartData< EOT > > + { + public: + using SendTaskFunction< MultiStartData< EOT > >::_data; + + void operator()( int wrkRank ) + { + --(_data->runs); + } + }; + + template< class EOT > + class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< EOT > > + { + public: + using HandleResponseFunction< MultiStartData< EOT > >::_data; + + void operator()( int wrkRank ) + { + EOT individual; + MultiStartData< EOT >& d = *_data; + d.comm.recv( wrkRank, 1, individual ); + d.bests.push_back( individual ); + } + }; + + template< class EOT > + class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT > > + { + public: + using ProcessTaskFunction< MultiStartData >::_data; + + void operator()() + { + _data->resetAlgo( _data->pop ); + _data->algo( _data->pop ); + _data->comm.send( _data->masterRank, 1, _data->pop.best_element() ); + } + }; + + template< class EOT > + class IsFinishedMultiStart : public IsFinishedFunction< MultiStartData< EOT > > + { + public: + using IsFinishedFunction< MultiStartData< EOT > >::_data; + + bool operator()() + { + return _data->runs <= 0; + } + }; + + template< class EOT > + class MultiStartStore : public JobStore< MultiStartData< EOT > > + { + public: + + typedef typename MultiStartData::ResetAlgo ResetAlgo; + typedef eoUF< int, std::vector > GetSeeds; + + MultiStartStore( + eoAlgo & algo, + int masterRank, + ResetAlgo & resetAlgo, + GetSeeds & getSeeds + ) + : _data( eo::mpi::Node::comm(), algo, masterRank, resetAlgo ), + _getSeeds( getSeeds ), + _masterRank( masterRank ) + { + this->_iff = new IsFinishedMultiStart< EOT >; + this->_iff->needDelete(true); + this->_stf = new SendTaskMultiStart< EOT >; + this->_stf->needDelete(true); + this->_hrf = new HandleResponseMultiStart< EOT >; + this->_hrf->needDelete(true); + this->_ptf = new ProcessTaskMultiStart< EOT >; + this->_ptf->needDelete(true); + } + + void init( const std::vector& workers, int runs ) + { + _data.runs = runs; + + int nbWorkers = workers.size(); + std::vector< int > seeds = _getSeeds( nbWorkers ); + if( eo::mpi::Node::comm().rank() == _masterRank ) + { + if( seeds.size() < nbWorkers ) + { + // Random seeds + for( int i = seeds.size(); i < nbWorkers; ++i ) + { + seeds.push_back( eo::rng.rand() ); + } + } + + for( int i = 0 ; i < nbWorkers ; ++i ) + { + int wrkRank = workers[i]; + eo::mpi::Node::comm().send( wrkRank, 1, seeds[ i ] ); + } + } else + { + int seed; + eo::mpi::Node::comm().recv( _masterRank, 1, seed ); + eo::log << eo::debug << eo::mpi::Node::comm().rank() << "- Seed: " << seed << std::endl; + eo::rng.reseed( seed ); + } + } + + MultiStartData* data() + { + return &_data; + } + + private: + MultiStartData< EOT > _data; + GetSeeds & _getSeeds; + int _masterRank; + }; + + template + // No seeds! Use default generator + struct DummyGetSeeds : public MultiStartStore::GetSeeds + { + std::vector operator()( int n ) + { + return std::vector(); + } + }; + + template + // Multiple of a seed + struct MultiplesOfNumber : public MultiStartStore::GetSeeds + { + MultiplesOfNumber ( int n = 0 ) + { + while( n == 0 ) + { + n = eo::rng.rand(); + } + _seed = n; + _i = 0; + } + + std::vector operator()( int n ) + { + std::vector ret; + for( unsigned int i = 0; i < n; ++i ) + { + ret.push_back( (++_i) * _seed ); + } + return ret; + } + + private: + + unsigned int _seed; + unsigned int _i; + }; + + template + struct GetRandomSeeds : public MultiStartStore::GetSeeds + { + GetRandomSeeds( int seed ) + { + eo::rng.reseed( seed ); + } + + std::vector operator()( int n ) + { + std::vector ret; + for(int i = 0; i < n; ++i) + { + ret.push_back( eo::rng.rand() ); + } + return ret; + } + }; + + template + struct ReuseOriginalPopEA: public MultiStartStore::ResetAlgo + { + ReuseOriginalPopEA( + eoGenContinue & continuator, + const eoPop& originalPop, + eoEvalFunc& eval) : + _initial( continuator.totalGenerations() ), + _continuator( continuator ), + _originalPop( originalPop ), + _eval( eval ) + { + // empty + } + + void operator()( eoPop& pop ) + { + pop = _originalPop; + for(unsigned i = 0, size = pop.size(); i < size; ++i) + { + _eval( pop[i] ); + } + _continuator.totalGenerations( _initial ); + } + + private: + unsigned int _initial; + eoGenContinue & _continuator; + const eoPop& _originalPop; + eoEvalFunc& _eval; + }; + + template< class EOT > + struct ReuseSamePopEA : public MultiStartStore::ResetAlgo + { + ReuseSamePopEA( + eoGenContinue& continuator, + const eoPop& originalPop, + eoEvalFunc& eval + ) : + _continuator( continuator ), + _originalPop( originalPop ), + _firstTime( true ), + _initial( continuator.totalGenerations() ) + { + for( unsigned i = 0, size = originalPop.size(); + i < size; ++i ) + { + eval(_originalPop[i]); + } + } + + void operator()( eoPop& pop ) + { + if( _firstTime ) + { + pop = _originalPop; + _firstTime = false; + } + _continuator.totalGenerations( _initial ); + } + + protected: + + eoGenContinue& _continuator; + eoPop _originalPop; + bool _firstTime; + int _initial; + }; + + template< class EOT > + class MultiStart : public OneShotJob< MultiStartData< EOT > > + { + public: + + MultiStart( AssignmentAlgorithm & algo, + int masterRank, + MultiStartStore< EOT > & store, + // dynamic parameters + int runs, + const std::vector& seeds = std::vector() ) : + OneShotJob< MultiStartData< EOT > >( algo, masterRank, store ) + { + store.init( algo.idles(), runs ); + } + + eoPop& best_individuals() + { + return this->store.data()->bests; + } + }; + + } // namespace mpi + +} // namespace eo + +# endif // __EO_MULTISTART_H__ diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index 91db72e3a..a50c17a68 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -1,4 +1,4 @@ -# include +# include using namespace eo::mpi; #include @@ -76,306 +76,6 @@ double real_value(const Indi & _indi) } /************************** PARALLELIZATION JOB *******************************/ - -template< class EOT > -struct MultiStartData -{ - typedef eoUF< eoPop&, void> ResetAlgo; - - MultiStartData( mpi::communicator& _comm, eoAlgo& _algo, int _masterRank, ResetAlgo & _resetAlgo ) - : - runs( 0 ), pop(), bests(), - comm( _comm ), algo( _algo ), masterRank( _masterRank ), resetAlgo( _resetAlgo ) - { - // empty - } - - // dynamic parameters - int runs; - eoPop< EOT > bests; - eoPop< EOT > pop; - - // static parameters - mpi::communicator& comm; - eoAlgo& algo; - ResetAlgo& resetAlgo; - int masterRank; -}; - -template< class EOT > -class SendTaskMultiStart : public SendTaskFunction< MultiStartData< EOT > > -{ - public: - using SendTaskFunction< MultiStartData< EOT > >::_data; - - void operator()( int wrkRank ) - { - --(_data->runs); - } -}; - -template< class EOT > -class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< EOT > > -{ - public: - using HandleResponseFunction< MultiStartData< EOT > >::_data; - - void operator()( int wrkRank ) - { - EOT individual; - MultiStartData< EOT >& d = *_data; - d.comm.recv( wrkRank, 1, individual ); - d.bests.push_back( individual ); - } -}; - -template< class EOT > -class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT > > -{ - public: - using ProcessTaskFunction< MultiStartData >::_data; - - void operator()() - { - _data->resetAlgo( _data->pop ); - _data->algo( _data->pop ); - _data->comm.send( _data->masterRank, 1, _data->pop.best_element() ); - } -}; - -template< class EOT > -class IsFinishedMultiStart : public IsFinishedFunction< MultiStartData< EOT > > -{ - public: - using IsFinishedFunction< MultiStartData< EOT > >::_data; - - bool operator()() - { - return _data->runs <= 0; - } -}; - -template< class EOT > -class MultiStartStore : public JobStore< MultiStartData< EOT > > -{ - public: - - typedef typename MultiStartData::ResetAlgo ResetAlgo; - typedef eoUF< int, std::vector > GetSeeds; - - MultiStartStore( - eoAlgo & algo, - int masterRank, - ResetAlgo & resetAlgo, - GetSeeds & getSeeds - ) - : _data( Node::comm(), algo, masterRank, resetAlgo ), - _getSeeds( getSeeds ), - _masterRank( masterRank ) - { - this->_iff = new IsFinishedMultiStart< EOT >; - this->_iff->needDelete(true); - this->_stf = new SendTaskMultiStart< EOT >; - this->_stf->needDelete(true); - this->_hrf = new HandleResponseMultiStart< EOT >; - this->_hrf->needDelete(true); - this->_ptf = new ProcessTaskMultiStart< EOT >; - this->_ptf->needDelete(true); - } - - void init( const std::vector& workers, int runs ) - { - _data.runs = runs; - - int nbWorkers = workers.size(); - std::vector< int > seeds = _getSeeds( nbWorkers ); - if( Node::comm().rank() == _masterRank ) - { - if( seeds.size() < nbWorkers ) - { - // Random seeds - for( int i = seeds.size(); i < nbWorkers; ++i ) - { - seeds.push_back( eo::rng.rand() ); - } - } - - for( int i = 0 ; i < nbWorkers ; ++i ) - { - int wrkRank = workers[i]; - Node::comm().send( wrkRank, 1, seeds[ i ] ); - } - } else - { - int seed; - Node::comm().recv( _masterRank, 1, seed ); - eo::log << eo::debug << Node::comm().rank() << "- Seed: " << seed << std::endl; - eo::rng.reseed( seed ); - } - } - - MultiStartData* data() - { - return &_data; - } - - private: - MultiStartData< EOT > _data; - GetSeeds & _getSeeds; - int _masterRank; -}; - -template< class EOT > -class MultiStart : public OneShotJob< MultiStartData< EOT > > -{ - public: - - MultiStart( AssignmentAlgorithm & algo, - int masterRank, - MultiStartStore< EOT > & store, - // dynamic parameters - int runs, - const std::vector& seeds = std::vector() ) : - OneShotJob< MultiStartData< EOT > >( algo, masterRank, store ) - { - store.init( algo.idles(), runs ); - } - - eoPop& best_individuals() - { - return this->store.data()->bests; - } -}; - -template -// No seeds! Use default generator -struct DummyGetSeeds : public MultiStartStore::GetSeeds -{ - std::vector operator()( int n ) - { - return std::vector(); - } -}; - -template -// Multiple of a seed -struct MultiplesOfNumber : public MultiStartStore::GetSeeds -{ - MultiplesOfNumber ( int n = 0 ) - { - while( n == 0 ) - { - n = eo::rng.rand(); - } - _seed = n; - _i = 0; - } - - std::vector operator()( int n ) - { - std::vector ret; - for( unsigned int i = 0; i < n; ++i ) - { - ret.push_back( (++_i) * _seed ); - } - return ret; - } - - private: - - unsigned int _seed; - unsigned int _i; -}; - -template -struct GetRandomSeeds : public MultiStartStore::GetSeeds -{ - GetRandomSeeds( int seed ) - { - eo::rng.reseed( seed ); - } - - std::vector operator()( int n ) - { - std::vector ret; - for(int i = 0; i < n; ++i) - { - ret.push_back( eo::rng.rand() ); - } - return ret; - } -}; - -template -struct ReuseOriginalPopEA: public MultiStartStore::ResetAlgo -{ - ReuseOriginalPopEA( - eoGenContinue & continuator, - const eoPop& originalPop, - eoEvalFunc& eval) : - _initial( continuator.totalGenerations() ), - _continuator( continuator ), - _originalPop( originalPop ), - _eval( eval ) - { - // empty - } - - void operator()( eoPop& pop ) - { - pop = _originalPop; - for(unsigned i = 0, size = pop.size(); i < size; ++i) - { - _eval( pop[i] ); - } - _continuator.totalGenerations( _initial ); - } - - private: - unsigned int _initial; - eoGenContinue & _continuator; - const eoPop& _originalPop; - eoEvalFunc& _eval; -}; - -template< class EOT > -struct ReuseSamePopEA : public MultiStartStore::ResetAlgo -{ - ReuseSamePopEA( - eoGenContinue& continuator, - const eoPop& originalPop, - eoEvalFunc& eval - ) : - _continuator( continuator ), - _originalPop( originalPop ), - _firstTime( true ), - _initial( continuator.totalGenerations() ) - { - for( unsigned i = 0, size = originalPop.size(); - i < size; ++i ) - { - eval(_originalPop[i]); - } - } - - void operator()( eoPop& pop ) - { - if( _firstTime ) - { - pop = _originalPop; - _firstTime = false; - } - _continuator.totalGenerations( _initial ); - } - - protected: - - eoGenContinue& _continuator; - eoPop _originalPop; - bool _firstTime; - int _initial; -}; - int main(int argc, char **argv) { Node::init( argc, argv ); From a14526e01af4f349293e1e7ea44159477216646d Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 15:12:54 +0200 Subject: [PATCH 25/31] eoGenContinue and eoSteadyFitContinue now inherits from a common class eoCountContinue, which contains a overridable method reset. --- eo/src/eoContinue.h | 39 ++++++++++++++++++++++++++++++++++++ eo/src/eoGenContinue.h | 23 +++++++++------------ eo/src/eoSteadyFitContinue.h | 21 ++++++++++--------- 3 files changed, 60 insertions(+), 23 deletions(-) diff --git a/eo/src/eoContinue.h b/eo/src/eoContinue.h index dbbf24805..82544115b 100644 --- a/eo/src/eoContinue.h +++ b/eo/src/eoContinue.h @@ -67,4 +67,43 @@ public: } }; +/** + * Termination condition with a count condition (totalGenerations). This continuator contains + * a count of cycles, which can be retrieved or set. + * + * @ingroup Continuators + * @ingroup Core + */ +template< class EOT > +class eoCountContinue : public eoContinue< EOT > +{ + public: + + eoCountContinue( ) : + thisGenerationPlaceholder( 0 ), + thisGeneration( thisGenerationPlaceholder ) + { + // empty + } + + eoCountContinue( unsigned long& currentGen ) : + thisGenerationPlaceholder( 0 ), + thisGeneration( currentGen ) + { + // empty + } + + virtual std::string className( void ) const { return "eoCountContinue"; } + + virtual void reset( ) + { + thisGeneration = 0; + } + + protected: + + unsigned long thisGenerationPlaceholder; + unsigned long& thisGeneration; +}; + #endif diff --git a/eo/src/eoGenContinue.h b/eo/src/eoGenContinue.h index 0d01bb0a1..dc756795d 100644 --- a/eo/src/eoGenContinue.h +++ b/eo/src/eoGenContinue.h @@ -35,24 +35,24 @@ @ingroup Continuators */ template< class EOT> -class eoGenContinue: public eoContinue, public eoValueParam +class eoGenContinue: public eoCountContinue, public eoValueParam { public: + using eoCountContinue::thisGeneration; + using eoCountContinue::thisGenerationPlaceholder; + /// Ctor for setting a eoGenContinue( unsigned long _totalGens) - : eoValueParam(0, "Generations", "Generations"), - repTotalGenerations( _totalGens ), - thisGenerationPlaceHolder(0), - thisGeneration(thisGenerationPlaceHolder) + : eoCountContinue( ), + eoValueParam(0, "Generations", "Generations"), + repTotalGenerations( _totalGens ) {}; /// Ctor for enabling the save/load the no. of generations counted eoGenContinue( unsigned long _totalGens, unsigned long& _currentGen) - : eoValueParam(0, "Generations", "Generations"), - repTotalGenerations( _totalGens ), - thisGenerationPlaceHolder(0), - thisGeneration(_currentGen) + : eoCountContinue( _currentGen ), eoValueParam(0, "Generations", "Generations"), + repTotalGenerations( _totalGens ) {}; /** Returns false when a certain number of generations is @@ -77,7 +77,7 @@ public: */ virtual void totalGenerations( unsigned long _tg ) { repTotalGenerations = _tg; - thisGeneration = 0; + eoCountContinue::reset(); }; /** Returns the number of generations to reach*/ @@ -86,7 +86,6 @@ public: return repTotalGenerations; }; - virtual std::string className(void) const { return "eoGenContinue"; } /** Read from a stream @@ -107,8 +106,6 @@ public: private: unsigned long repTotalGenerations; - unsigned long thisGenerationPlaceHolder; - unsigned long& thisGeneration; }; #endif diff --git a/eo/src/eoSteadyFitContinue.h b/eo/src/eoSteadyFitContinue.h index 229d93bc1..f05697b50 100644 --- a/eo/src/eoSteadyFitContinue.h +++ b/eo/src/eoSteadyFitContinue.h @@ -35,23 +35,26 @@ @ingroup Continuators */ template< class EOT> -class eoSteadyFitContinue: public eoContinue +class eoSteadyFitContinue: public eoCountContinue { public: typedef typename EOT::Fitness Fitness; + using eoCountContinue::thisGenerationPlaceholder; + using eoCountContinue::thisGeneration; + /// Ctor for setting a eoSteadyFitContinue( unsigned long _minGens, unsigned long _steadyGens) - : repMinGenerations( _minGens ), repSteadyGenerations( _steadyGens), - steadyState(false), thisGenerationPlaceHolder(0), - thisGeneration(thisGenerationPlaceHolder){}; + : eoCountContinue( ), repMinGenerations( _minGens ), repSteadyGenerations( _steadyGens), + steadyState(false) + {}; /// Ctor for enabling the save/load the no. of generations counted eoSteadyFitContinue( unsigned long _minGens, unsigned long _steadyGen, unsigned long& _currentGen) - : repMinGenerations( _minGens ), repSteadyGenerations( _steadyGen), - steadyState(_currentGen>_minGens), thisGenerationPlaceHolder(0), - thisGeneration(_currentGen){}; + : eoCountContinue( _currentGen ), repMinGenerations( _minGens ), repSteadyGenerations( _steadyGen), + steadyState(_currentGen>_minGens) + {}; /** Returns false when a certain number of generations is * reached withtout improvement */ @@ -96,7 +99,7 @@ public: /// Resets the state after it's been reached virtual void reset () { steadyState=false; - thisGeneration = 0; + eoCountContinue::reset(); } /** accessors*/ @@ -110,8 +113,6 @@ private: unsigned long repMinGenerations; unsigned long repSteadyGenerations; bool steadyState; - unsigned long thisGenerationPlaceHolder; - unsigned long& thisGeneration; unsigned int lastImprovement; Fitness bestSoFar; }; From 2ca5483f5df291940311a4d0f6637db8b84c43e3 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 15:13:36 +0200 Subject: [PATCH 26/31] MPI Multistart: using eoCountContinue in resetAlgo functors instead of eoGenContinue. --- eo/src/mpi/eoMultiStart.h | 18 +++++++----------- eo/test/mpi/t-mpi-multistart.cpp | 1 + 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/eo/src/mpi/eoMultiStart.h b/eo/src/mpi/eoMultiStart.h index 456124f11..a58ba2e91 100644 --- a/eo/src/mpi/eoMultiStart.h +++ b/eo/src/mpi/eoMultiStart.h @@ -219,10 +219,9 @@ namespace eo struct ReuseOriginalPopEA: public MultiStartStore::ResetAlgo { ReuseOriginalPopEA( - eoGenContinue & continuator, + eoCountContinue & continuator, const eoPop& originalPop, eoEvalFunc& eval) : - _initial( continuator.totalGenerations() ), _continuator( continuator ), _originalPop( originalPop ), _eval( eval ) @@ -237,12 +236,11 @@ namespace eo { _eval( pop[i] ); } - _continuator.totalGenerations( _initial ); + _continuator.reset(); } private: - unsigned int _initial; - eoGenContinue & _continuator; + eoCountContinue & _continuator; const eoPop& _originalPop; eoEvalFunc& _eval; }; @@ -251,14 +249,13 @@ namespace eo struct ReuseSamePopEA : public MultiStartStore::ResetAlgo { ReuseSamePopEA( - eoGenContinue& continuator, + eoCountContinue& continuator, const eoPop& originalPop, eoEvalFunc& eval ) : _continuator( continuator ), _originalPop( originalPop ), - _firstTime( true ), - _initial( continuator.totalGenerations() ) + _firstTime( true ) { for( unsigned i = 0, size = originalPop.size(); i < size; ++i ) @@ -274,15 +271,14 @@ namespace eo pop = _originalPop; _firstTime = false; } - _continuator.totalGenerations( _initial ); + _continuator.reset(); } protected: - eoGenContinue& _continuator; + eoCountContinue& _continuator; eoPop _originalPop; bool _firstTime; - int _initial; }; template< class EOT > diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index a50c17a68..4aa0fde37 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -162,6 +162,7 @@ int main(int argc, char **argv) ///////////////////////////////////// // stop after MAX_GEN generations eoGenContinue continuator(MAX_GEN); + // eoSteadyFitContinue< Indi > continuator( 10, 50 ); // GENERATION ///////////////////////////////////////// From 61c31a4a71e810d3fe2c9d69dc72db7cb5aa05e8 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 16:01:04 +0200 Subject: [PATCH 27/31] MPI Multistart: everybody loves comments, except the one who writes them. --- eo/src/mpi/eoMultiStart.h | 257 ++++++++++++++++++++++++++----- eo/test/mpi/t-mpi-multistart.cpp | 103 ++++--------- 2 files changed, 252 insertions(+), 108 deletions(-) diff --git a/eo/src/mpi/eoMultiStart.h b/eo/src/mpi/eoMultiStart.h index a58ba2e91..5e01f52a9 100644 --- a/eo/src/mpi/eoMultiStart.h +++ b/eo/src/mpi/eoMultiStart.h @@ -4,10 +4,50 @@ # include # include "eoMpi.h" +/** + * @ingroup MPI + * @{ + */ + +/** + * @file eoMultiStart.h + * + * Contains implementation of a MPI job which consists in a multi start, which basically consists in the following: + * the same eoAlgo is launched on computers of a clusters, with different seeds for each. As the eoAlgo are most of + * the time stochastics, the results won't be the same. It is fully equivalent to launch the same program but with + * different seeds. + * + * It follows the structure of a MPI job, as described in eoMpi.h. The basic algorithm is trivial: + * - Loop while we have a run to perform. + * - Worker performs runs and send their best solution (individual with best fitness) to the master. + * - Master retrieves the best solution and adds it to a eoPop of best solutions (the user can chooses what he does + * with this population, for instance: retrieve the best element, etc.) + * + * The principal concerns about this algorithm are: + * - How do we reinitialize the algorithm? An eoAlgo can have several forms, and initializations have to be performed + * before each "start". We can hence decide whether we reinits the population or keep the same population obtained + * after the previous start, we have to reinitialize continuator, etc. This is customizable in the store. + * + * - Which seeds should be chosen? If we want the run to be re-runnable with the same results, we need to be sure that + * the seeds are the same. But user can not care about this, and just want random seeds. This is customizable in the + * store. + * + * These concerns are handled by functors, inheriting from MultiStartStore::ResetAlgo (for the first concern), and + * MultiStartStore::GetSeeds (for the second one). There are default implementations, but there is no problem about + * specializing them or coding your own, by directly inheriting from them. + * + * @ingroup MPI + */ + namespace eo { namespace mpi { + /** + * @brief Data used by the Multi Start job. + * + * This data is shared between the different Job functors. More details are given for each attribute. + */ template< class EOT > struct MultiStartData { @@ -22,17 +62,49 @@ namespace eo } // dynamic parameters + /** + * @brief Total remaining number of runs. + * + * It's decremented as the runs are performed. + */ int runs; + + /** + * @brief eoPop of the best individuals, which are the one sent by the workers. + */ eoPop< EOT > bests; + + /** + * @brief eoPop on which the worker is working. + */ eoPop< EOT > pop; // static parameters + /** + * @brief Communicator, used to send and retrieve messages. + */ bmpi::communicator& comm; + + /** + * @brief Algorithm which will be performed by the worker. + */ eoAlgo& algo; + + /** + * @brief Reset Algo functor, which defines how to reset the algo (above) before re running it. + */ ResetAlgo& resetAlgo; + + // Rank of master int masterRank; }; + /** + * @brief Send task (master side) in the Multi Start job. + * + * It only consists in decrementing the number of runs, as the worker already have the population and + * all the necessary parameters to run the eoAlgo. + */ template< class EOT > class SendTaskMultiStart : public SendTaskFunction< MultiStartData< EOT > > { @@ -41,10 +113,17 @@ namespace eo void operator()( int wrkRank ) { + wrkRank++; // unused --(_data->runs); } }; + /** + * @brief Handle Response (master side) in the Multi Start job. + * + * It consists in retrieving the best solution sent by the worker and adds it to a population of best + * solutions. + */ template< class EOT > class HandleResponseMultiStart : public HandleResponseFunction< MultiStartData< EOT > > { @@ -60,6 +139,12 @@ namespace eo } }; + /** + * @brief Process Task (worker side) in the Multi Start job. + * + * Consists in resetting the algorithm and launching it on the population, then + * send the best individual (the one with the best fitness) to the master. + */ template< class EOT > class ProcessTaskMultiStart : public ProcessTaskFunction< MultiStartData< EOT > > { @@ -74,6 +159,11 @@ namespace eo } }; + /** + * @brief Is Finished (master side) in the Multi Start job. + * + * The job is finished if and only if all the runs have been performed. + */ template< class EOT > class IsFinishedMultiStart : public IsFinishedFunction< MultiStartData< EOT > > { @@ -86,14 +176,41 @@ namespace eo } }; + /** + * @brief Store for the Multi Start job. + * + * Contains the data used by the workers (algo,...) and functor to + * send the seeds. + */ template< class EOT > class MultiStartStore : public JobStore< MultiStartData< EOT > > { public: + /** + * @brief Generic functor to reset an algorithm before it's launched by + * the worker. + * + * This reset algorithm should reinits population (if necessary), continuator, etc. + */ typedef typename MultiStartData::ResetAlgo ResetAlgo; + + /** + * @brief Generic functor which returns a vector of seeds for the workers. + * + * If this vector hasn't enough seeds to send, random ones are generated and + * sent to the workers. + */ typedef eoUF< int, std::vector > GetSeeds; + /** + * @brief Default ctor for MultiStartStore. + * + * @param algo The algorithm to launch in parallel + * @param masterRank The MPI rank of the master + * @param resetAlgo The ResetAlgo functor + * @param getSeeds The GetSeeds functor + */ MultiStartStore( eoAlgo & algo, int masterRank, @@ -103,17 +220,27 @@ namespace eo : _data( eo::mpi::Node::comm(), algo, masterRank, resetAlgo ), _getSeeds( getSeeds ), _masterRank( masterRank ) - { - this->_iff = new IsFinishedMultiStart< EOT >; - this->_iff->needDelete(true); - this->_stf = new SendTaskMultiStart< EOT >; - this->_stf->needDelete(true); - this->_hrf = new HandleResponseMultiStart< EOT >; - this->_hrf->needDelete(true); - this->_ptf = new ProcessTaskMultiStart< EOT >; - this->_ptf->needDelete(true); - } + { + // Default job functors for this one. + this->_iff = new IsFinishedMultiStart< EOT >; + this->_iff->needDelete(true); + this->_stf = new SendTaskMultiStart< EOT >; + this->_stf->needDelete(true); + this->_hrf = new HandleResponseMultiStart< EOT >; + this->_hrf->needDelete(true); + this->_ptf = new ProcessTaskMultiStart< EOT >; + this->_ptf->needDelete(true); + } + /** + * @brief Send new seeds to the workers before a job. + * + * Uses the GetSeeds functor given in constructor. If there's not + * enough seeds to send, random seeds are sent to the workers. + * + * @param workers Vector of MPI ranks of the workers + * @param runs The number of runs to perform + */ void init( const std::vector& workers, int runs ) { _data.runs = runs; @@ -156,8 +283,51 @@ namespace eo int _masterRank; }; + /** + * @brief MultiStart job, created for convenience. + * + * This is an OneShotJob, which means workers leave it along with + * the master. + */ + template< class EOT > + class MultiStart : public OneShotJob< MultiStartData< EOT > > + { + public: + + MultiStart( AssignmentAlgorithm & algo, + int masterRank, + MultiStartStore< EOT > & store, + // dynamic parameters + int runs, + const std::vector& seeds = std::vector() ) : + OneShotJob< MultiStartData< EOT > >( algo, masterRank, store ) + { + store.init( algo.idles(), runs ); + } + + /** + * @brief Returns the best solution, at the end of the job. + * + * Warning: if you call this function from a worker, or from the master before the + * launch of the job, you will only get an empty population! + * + * @return Population of best individuals retrieved by the master. + */ + eoPop& best_individuals() + { + return this->store.data()->bests; + } + }; + + /************************************* + * DEFAULT GET SEEDS IMPLEMENTATIONS * + ************************************/ + + /** + * @brief Uses the internal default seed generator to get seeds, + * which means: random seeds are sent. + */ template - // No seeds! Use default generator struct DummyGetSeeds : public MultiStartStore::GetSeeds { std::vector operator()( int n ) @@ -166,8 +336,14 @@ namespace eo } }; + /** + * @brief Sends seeds to the workers, which are multiple of a number + * given by the master. If no number is given, a random one is used. + * + * This functor ensures that even if the same store is used with + * different jobs, the seeds will be different. + */ template - // Multiple of a seed struct MultiplesOfNumber : public MultiStartStore::GetSeeds { MultiplesOfNumber ( int n = 0 ) @@ -196,6 +372,10 @@ namespace eo unsigned int _i; }; + /** + * @brief Returns random seeds to the workers. We can controle which seeds are generated + * by precising the seed of the master. + */ template struct GetRandomSeeds : public MultiStartStore::GetSeeds { @@ -215,6 +395,17 @@ namespace eo } }; + /************************************** + * DEFAULT RESET ALGO IMPLEMENTATIONS * + ************************************** + + /** + * @brief For a Genetic Algorithm, reinits the population by copying the original one + * given in constructor, and reinits the continuator. + * + * The evaluator should also be given, as the population needs to be evaluated + * before each run. + */ template struct ReuseOriginalPopEA: public MultiStartStore::ResetAlgo { @@ -231,7 +422,7 @@ namespace eo void operator()( eoPop& pop ) { - pop = _originalPop; + pop = _originalPop; // copies the original population for(unsigned i = 0, size = pop.size(); i < size; ++i) { _eval( pop[i] ); @@ -245,6 +436,16 @@ namespace eo eoEvalFunc& _eval; }; + /** + * @brief For a Genetic Algorithm, reuses the same population without + * modifying it after a run. + * + * This means, if you launch a run after another one, you'll make evolve + * the same population. + * + * The evaluator should also be sent, as the population needs to be evaluated + * at the first time. + */ template< class EOT > struct ReuseSamePopEA : public MultiStartStore::ResetAlgo { @@ -279,32 +480,12 @@ namespace eo eoCountContinue& _continuator; eoPop _originalPop; bool _firstTime; - }; - - template< class EOT > - class MultiStart : public OneShotJob< MultiStartData< EOT > > - { - public: - - MultiStart( AssignmentAlgorithm & algo, - int masterRank, - MultiStartStore< EOT > & store, - // dynamic parameters - int runs, - const std::vector& seeds = std::vector() ) : - OneShotJob< MultiStartData< EOT > >( algo, masterRank, store ) - { - store.init( algo.idles(), runs ); - } - - eoPop& best_individuals() - { - return this->store.data()->bests; - } - }; - + } } // namespace mpi - } // namespace eo +/** + * @} + */ + # endif // __EO_MULTISTART_H__ diff --git a/eo/test/mpi/t-mpi-multistart.cpp b/eo/test/mpi/t-mpi-multistart.cpp index 4aa0fde37..fb614e8ee 100644 --- a/eo/test/mpi/t-mpi-multistart.cpp +++ b/eo/test/mpi/t-mpi-multistart.cpp @@ -8,9 +8,24 @@ using namespace eo::mpi; #include #include -// Use functions from namespace std +/* + * This file is based on the tutorial lesson 1. We'll consider that you know all the EO + * related parts of the algorithm and we'll focus our attention on parallelization. + * + * This file shows an example of multistart applied to a eoSGA (simple genetic + * algorithm). As individuals need to be serialized, we implement a class inheriting + * from eoReal (which is the base individual), so as to manipulate individuals as they + * were eoReal AND serialize them. + * + * The main function shows how to launch a multistart job, with default functors. If you + * don't know which functors to use, these ones should fit the most of your purposes. + */ + using namespace std; +/* + * eoReal is a vector of double: we just have to serializes the value and the fitness. + */ class SerializableEOReal: public eoReal, public eoserial::Persistent { public: @@ -91,99 +106,48 @@ int main(int argc, char **argv) const double EPSILON = 0.01; // range for real uniform mutation const float MUT_RATE = 0.5; // mutation rate - // GENERAL - ////////////////////////// - // Random seed - ////////////////////////// - //reproducible random seed: if you don't change SEED above, - // you'll aways get the same result, NOT a random run - // rng.reseed(SEED); - - // EVAL - ///////////////////////////// - // Fitness function - //////////////////////////// - // Evaluation: from a plain C++ fn to an EvalFunc Object eoEvalFuncPtr eval( real_value ); - - // INIT - //////////////////////////////// - // Initilisation of population - //////////////////////////////// - - // declare the population eoPop pop; - // fill it! - /* - for (unsigned int igeno=0; igeno generator; eoInitFixedLength< Indi > init( VEC_SIZE, generator ); - // eoInitAndEval< Indi > init( real_init, eval, continuator ); pop = eoPop( POP_SIZE, init ); - // ENGINE - ///////////////////////////////////// - // selection and replacement - //////////////////////////////////// - // SELECT - // The robust tournament selection - eoDetTournamentSelect select(T_SIZE); // T_SIZE in [2,POP_SIZE] - - // REPLACE - // eoSGA uses generational replacement by default - // so no replacement procedure has to be given - - // OPERATORS - ////////////////////////////////////// - // The variation operators - ////////////////////////////////////// - // CROSSOVER - // offspring(i) is a linear combination of parent(i) + eoDetTournamentSelect select(T_SIZE); eoSegmentCrossover xover; - // MUTATION - // offspring(i) uniformly chosen in [parent(i)-epsilon, parent(i)+epsilon] eoUniformMutation mutation(EPSILON); - // STOP - // CHECKPOINT - ////////////////////////////////////// - // termination condition - ///////////////////////////////////// - // stop after MAX_GEN generations eoGenContinue continuator(MAX_GEN); + /* Does work too with a steady fit continuator. */ // eoSteadyFitContinue< Indi > continuator( 10, 50 ); - // GENERATION - ///////////////////////////////////////// - // the algorithm - //////////////////////////////////////// - // standard Generational GA requires - // selection, evaluation, crossover and mutation, stopping criterion - eoSGA gga(select, xover, CROSS_RATE, mutation, MUT_RATE, eval, continuator); + /* How to assign tasks, which are starts? */ DynamicAssignmentAlgorithm assignmentAlgo; + /* Before a worker starts its algorithm, how does it reinits the population? + * There are a few default usable functors, defined in eoMultiStart.h. + * + * This one (ReuseSamePopEA) doesn't modify the population after a start, so + * the same population is reevaluated on each multistart: the solution tend + * to get better and better. + */ ReuseSamePopEA< Indi > resetAlgo( continuator, pop, eval ); + /** + * How to send seeds to the workers, at the beginning of the parallel job? + * This functors indicates that seeds should be random values. + */ GetRandomSeeds< Indi > getSeeds( SEED ); + // Builds the store MultiStartStore< Indi > store( gga, DEFAULT_MASTER, resetAlgo, getSeeds); + // Creates the multistart job and runs it. + // The last argument indicates that we want to launch 5 runs. MultiStart< Indi > msjob( assignmentAlgo, DEFAULT_MASTER, store, 5 ); msjob.run(); @@ -202,5 +166,4 @@ int main(int argc, char **argv) std::cout << "Global best individual has fitness " << msjob10.best_individuals().best_element().fitness() << std::endl; } return 0; - } From 9eb54d16bb8629652b97ab2c4313dc5f80b42a7c Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 16:02:20 +0200 Subject: [PATCH 28/31] MPI Multistart: oups I forgot a semi colon. --- eo/src/mpi/eoMultiStart.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eo/src/mpi/eoMultiStart.h b/eo/src/mpi/eoMultiStart.h index 5e01f52a9..619364942 100644 --- a/eo/src/mpi/eoMultiStart.h +++ b/eo/src/mpi/eoMultiStart.h @@ -480,7 +480,7 @@ namespace eo eoCountContinue& _continuator; eoPop _originalPop; bool _firstTime; - } + }; } // namespace mpi } // namespace eo From 8fb0d5509cf29df52f6710a89076bd286b2d633d Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 16:16:14 +0200 Subject: [PATCH 29/31] Bugfix: when retrieving the best element of an empty population eoPop, segfault. Fixed by launching an exception in this case. --- eo/src/eoPop.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eo/src/eoPop.h b/eo/src/eoPop.h index 3ea53f466..371fdfd01 100644 --- a/eo/src/eoPop.h +++ b/eo/src/eoPop.h @@ -201,6 +201,8 @@ public: #else typename eoPop::const_iterator it = std::max_element(begin(), end()); #endif + if( it == end() ) + throw std::runtime_error("eoPop: Empty population, when calling best_element()."); return (*it); } From 7b5c98f3a6c7ac9de667935397ecbd05bd235451 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Thu, 26 Jul 2012 16:17:07 +0200 Subject: [PATCH 30/31] Multistart example is automatically compiled when compiling MPI examples. --- eo/test/mpi/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/eo/test/mpi/CMakeLists.txt b/eo/test/mpi/CMakeLists.txt index f83ecd45d..22734d441 100644 --- a/eo/test/mpi/CMakeLists.txt +++ b/eo/test/mpi/CMakeLists.txt @@ -31,6 +31,7 @@ SET (TEST_LIST t-mpi-wrapper t-mpi-multipleRoles t-mpi-eval + t-mpi-multistart ) FOREACH (test ${TEST_LIST}) From cbaea85ec3d8dc08f71c50946aedf521ab1d8088 Mon Sep 17 00:00:00 2001 From: Johann Dreo Date: Thu, 26 Jul 2012 17:06:40 +0200 Subject: [PATCH 31/31] use mpicxx without a fixed path --- eo/src/CMakeLists.txt | 3 ++- eo/test/CMakeLists.txt | 2 +- eo/test/mpi/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eo/src/CMakeLists.txt b/eo/src/CMakeLists.txt index 6509908ae..317080f1e 100644 --- a/eo/src/CMakeLists.txt +++ b/eo/src/CMakeLists.txt @@ -11,7 +11,8 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) IF(WITH_MPI) MESSAGE("[EO] Compilation with MPI.") - SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") + #SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") + SET(CMAKE_CXX_COMPILER mpicxx) # headers location INCLUDE_DIRECTORIES(${MPI_DIR}/include) diff --git a/eo/test/CMakeLists.txt b/eo/test/CMakeLists.txt index e3a5a624b..9bfc296b4 100644 --- a/eo/test/CMakeLists.txt +++ b/eo/test/CMakeLists.txt @@ -26,7 +26,7 @@ LINK_DIRECTORIES(${EO_BINARY_DIR}/lib) IF(WITH_MPI) LINK_DIRECTORIES(${MPI_DIR}/lib) - SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") + SET(CMAKE_CXX_COMPILER "mpicxx") ENDIF() ###################################################################################### diff --git a/eo/test/mpi/CMakeLists.txt b/eo/test/mpi/CMakeLists.txt index 22734d441..d89f3ab3f 100644 --- a/eo/test/mpi/CMakeLists.txt +++ b/eo/test/mpi/CMakeLists.txt @@ -38,7 +38,7 @@ FOREACH (test ${TEST_LIST}) SET ("T_${test}_SOURCES" "${test}.cpp") ENDFOREACH (test) -SET(CMAKE_CXX_COMPILER "${MPI_DIR}/bin/mpicxx") +SET(CMAKE_CXX_COMPILER "mpicxx") ADD_DEFINITIONS(-DWITH_MPI) IF(ENABLE_CMAKE_TESTING)