From a3a04e011237358507ce68d20f425bb6fd6f839d Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 31 Aug 2012 22:37:47 -0400 Subject: [PATCH 01/26] Fixed comments. Now inputs in parser should be in milliseconds, second isn't a precise enough unit. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 4fad9d44..a6430dd0 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -46,9 +46,10 @@ class Distribution : public std::vector< type > */ virtual int next_element() = 0; - // Idea for function name: enlarge_your_parser /** * @brief Creates params and retrieves values from parser + * + * Parser's params should take milliseconds as inputs. */ virtual void make_parser( eoParser & parser ) = 0; @@ -56,7 +57,7 @@ class Distribution : public std::vector< type > * @brief Returns true if this distribution has been activated by the * command line. * - * Serves to main program to check if at least one distribution has been + * Used by the main program so as to check if at least one distribution has been * activated. */ bool isActive() { return _active; } @@ -78,13 +79,13 @@ class UniformDistribution : public Distribution void make_parser( eoParser & parser ) { _active = parser.createParam( false, "uniform", "Uniform distribution", '\0', "Uniform").value(); - _min = parser.createParam( 0.0, "uniform-min", "Minimum for uniform distribution", '\0', "Uniform").value(); - _max = parser.createParam( 1.0, "uniform-max", "Maximum for uniform distribution", '\0', "Uniform").value(); + _min = parser.createParam( 0.0, "uniform-min", "Minimum for uniform distribution, in ms.", '\0', "Uniform").value(); + _max = parser.createParam( 1.0, "uniform-max", "Maximum for uniform distribution, in ms.", '\0', "Uniform").value(); } int next_element() { - return std::floor( 1000. * _rng.uniform( _min, _max ) ); + return std::floor( _rng.uniform( _min, _max ) ); } protected: From 7c3eee4dddcacc8ed9ab12b3604dd8d907899f1c Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 31 Aug 2012 22:45:03 -0400 Subject: [PATCH 02/26] MPI distrib exp: added the normal Gaussian distribution --- eo/test/mpi/t-mpi-distrib-exp.cpp | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index a6430dd0..9391fa13 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -97,6 +97,58 @@ class UniformDistribution : public Distribution } uniformDistribution; +/** + * @brief Normal (gaussian) distribution of times. + * + * A normal distribution is defined by a mean and a standard deviation. + */ +class NormalDistribution : public Distribution +{ + public: + + NormalDistribution() : _rng( 0 ) + { + // empty + } + + ~NormalDistribution() + { + if( _rng ) + { + delete _rng; + } + } + + void make_parser( eoParser & parser ) + { + _active = parser.createParam( false, "normal", "Normal distribution", '\0', "Normal").value(); + _mean = parser.createParam( 0.0, "normal-mean", "Mean for the normal distribution (0 by default), in ms.", '\0', "Normal").value(); + double _stddev = parser.createParam( 1.0, "normal-stddev", "Standard deviation for the normal distribution (1ms by default), 0 isn't acceptable.", '\0', "Normal").value(); + + if( _active ) + { + _rng = new eoNormalGenerator( _stddev ); + } + } + + int next_element() + { + int next = std::floor( _mean + (*_rng)() ); + if( next < 0 ) + { + next = 0; + } + return next; + } + + protected: + + eoNormalGenerator * _rng; + + double _mean; + +} normalDistribution; + int main( int argc, char** argv ) { Node::init( argc, argv ); From c4af81caeb624ae985ec10b7e0c97a874af9fef6 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 31 Aug 2012 22:45:43 -0400 Subject: [PATCH 03/26] MPI distrib exp: added managment of multiples distributions. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 43 ++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 9391fa13..dadef89e 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -154,31 +154,54 @@ int main( int argc, char** argv ) Node::init( argc, argv ); eoParser parser( argc, argv ); - // TODO for each available distribution, check if activated. + // General parameters for the experimentation + unsigned size = parser.createParam( 10U, "size", "Number of elements to distribute.", 's', "Distribution").value(); + unsigned packet_size = parser.createParam( 1U, "packet_size", "Number of elements to distribute at each time for a single worker.", 'p', "Parallelization").value(); + + std::vector distribs; + distribs.push_back( &uniformDistribution ); + distribs.push_back( &normalDistribution ); + + // for each available distribution, check if activated. // If no distribution is activated, show an error message // If two distributions or more are activated, show an error message // Otherwise, use the activated distribution as distrib - Distribution & distrib = uniformDistribution; - // Make parser of distribution here - distrib.make_parser( parser ); - - unsigned size = parser.createParam( 10U, "size", "Number of elements to distribute.", 's', "Distribution").value(); - unsigned packet_size = parser.createParam( 1U, "packet_size", "Number of elements to distribute at each time for a single worker.", 'p', "Parallelization").value(); + bool isChosenDistrib = false; + Distribution* pdistrib = 0; + for( int i = 0, s = distribs.size(); i < s; ++i ) + { + distribs[i]->make_parser( parser ); + if( distribs[i]->isActive() ) + { + if( isChosenDistrib ) + { + throw std::runtime_error("Only one distribution can be chosen during a launch!"); + } else + { + isChosenDistrib = true; + pdistrib = distribs[i]; + } + } + } make_parallel( parser ); make_help( parser ); - ParallelApplyStore< type> store( wait, DEFAULT_MASTER, packet_size ); + if( !isChosenDistrib ) + { + throw std::runtime_error("No distribution chosen. One distribution should be chosen."); + } // Fill distribution + Distribution& distrib = *pdistrib; distrib.fill( size ); - store.data( distrib ); + ParallelApplyStore< type > store( wait, DEFAULT_MASTER, packet_size ); + store.data( distrib ); DynamicAssignmentAlgorithm scheduling; ParallelApply< type > job( scheduling, DEFAULT_MASTER, store ); job.run(); - if( job.isMaster() ) { EmptyJob( scheduling, DEFAULT_MASTER ); // to terminate parallel apply From b4650e040c1d06fe69b5da9ac041d68793ce14a0 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 7 Sep 2012 11:04:12 -0400 Subject: [PATCH 04/26] MPI Distrib exp: use of eoRng instead of eoUniformGenerator --- eo/test/mpi/t-mpi-distrib-exp.cpp | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index dadef89e..72e934e5 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -111,42 +111,23 @@ class NormalDistribution : public Distribution // empty } - ~NormalDistribution() - { - if( _rng ) - { - delete _rng; - } - } - void make_parser( eoParser & parser ) { _active = parser.createParam( false, "normal", "Normal distribution", '\0', "Normal").value(); _mean = parser.createParam( 0.0, "normal-mean", "Mean for the normal distribution (0 by default), in ms.", '\0', "Normal").value(); - double _stddev = parser.createParam( 1.0, "normal-stddev", "Standard deviation for the normal distribution (1ms by default), 0 isn't acceptable.", '\0', "Normal").value(); - - if( _active ) - { - _rng = new eoNormalGenerator( _stddev ); - } + _stddev = parser.createParam( 1.0, "normal-stddev", "Standard deviation for the normal distribution (1ms by default), 0 isn't acceptable.", '\0', "Normal").value(); } int next_element() { - int next = std::floor( _mean + (*_rng)() ); - if( next < 0 ) - { - next = 0; - } - return next; + return std::floor( _rng.normal( _mean, _stddev ) ); } protected: - eoNormalGenerator * _rng; - + eoRng _rng; double _mean; - + double _stddev; } normalDistribution; int main( int argc, char** argv ) From 211ac8141e2e3d2c6d63533af1f9b7d1774b0b92 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 7 Sep 2012 11:37:21 -0400 Subject: [PATCH 05/26] MPI Distrib exp: added exponential distribution --- eo/test/mpi/t-mpi-distrib-exp.cpp | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 72e934e5..bfee3d9f 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -130,6 +130,33 @@ class NormalDistribution : public Distribution double _stddev; } normalDistribution; +class ExponentialDistribution : public Distribution +{ + public: + + ExponentialDistribution() : _rng( 0 ) + { + // empty + } + + void make_parser( eoParser & parser ) + { + _active = parser.createParam( false, "exponential", "Exponential distribution", '\0', "Exponential").value(); + _mean = parser.createParam( 0.0, "exponential-mean", "Mean for the exponential distribution (0 by default), in ms.", '\0', "Exponential").value(); + } + + int next_element() + { + return std::floor( _rng.negexp( _mean ) ); + } + + protected: + + eoRng _rng; + double _mean; + +} exponentialDistribution; + int main( int argc, char** argv ) { Node::init( argc, argv ); @@ -142,6 +169,7 @@ int main( int argc, char** argv ) std::vector distribs; distribs.push_back( &uniformDistribution ); distribs.push_back( &normalDistribution ); + distribs.push_back( &exponentialDistribution ); // for each available distribution, check if activated. // If no distribution is activated, show an error message From f8ca95b352eff561fc166110e70e1ddfd01a48f5 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 7 Sep 2012 11:43:08 -0400 Subject: [PATCH 06/26] MPI Distrib exp: short relative path to t-mpi-common --- eo/test/mpi/t-mpi-distrib-exp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index bfee3d9f..1b390667 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -7,7 +7,7 @@ # include # include -# include "../test/mpi/t-mpi-common.h" +# include "t-mpi-common.h" using namespace eo::mpi; From 16cbf69d5d55a01a44161cd89003c7e6ee9e252a Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 7 Sep 2012 11:43:43 -0400 Subject: [PATCH 07/26] MPI Distrib exp: replace next generated waiting time by 0 if it's negative --- eo/test/mpi/t-mpi-distrib-exp.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 1b390667..e34170de 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -34,7 +34,9 @@ class Distribution : public std::vector< type > { for( unsigned i = 0; i < size; ++i ) { - push_back( next_element() ); + int next = next_element(); + if( next < 0 ) next = 0; + push_back( next ); } } From aa5e8bf6f32e6613f64d2b3a17e603431f7c0451 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Tue, 18 Sep 2012 20:50:26 -0400 Subject: [PATCH 08/26] Added method clean for eoTimerStat --- eo/src/utils/eoTimer.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eo/src/utils/eoTimer.h b/eo/src/utils/eoTimer.h index 4ff451b4..23d6df92 100644 --- a/eo/src/utils/eoTimer.h +++ b/eo/src/utils/eoTimer.h @@ -318,6 +318,14 @@ class eoTimerStat return _stats; } + /** + * @brief Empties the statistics map. + */ + void clear() + { + _stats.clear(); + } + protected: // Statistics map: links a key (string) to a statistic. std::map< std::string, Stat > _stats; From 243dd7424facb3c1cf3182ba64030f41a5d80164 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Tue, 18 Sep 2012 20:53:32 -0400 Subject: [PATCH 09/26] MPI Distrib exp: added license --- eo/test/mpi/t-mpi-distrib-exp.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index e34170de..505b80e5 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -1,3 +1,21 @@ +/* + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + * Authors: + * Benjamin Bouvier + */ # include // usleep # include From b5d44c809df23b820cfcf2f61b4435b127df25e6 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Tue, 18 Sep 2012 20:53:47 -0400 Subject: [PATCH 10/26] MPI Distrib exp: comments for each class. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 53 ++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 505b80e5..6b08bfdc 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -16,6 +16,19 @@ * Authors: * Benjamin Bouvier */ + +/** + * @file t-mpi-distrib-exp.cpp + * @brief File for parallel experimentations. + * + * When using parallel evaluation, the individuals to evaluate are sent by packets (group), + * so as to avoid that communication time be more important than worker's execution time. + * However, the ideal size of packet depends on the problem and the time needed to carry out + * the atomic operation on each individual. This experiment tries to find a relation between + * the total number of elements to process (size), the execution time and the size of packet. + * This could lead to an heuristic allowing to optimize the size of packet according to the + * processing times. + */ # include // usleep # include @@ -29,8 +42,14 @@ using namespace eo::mpi; +// Serializable int typedef SerializableBase type; +/* + * The task is the following: the worker receives a number of milliseconds to wait, which + * simulates the process of one individual. This way, the sequences of processing times are + * generated only by the master and are more easily reproductible. + */ struct Wait : public eoUF< type &, void > { void operator()( type & milliseconds ) @@ -41,6 +60,9 @@ struct Wait : public eoUF< type &, void > } } wait; +/** + * @brief Represents a distribution of processing times. + */ class Distribution : public std::vector< type > { public: @@ -62,7 +84,8 @@ class Distribution : public std::vector< type > * @brief Returns the next element of the distribution to put in the * vector. * - * @returns Number of milliseconds to wait + * @returns Number of milliseconds to wait. Can be negative ; in this case, + * the number will be truncated to 0ms. */ virtual int next_element() = 0; @@ -87,6 +110,19 @@ class Distribution : public std::vector< type > bool _active; }; +/** + * @brief Uniform distribution. + * + * This is an uniform distribution, defined by a minimum value and a maximum value. + * In the uniform distribution, every number from min to max has the same probability + * to appear. + * + * The 3 parameters activable from a parser are the following: + * - uniform=1 : if we want to use the uniform distribution + * - uniform-min=x : use x milliseconds as the minimum value of waiting time. + * - uniform-max=y : use y milliseconds as the maximum value of waiting time. + * Ensure that x < y, or the results are unpredictable. + */ class UniformDistribution : public Distribution { public: @@ -121,6 +157,10 @@ class UniformDistribution : public Distribution * @brief Normal (gaussian) distribution of times. * * A normal distribution is defined by a mean and a standard deviation. + * The 3 parameters activable from the parser are the following: + * - normal=1: activates the gaussian distribution. + * - normal-mean=50: use 50ms as the mean of the distribution. + * - normal-stddev=10: use 10ms as the standard deviation of the distribution. */ class NormalDistribution : public Distribution { @@ -150,6 +190,17 @@ class NormalDistribution : public Distribution double _stddev; } normalDistribution; +/** + * @brief Exponential distribution. + * + * This distribution belongs to the category of the decreasing power laws and are affected by long trails + * phenomenons. + * An exponential distribution is only defined by its mean. + * + * The 2 parameters activable from the parser are the following: + * - exponential=1: to activate the exponential distribution. + * - exponential-mean=50: indicates that the mean must be 50ms. + */ class ExponentialDistribution : public Distribution { public: From 9d4742c9953fb9f4b0085193e7b517a188d59530 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Tue, 18 Sep 2012 22:03:07 -0400 Subject: [PATCH 11/26] TimerStat: better precision for wallclock time. --- eo/src/utils/eoTimer.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/eo/src/utils/eoTimer.h b/eo/src/utils/eoTimer.h index 23d6df92..fbf90c21 100644 --- a/eo/src/utils/eoTimer.h +++ b/eo/src/utils/eoTimer.h @@ -22,7 +22,7 @@ Authors: # ifndef __EO_TIMER_H__ # define __EO_TIMER_H__ -# include // time() +# include // gettimeofday() # include // rusage() # include // std::vector @@ -61,7 +61,7 @@ class eoTimer */ void restart() { - wc_start = time(NULL); + gettimeofday( &wc_start, NULL ); getrusage( RUSAGE_SELF, &_start ); } @@ -138,7 +138,9 @@ class eoTimer */ double wallclock() { - return std::difftime( std::time(NULL) , wc_start ); + struct timeval wc_end; + gettimeofday( &wc_end, NULL ); + return ( wc_end.tv_sec - wc_start.tv_sec ) + ( wc_end.tv_usec - wc_start.tv_usec ) / 1000000.; } protected: @@ -149,7 +151,7 @@ class eoTimer // Remainder (in milliseconds) for system time. long int usremainder; // Structure used to measure wallclock time. - time_t wc_start; + struct timeval wc_start; }; /** From 748ea803523d1a047f3dc78c6c7b0d76b0ac6e2f Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Tue, 18 Sep 2012 22:03:43 -0400 Subject: [PATCH 12/26] MPI Distrib exp: retrieve statistics from workers and print them in standard output --- eo/test/mpi/t-mpi-distrib-exp.cpp | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 6b08bfdc..6f5b9db6 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -32,6 +32,7 @@ # include // usleep # include +# include # include # include @@ -231,11 +232,15 @@ class ExponentialDistribution : public Distribution int main( int argc, char** argv ) { Node::init( argc, argv ); + mpi::communicator& comm = eo::mpi::Node::comm(); eoParser parser( argc, argv ); + // forces the statistics to be retrieved + parser.setORcreateParam( true, "parallelize-do-measure", "Do some measures during execution" ); + // General parameters for the experimentation unsigned size = parser.createParam( 10U, "size", "Number of elements to distribute.", 's', "Distribution").value(); - unsigned packet_size = parser.createParam( 1U, "packet_size", "Number of elements to distribute at each time for a single worker.", 'p', "Parallelization").value(); + unsigned packet_size = parser.createParam( 1U, "packet-size", "Number of elements to distribute at each time for a single worker.", 'p', "Parallelization").value(); std::vector distribs; distribs.push_back( &uniformDistribution ); @@ -285,6 +290,32 @@ int main( int argc, char** argv ) if( job.isMaster() ) { EmptyJob( scheduling, DEFAULT_MASTER ); // to terminate parallel apply + // Receive statistics + typedef std::map< std::string, eoTimerStat::Stat > typeStats; + std::cout << std::fixed << std::setprecision( 5 ); + for( int i = 1, s = comm.size(); i < s; ++i ) + { + eoTimerStat timerStat; + comm.recv( i, eo::mpi::Channel::Commands, timerStat ); + typeStats stats = timerStat.stats(); + for( typeStats::iterator it = stats.begin(), + end = stats.end(); + it != end; + ++it ) + { + std::cout << "Worker " << i << ": Wallclock time of " << it->first << std::endl; + for( int j = 0, t = it->second.wtime.size(); j < t; ++j ) + { + std::cout << it->second.wtime[j] << " "; + } + std::cout << std::endl; + } + std::cout << std::endl; + } + } else + { + // Send statistics + comm.send( DEFAULT_MASTER, eo::mpi::Channel::Commands, eo::mpi::timerStat ); } return 0; From 2bc69b40779f2a3b41d5ecc385dc94628bc73996 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Tue, 18 Sep 2012 22:11:13 -0400 Subject: [PATCH 13/26] MPI Distrib exp: switch for choosing whether workers should print the waiting time or not. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 6f5b9db6..8f675023 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -53,13 +53,22 @@ typedef SerializableBase type; */ struct Wait : public eoUF< type &, void > { + Wait( bool print ) : _print( print ) + { + // empty + } + void operator()( type & milliseconds ) { - std::cout << "Sleeping for " << milliseconds << "ms..." << std::endl; + if( _print ) + std::cout << "Sleeping for " << milliseconds << "ms..." << std::endl; // usleep takes an input in microseconds usleep( milliseconds * 1000 ); } -} wait; + + private: + bool _print; +}; /** * @brief Represents a distribution of processing times. @@ -241,6 +250,7 @@ int main( int argc, char** argv ) // General parameters for the experimentation unsigned size = parser.createParam( 10U, "size", "Number of elements to distribute.", 's', "Distribution").value(); unsigned packet_size = parser.createParam( 1U, "packet-size", "Number of elements to distribute at each time for a single worker.", 'p', "Parallelization").value(); + bool worker_print_waiting_time = parser.createParam( false, "print-waiting-time", "Do the workers need to print the time they wait?", '\0', "Parallelization").value(); std::vector distribs; distribs.push_back( &uniformDistribution ); @@ -281,6 +291,7 @@ int main( int argc, char** argv ) Distribution& distrib = *pdistrib; distrib.fill( size ); + Wait wait( worker_print_waiting_time ); ParallelApplyStore< type > store( wait, DEFAULT_MASTER, packet_size ); store.data( distrib ); DynamicAssignmentAlgorithm scheduling; From a4ec17a1930f342bbd6e87ff0cda32374b099295 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Tue, 18 Sep 2012 22:18:35 -0400 Subject: [PATCH 14/26] MPI Distrib exp: added timer for main program. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 8f675023..a21cb11f 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -281,6 +281,7 @@ int main( int argc, char** argv ) make_parallel( parser ); make_help( parser ); + timerStat.start("main"); if( !isChosenDistrib ) { @@ -298,6 +299,8 @@ int main( int argc, char** argv ) ParallelApply< type > job( scheduling, DEFAULT_MASTER, store ); job.run(); + + timerStat.stop("main"); if( job.isMaster() ) { EmptyJob( scheduling, DEFAULT_MASTER ); // to terminate parallel apply From 95e5d7d8db703d9fca0f94d4d759b78009cfc67c Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 21:49:22 -0400 Subject: [PATCH 15/26] [MPI Distrib exp: distributions are now serializable. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 45 ++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index a21cb11f..1b5afbf9 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -73,7 +73,7 @@ struct Wait : public eoUF< type &, void > /** * @brief Represents a distribution of processing times. */ -class Distribution : public std::vector< type > +class Distribution : public std::vector< type >, public eoserial::Persistent { public: @@ -154,6 +154,21 @@ class UniformDistribution : public Distribution return std::floor( _rng.uniform( _min, _max ) ); } + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "name", eoserial::make( "uniform" ) ); + obj->add( "min", eoserial::make( _min ) ); + obj->add( "max", eoserial::make( _max ) ); + return obj; + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpack( *obj, "min", _min ); + eoserial::unpack( *obj, "max", _max ); + } + protected: eoRng _rng; @@ -193,6 +208,21 @@ class NormalDistribution : public Distribution return std::floor( _rng.normal( _mean, _stddev ) ); } + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "name", eoserial::make( "normal" ) ); + obj->add( "mean", eoserial::make( _mean ) ); + obj->add( "stddev", eoserial::make( _stddev ) ); + return obj; + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpack( *obj, "mean", _mean ); + eoserial::unpack( *obj, "stddev", _stddev ); + } + protected: eoRng _rng; @@ -231,6 +261,19 @@ class ExponentialDistribution : public Distribution return std::floor( _rng.negexp( _mean ) ); } + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "name", eoserial::make( "exponential" ) ); + obj->add( "mean", eoserial::make( _mean ) ); + return obj; + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpack( *obj, "mean", _mean ); + } + protected: eoRng _rng; From 6503f61521126418b9bb0d56d19e619f53d149ce Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 21:50:23 -0400 Subject: [PATCH 16/26] [MPI Distrib exp: serializable class Experiment added. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 161 ++++++++++++++++++++++-------- 1 file changed, 117 insertions(+), 44 deletions(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 1b5afbf9..43a7291a 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -281,10 +281,120 @@ class ExponentialDistribution : public Distribution } exponentialDistribution; +class Experiment : public eoserial::Persistent +{ + public: + + Experiment() : _distribution(0) + { + // empty + } + + Experiment( Distribution* distrib, unsigned size, unsigned packet_size, bool print_waiting_time ) : + _distribution( distrib ), + _size( size ), + _packet_size( packet_size ), + _worker_print_waiting_time( print_waiting_time ) + { + // empty + } + + eoserial::Object* pack( void ) const + { + eoserial::Object* obj = new eoserial::Object; + obj->add( "size", eoserial::make( _size ) ); + obj->add( "packet-size", eoserial::make( _packet_size ) ); + obj->add( "worker-print-waiting-time", eoserial::make( _worker_print_waiting_time ) ); + if( _distribution ) + { + obj->add( "distribution", _distribution ); + } + return obj; + } + + void unpack( const eoserial::Object* obj ) + { + eoserial::unpack( *obj, "size", _size ); + eoserial::unpack( *obj, "packet-size", _packet_size ); + eoserial::unpack( *obj, "worker-print-waiting-time", _worker_print_waiting_time ); + + eoserial::Object* distribObject = static_cast( obj->find("distribution")->second ); + std::string distribName = *static_cast( distribObject->find("name")->second ); + + // TODO find a better design... + if( distribName == "normal" ) { + _distribution = & normalDistribution; + } else if( distribName == "uniform" ) { + _distribution = & uniformDistribution; + } else if( distribName == "exponential" ) { + _distribution = & exponentialDistribution; + } else { + throw std::runtime_error("When unpacking experience, no distribution found."); + } + + eoserial::unpackObject( *obj, "distribution", *_distribution ); + } + + void run() + { + mpi::communicator& comm = eo::mpi::Node::comm(); + timerStat.start("run"); + _distribution->clear(); + _distribution->fill( _size ); + + Wait wait( _worker_print_waiting_time ); + ParallelApplyStore< type > store( wait, DEFAULT_MASTER, _packet_size ); + store.data( *_distribution ); + DynamicAssignmentAlgorithm scheduling; + ParallelApply< type > job( scheduling, DEFAULT_MASTER, store ); + + job.run(); + timerStat.stop("run"); + if( job.isMaster() ) + { + EmptyJob( scheduling, DEFAULT_MASTER ); // to terminate parallel apply + // Receive statistics + typedef std::map< std::string, eoTimerStat::Stat > typeStats; + // TODO put that in a file instead + std::cout << std::fixed << std::setprecision( 5 ); + for( int i = 1, s = comm.size(); i < s; ++i ) + { + eoTimerStat timerStat; + comm.recv( i, eo::mpi::Channel::Commands, timerStat ); + typeStats stats = timerStat.stats(); + for( typeStats::iterator it = stats.begin(), + end = stats.end(); + it != end; + ++it ) + { + std::cout << "Worker " << i << ": Wallclock time of " << it->first << std::endl; + for( int j = 0, t = it->second.wtime.size(); j < t; ++j ) + { + std::cout << it->second.wtime[j] << " "; + } + std::cout << std::endl; + } + std::cout << std::endl; + } + } else + { + // Send statistics + comm.send( DEFAULT_MASTER, eo::mpi::Channel::Commands, eo::mpi::timerStat ); + } + timerStat.clear(); + } + + private: + + Distribution* _distribution; + unsigned _size; + unsigned _packet_size; + bool _worker_print_waiting_time; +}; + int main( int argc, char** argv ) { Node::init( argc, argv ); - mpi::communicator& comm = eo::mpi::Node::comm(); eoParser parser( argc, argv ); // forces the statistics to be retrieved @@ -324,56 +434,19 @@ int main( int argc, char** argv ) make_parallel( parser ); make_help( parser ); - timerStat.start("main"); if( !isChosenDistrib ) { throw std::runtime_error("No distribution chosen. One distribution should be chosen."); } - // Fill distribution - Distribution& distrib = *pdistrib; - distrib.fill( size ); + Experiment e( pdistrib, size, packet_size, worker_print_waiting_time ); + eoserial::Object* obj = e.pack(); + obj->print( std::cout ); + delete obj; + std::cout << '\n' << std::endl; - Wait wait( worker_print_waiting_time ); - ParallelApplyStore< type > store( wait, DEFAULT_MASTER, packet_size ); - store.data( distrib ); - DynamicAssignmentAlgorithm scheduling; - ParallelApply< type > job( scheduling, DEFAULT_MASTER, store ); - - job.run(); - - timerStat.stop("main"); - if( job.isMaster() ) - { - EmptyJob( scheduling, DEFAULT_MASTER ); // to terminate parallel apply - // Receive statistics - typedef std::map< std::string, eoTimerStat::Stat > typeStats; - std::cout << std::fixed << std::setprecision( 5 ); - for( int i = 1, s = comm.size(); i < s; ++i ) - { - eoTimerStat timerStat; - comm.recv( i, eo::mpi::Channel::Commands, timerStat ); - typeStats stats = timerStat.stats(); - for( typeStats::iterator it = stats.begin(), - end = stats.end(); - it != end; - ++it ) - { - std::cout << "Worker " << i << ": Wallclock time of " << it->first << std::endl; - for( int j = 0, t = it->second.wtime.size(); j < t; ++j ) - { - std::cout << it->second.wtime[j] << " "; - } - std::cout << std::endl; - } - std::cout << std::endl; - } - } else - { - // Send statistics - comm.send( DEFAULT_MASTER, eo::mpi::Channel::Commands, eo::mpi::timerStat ); - } + e.run(); return 0; } From 7fe2bc587a0173e9acc640f527fafe55bb95d502 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 22:23:07 -0400 Subject: [PATCH 17/26] eoRNG: added a clearCache() function for cleaning the normal() cached value --- eo/src/utils/eoRNG.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eo/src/utils/eoRNG.h b/eo/src/utils/eoRNG.h index 78223c35..ce648cab 100644 --- a/eo/src/utils/eoRNG.h +++ b/eo/src/utils/eoRNG.h @@ -245,6 +245,18 @@ public : double normal(double mean, double stdev) { return mean + normal(stdev); } + /** + * @brief Forgets the last cached value of normal(), so as to be able to perform some repeatable calls to normal(). + * + * As normal() stores a cached value for performance purposes, sequences of pseudo random numbers can't be repeated + * when reseeding, since the cached value can be yield before a number is generated. To avoid that, this method + * allows one to clean the cache and force to regenerate a new pseudo random number. + */ + void clearCache() + { + cached = false; + } + /** Random numbers using a negative exponential distribution @param mean Mean value of distribution From 67df7756e47afb17647db35d7f4e8661f398dc3b Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 22:24:27 -0400 Subject: [PATCH 18/26] MPI Distrib exp: seed can be set at launch. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 43a7291a..5b946a8d 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -137,7 +137,7 @@ class UniformDistribution : public Distribution { public: - UniformDistribution() : _rng(0) + UniformDistribution() { // empty } @@ -151,7 +151,7 @@ class UniformDistribution : public Distribution int next_element() { - return std::floor( _rng.uniform( _min, _max ) ); + return std::floor( eo::rng.uniform( _min, _max ) ); } eoserial::Object* pack( void ) const @@ -171,8 +171,6 @@ class UniformDistribution : public Distribution protected: - eoRng _rng; - double _min; double _max; @@ -191,7 +189,7 @@ class NormalDistribution : public Distribution { public: - NormalDistribution() : _rng( 0 ) + NormalDistribution() { // empty } @@ -205,7 +203,7 @@ class NormalDistribution : public Distribution int next_element() { - return std::floor( _rng.normal( _mean, _stddev ) ); + return std::floor( eo::rng.normal( _mean, _stddev ) ); } eoserial::Object* pack( void ) const @@ -225,7 +223,6 @@ class NormalDistribution : public Distribution protected: - eoRng _rng; double _mean; double _stddev; } normalDistribution; @@ -245,7 +242,7 @@ class ExponentialDistribution : public Distribution { public: - ExponentialDistribution() : _rng( 0 ) + ExponentialDistribution() { // empty } @@ -258,7 +255,7 @@ class ExponentialDistribution : public Distribution int next_element() { - return std::floor( _rng.negexp( _mean ) ); + return std::floor( eo::rng.negexp( _mean ) ); } eoserial::Object* pack( void ) const @@ -276,7 +273,6 @@ class ExponentialDistribution : public Distribution protected: - eoRng _rng; double _mean; } exponentialDistribution; @@ -290,11 +286,12 @@ class Experiment : public eoserial::Persistent // empty } - Experiment( Distribution* distrib, unsigned size, unsigned packet_size, bool print_waiting_time ) : + Experiment( Distribution* distrib, unsigned size, unsigned packet_size, bool print_waiting_time, unsigned seed ) : _distribution( distrib ), _size( size ), _packet_size( packet_size ), - _worker_print_waiting_time( print_waiting_time ) + _worker_print_waiting_time( print_waiting_time ), + _seed( seed ) { // empty } @@ -305,6 +302,7 @@ class Experiment : public eoserial::Persistent obj->add( "size", eoserial::make( _size ) ); obj->add( "packet-size", eoserial::make( _packet_size ) ); obj->add( "worker-print-waiting-time", eoserial::make( _worker_print_waiting_time ) ); + obj->add( "seed", eoserial::make( _seed ) ); if( _distribution ) { obj->add( "distribution", _distribution ); @@ -317,6 +315,7 @@ class Experiment : public eoserial::Persistent eoserial::unpack( *obj, "size", _size ); eoserial::unpack( *obj, "packet-size", _packet_size ); eoserial::unpack( *obj, "worker-print-waiting-time", _worker_print_waiting_time ); + eoserial::unpack( *obj, "seed", _seed ); eoserial::Object* distribObject = static_cast( obj->find("distribution")->second ); std::string distribName = *static_cast( distribObject->find("name")->second ); @@ -338,10 +337,12 @@ class Experiment : public eoserial::Persistent void run() { mpi::communicator& comm = eo::mpi::Node::comm(); - timerStat.start("run"); + eo::rng.reseed( _seed ); + eo::rng.clearCache(); _distribution->clear(); _distribution->fill( _size ); + timerStat.start("run"); Wait wait( _worker_print_waiting_time ); ParallelApplyStore< type > store( wait, DEFAULT_MASTER, _packet_size ); store.data( *_distribution ); @@ -390,6 +391,7 @@ class Experiment : public eoserial::Persistent unsigned _size; unsigned _packet_size; bool _worker_print_waiting_time; + unsigned _seed; }; int main( int argc, char** argv ) @@ -404,6 +406,7 @@ int main( int argc, char** argv ) unsigned size = parser.createParam( 10U, "size", "Number of elements to distribute.", 's', "Distribution").value(); unsigned packet_size = parser.createParam( 1U, "packet-size", "Number of elements to distribute at each time for a single worker.", 'p', "Parallelization").value(); bool worker_print_waiting_time = parser.createParam( false, "print-waiting-time", "Do the workers need to print the time they wait?", '\0', "Parallelization").value(); + unsigned seed = parser.createParam( 0U, "seed", "Seed of random generator", '\0', "General").value(); std::vector distribs; distribs.push_back( &uniformDistribution ); @@ -440,7 +443,7 @@ int main( int argc, char** argv ) throw std::runtime_error("No distribution chosen. One distribution should be chosen."); } - Experiment e( pdistrib, size, packet_size, worker_print_waiting_time ); + Experiment e( pdistrib, size, packet_size, worker_print_waiting_time, seed ); eoserial::Object* obj = e.pack(); obj->print( std::cout ); delete obj; From 84afb1866ed6f231435889d57192024eed721469 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 22:43:24 -0400 Subject: [PATCH 19/26] MPI Distrib exp: comments + redirect output of an experiment to a file. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 55 ++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 5b946a8d..73920b05 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -277,21 +277,33 @@ class ExponentialDistribution : public Distribution } exponentialDistribution; +/** + * @brief Serializable experiment. + * + * Allows an experiment to be saved and loaded via a file, using eoserial. + * + * Construct the experiment with the good parameters from the command line or load experiments from a file. Then call run() to launch the parallel job. + * + * If a filename is given to the constructor (or during the loading), the results of the experiments (time series) will + * be redirected to the file with the given file name. Otherwise (filename == ""), the output will just be shown on the + * standard output. + */ class Experiment : public eoserial::Persistent { public: - Experiment() : _distribution(0) + Experiment() : _distribution(0), _fileName("") { // empty } - Experiment( Distribution* distrib, unsigned size, unsigned packet_size, bool print_waiting_time, unsigned seed ) : + Experiment( Distribution* distrib, unsigned size, unsigned packet_size, bool print_waiting_time, unsigned seed, const std::string& fileName = "" ) : _distribution( distrib ), _size( size ), _packet_size( packet_size ), _worker_print_waiting_time( print_waiting_time ), - _seed( seed ) + _seed( seed ), + _fileName( fileName ) { // empty } @@ -307,6 +319,7 @@ class Experiment : public eoserial::Persistent { obj->add( "distribution", _distribution ); } + obj->add( "filename", eoserial::make( _fileName ) ); return obj; } @@ -316,6 +329,7 @@ class Experiment : public eoserial::Persistent eoserial::unpack( *obj, "packet-size", _packet_size ); eoserial::unpack( *obj, "worker-print-waiting-time", _worker_print_waiting_time ); eoserial::unpack( *obj, "seed", _seed ); + eoserial::unpack( *obj, "filename", _fileName ); eoserial::Object* distribObject = static_cast( obj->find("distribution")->second ); std::string distribName = *static_cast( distribObject->find("name")->second ); @@ -337,8 +351,9 @@ class Experiment : public eoserial::Persistent void run() { mpi::communicator& comm = eo::mpi::Node::comm(); + // reinits every objects eo::rng.reseed( _seed ); - eo::rng.clearCache(); + eo::rng.clearCache(); // trick for repeatable sequences of normal numbers, cf eo::rng _distribution->clear(); _distribution->fill( _size ); @@ -356,8 +371,20 @@ class Experiment : public eoserial::Persistent EmptyJob( scheduling, DEFAULT_MASTER ); // to terminate parallel apply // Receive statistics typedef std::map< std::string, eoTimerStat::Stat > typeStats; - // TODO put that in a file instead - std::cout << std::fixed << std::setprecision( 5 ); + + std::ostream* pout; + std::ofstream file; + bool fileSaveActivated = false; + if( _fileName == "" ) { + pout = & std::cout; + } else { + pout = & file; + file.open( _fileName.c_str() ); + fileSaveActivated = true; + } + std::ostream& out = *pout; + + out << std::fixed << std::setprecision( 5 ); for( int i = 1, s = comm.size(); i < s; ++i ) { eoTimerStat timerStat; @@ -368,14 +395,18 @@ class Experiment : public eoserial::Persistent it != end; ++it ) { - std::cout << "Worker " << i << ": Wallclock time of " << it->first << std::endl; + out << i << " " << it->first << std::endl; for( int j = 0, t = it->second.wtime.size(); j < t; ++j ) { - std::cout << it->second.wtime[j] << " "; + out << it->second.wtime[j] << " "; } - std::cout << std::endl; + out << std::endl; } - std::cout << std::endl; + out << std::endl; + } + + if( fileSaveActivated ) { + file.close(); } } else { @@ -392,6 +423,7 @@ class Experiment : public eoserial::Persistent unsigned _packet_size; bool _worker_print_waiting_time; unsigned _seed; + std::string _fileName; }; int main( int argc, char** argv ) @@ -407,6 +439,7 @@ int main( int argc, char** argv ) unsigned packet_size = parser.createParam( 1U, "packet-size", "Number of elements to distribute at each time for a single worker.", 'p', "Parallelization").value(); bool worker_print_waiting_time = parser.createParam( false, "print-waiting-time", "Do the workers need to print the time they wait?", '\0', "Parallelization").value(); unsigned seed = parser.createParam( 0U, "seed", "Seed of random generator", '\0', "General").value(); + std::string fileName = parser.createParam( std::string(""), "filename", "File name to which redirect the results (for a single experiment)", '\0', "General").value(); std::vector distribs; distribs.push_back( &uniformDistribution ); @@ -443,7 +476,7 @@ int main( int argc, char** argv ) throw std::runtime_error("No distribution chosen. One distribution should be chosen."); } - Experiment e( pdistrib, size, packet_size, worker_print_waiting_time, seed ); + Experiment e( pdistrib, size, packet_size, worker_print_waiting_time, seed, fileName ); eoserial::Object* obj = e.pack(); obj->print( std::cout ); delete obj; From 3769021a233260c4ed3f4dd4c850decf1361a4f5 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 23:25:51 -0400 Subject: [PATCH 20/26] Bugfix Serialization parser: spaces can be inserted before or after the arrays. --- eo/src/serial/Parser.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eo/src/serial/Parser.cpp b/eo/src/serial/Parser.cpp index 258174c8..f46c829e 100644 --- a/eo/src/serial/Parser.cpp +++ b/eo/src/serial/Parser.cpp @@ -124,6 +124,7 @@ Entity* Parser::parseRight(const std::string & str, size_t & pos) DEBUG("We read an object.") Object* obj = new Object; pos += 1; + ignoreChars( str, pos ); while( pos < str.size() && str[ pos ] != '}' ) { parseLeft( str, pos, obj ); @@ -145,11 +146,13 @@ Entity* Parser::parseRight(const std::string & str, size_t & pos) DEBUG("We read an array") Array* array = new Array; pos += 1; + ignoreChars( str, pos ); while( pos < str.size() && str[ pos ] != ']' ) { Entity* child = parseRight( str, pos ); if ( child ) array->push_back( child ); + ignoreChars( str, pos ); } DEBUG("We've finished to read our array.") pos += 1; // we're on the ], go to the next char From bd9767a05d38edc3bd29af9741c548a0e2d5beef Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 23:26:21 -0400 Subject: [PATCH 21/26] Timer: force measures to be retrieved even if parallel.doMeasure() isn't set. --- eo/src/utils/eoTimer.h | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/eo/src/utils/eoTimer.h b/eo/src/utils/eoTimer.h index fbf90c21..9f9371dd 100644 --- a/eo/src/utils/eoTimer.h +++ b/eo/src/utils/eoTimer.h @@ -205,6 +205,14 @@ class eoTimerStat { public: + /** + * @brief Initializes a timer stat object. + */ + eoTimerStat() : _forceDoMeasure(false) + { + // empty + } + /** * @brief Statistic related to a key (name). * @@ -274,6 +282,14 @@ class eoTimerStat } # endif + /** + * @brief Forces the measures to be retrieved. + */ + void forceDoMeasure() + { + _forceDoMeasure = true; + } + /** * @brief Starts a new measure for the given key. * @@ -284,7 +300,7 @@ class eoTimerStat */ void start( const std::string & key ) { - if( eo::parallel.doMeasure() ) + if( eo::parallel.doMeasure() or _forceDoMeasure ) { _timers[ key ].restart(); } @@ -302,7 +318,7 @@ class eoTimerStat */ void stop( const std::string& key ) { - if( eo::parallel.doMeasure() ) + if( eo::parallel.doMeasure() or _forceDoMeasure ) { Stat & s = _stats[ key ]; eoTimer & t = _timers[ key ]; @@ -333,6 +349,8 @@ class eoTimerStat std::map< std::string, Stat > _stats; // Timers map: links a key to its timer. std::map< std::string, eoTimer > _timers; + // boolean to force the retrieval of statistics + bool _forceDoMeasure; }; # endif // __TIMER_H__ From 57dcd01149c78a4a7e6d79fe3a1d74296123f08f Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Mon, 1 Oct 2012 23:27:41 -0400 Subject: [PATCH 22/26] MPI Distrib Exp: launch experiments from a file + example file. --- eo/test/mpi/experiments.json | 6 ++ eo/test/mpi/t-mpi-distrib-exp.cpp | 107 +++++++++++++++++++----------- 2 files changed, 74 insertions(+), 39 deletions(-) create mode 100644 eo/test/mpi/experiments.json diff --git a/eo/test/mpi/experiments.json b/eo/test/mpi/experiments.json new file mode 100644 index 00000000..c8dfde46 --- /dev/null +++ b/eo/test/mpi/experiments.json @@ -0,0 +1,6 @@ +{ + "experiments":[ + {"size":"10", "packet_size":"1", "seed":"1337", "distribution":{"name":"normal", "mean":"500", "stddev":"100"}, "worker_print_waiting_time":"1", "filename":""} + {"size":"10", "packet_size":"1", "seed":"1337", "distribution":{"name":"normal", "mean":"100", "stddev":"20"}, "worker_print_waiting_time":"1", "filename":"exp2.result.txt"} + ] +} diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 73920b05..6787b636 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -292,7 +292,7 @@ class Experiment : public eoserial::Persistent { public: - Experiment() : _distribution(0), _fileName("") + Experiment() : _distribution(0), _worker_print_waiting_time( false ), _fileName("") { // empty } @@ -312,8 +312,8 @@ class Experiment : public eoserial::Persistent { eoserial::Object* obj = new eoserial::Object; obj->add( "size", eoserial::make( _size ) ); - obj->add( "packet-size", eoserial::make( _packet_size ) ); - obj->add( "worker-print-waiting-time", eoserial::make( _worker_print_waiting_time ) ); + obj->add( "packet_size", eoserial::make( _packet_size ) ); + obj->add( "worker_print_waiting_time", eoserial::make( _worker_print_waiting_time ) ); obj->add( "seed", eoserial::make( _seed ) ); if( _distribution ) { @@ -326,8 +326,8 @@ class Experiment : public eoserial::Persistent void unpack( const eoserial::Object* obj ) { eoserial::unpack( *obj, "size", _size ); - eoserial::unpack( *obj, "packet-size", _packet_size ); - eoserial::unpack( *obj, "worker-print-waiting-time", _worker_print_waiting_time ); + eoserial::unpack( *obj, "packet_size", _packet_size ); + eoserial::unpack( *obj, "worker_print_waiting_time", _worker_print_waiting_time ); eoserial::unpack( *obj, "seed", _seed ); eoserial::unpack( *obj, "filename", _fileName ); @@ -432,7 +432,7 @@ int main( int argc, char** argv ) eoParser parser( argc, argv ); // forces the statistics to be retrieved - parser.setORcreateParam( true, "parallelize-do-measure", "Do some measures during execution" ); + eo::mpi::timerStat.forceDoMeasure(); // General parameters for the experimentation unsigned size = parser.createParam( 10U, "size", "Number of elements to distribute.", 's', "Distribution").value(); @@ -441,48 +441,77 @@ int main( int argc, char** argv ) unsigned seed = parser.createParam( 0U, "seed", "Seed of random generator", '\0', "General").value(); std::string fileName = parser.createParam( std::string(""), "filename", "File name to which redirect the results (for a single experiment)", '\0', "General").value(); - std::vector distribs; - distribs.push_back( &uniformDistribution ); - distribs.push_back( &normalDistribution ); - distribs.push_back( &exponentialDistribution ); + bool useExperimentFile = parser.createParam( false, "use-experiment-file", "Put to true if you want to launch experiments from a file formatted in JSON (see experiment-file).", '\0', "General").value(); + std::string experimentFile = parser.createParam( std::string("experiments.json"), "experiment-file", "File name of experiments to provide, in format JSON.", '\0', "General").value(); - // for each available distribution, check if activated. - // If no distribution is activated, show an error message - // If two distributions or more are activated, show an error message - // Otherwise, use the activated distribution as distrib - bool isChosenDistrib = false; - Distribution* pdistrib = 0; - for( int i = 0, s = distribs.size(); i < s; ++i ) + if( !useExperimentFile ) { - distribs[i]->make_parser( parser ); - if( distribs[i]->isActive() ) + std::vector distribs; + distribs.push_back( &uniformDistribution ); + distribs.push_back( &normalDistribution ); + distribs.push_back( &exponentialDistribution ); + + // for each available distribution, check if activated. + // If no distribution is activated, show an error message + // If two distributions or more are activated, show an error message + // Otherwise, use the activated distribution as distrib + bool isChosenDistrib = false; + Distribution* pdistrib = 0; + for( int i = 0, s = distribs.size(); i < s; ++i ) { - if( isChosenDistrib ) + distribs[i]->make_parser( parser ); + if( distribs[i]->isActive() ) { - throw std::runtime_error("Only one distribution can be chosen during a launch!"); - } else - { - isChosenDistrib = true; - pdistrib = distribs[i]; + if( isChosenDistrib ) + { + throw std::runtime_error("Only one distribution can be chosen during a launch!"); + } else + { + isChosenDistrib = true; + pdistrib = distribs[i]; + } } } + + make_parallel( parser ); + make_help( parser ); + + if( !isChosenDistrib ) + { + throw std::runtime_error("No distribution chosen. One distribution should be chosen."); + } + + Experiment e( pdistrib, size, packet_size, worker_print_waiting_time, seed, fileName ); + e.run(); } - - make_parallel( parser ); - make_help( parser ); - - if( !isChosenDistrib ) + else // use experiments file { - throw std::runtime_error("No distribution chosen. One distribution should be chosen."); + // read content of file + std::ifstream file( experimentFile.c_str() ); + std::string fileContent; + while( file ) + { + char temp[4096]; + file.getline( temp, 4096, '\n' ); + fileContent += temp; + fileContent += '\n'; + } + file.close(); + + // transform content into array of experiments + eoserial::Object* wrapper = eoserial::Parser::parse( fileContent ); + eoserial::Array& experiments = *static_cast< eoserial::Array* >( wrapper->find("experiments")->second ); + + for( unsigned i = 0, s = experiments.size(); i < s; ++i ) + { + std::cout << "Launching experiment " << (i+1) << "..." << std::endl; + eoserial::Object* expObj = static_cast< eoserial::Object* >( experiments[i] ); + Experiment exp; + exp.unpack( expObj ); + exp.run(); + } + delete wrapper; } - Experiment e( pdistrib, size, packet_size, worker_print_waiting_time, seed, fileName ); - eoserial::Object* obj = e.pack(); - obj->print( std::cout ); - delete obj; - std::cout << '\n' << std::endl; - - e.run(); - return 0; } From c48b3979c30d7597d1440cf71141d1bb86f58bd3 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 5 Oct 2012 18:06:45 -0400 Subject: [PATCH 23/26] eo::serial::Parser: if spaces were present after colons, they were not ignored. Fixed by this patch. --- eo/src/serial/Parser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/eo/src/serial/Parser.cpp b/eo/src/serial/Parser.cpp index f46c829e..57b52a0d 100644 --- a/eo/src/serial/Parser.cpp +++ b/eo/src/serial/Parser.cpp @@ -118,6 +118,7 @@ Entity* Parser::parseRight(const std::string & str, size_t & pos) { Entity* value = 0; + ignoreChars( str, pos ); if ( str[ pos ] == '{' ) { // next one is an object From 371946f1f1e8e79def3e79924ccd8e5ab61d6591 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 5 Oct 2012 18:07:25 -0400 Subject: [PATCH 24/26] MPI Distrib exp: the parameters of an experiment are recalled in the results file. --- eo/test/mpi/t-mpi-distrib-exp.cpp | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/eo/test/mpi/t-mpi-distrib-exp.cpp b/eo/test/mpi/t-mpi-distrib-exp.cpp index 6787b636..7d081b56 100644 --- a/eo/test/mpi/t-mpi-distrib-exp.cpp +++ b/eo/test/mpi/t-mpi-distrib-exp.cpp @@ -34,6 +34,7 @@ # include # include # include +# include # include # include @@ -115,6 +116,11 @@ class Distribution : public std::vector< type >, public eoserial::Persistent */ bool isActive() { return _active; } + /** + * @brief Prints the name and the parameters of the distribution + */ + virtual std::string toString() const = 0; + protected: bool _active; @@ -169,6 +175,15 @@ class UniformDistribution : public Distribution eoserial::unpack( *obj, "max", _max ); } + std::string toString() const + { + std::stringstream ss; + ss << "uniform" << '\n' + << "min: " << _min << '\n' + << "max: " << _max << '\n'; + return ss.str(); + } + protected: double _min; @@ -221,6 +236,15 @@ class NormalDistribution : public Distribution eoserial::unpack( *obj, "stddev", _stddev ); } + std::string toString() const + { + std::stringstream ss; + ss << "normal" << '\n' + << "mean: " << _mean << '\n' + << "stddev: " << _stddev << '\n'; + return ss.str(); + } + protected: double _mean; @@ -271,6 +295,14 @@ class ExponentialDistribution : public Distribution eoserial::unpack( *obj, "mean", _mean ); } + std::string toString() const + { + std::stringstream ss; + ss << "exponential" << '\n' + << "mean: " << _mean << '\n'; + return ss.str(); + } + protected: double _mean; @@ -384,6 +416,13 @@ class Experiment : public eoserial::Persistent } std::ostream& out = *pout; + // Reminder of the parameters + out << "size: " << _size << '\n' + << "packet_size: " << _packet_size << '\n' + << "distribution: " << _distribution->toString() + << "seed: " << _seed << '\n' << std::endl; + + // Results out << std::fixed << std::setprecision( 5 ); for( int i = 1, s = comm.size(); i < s; ++i ) { From 3822069db07c18f0df9e1bde21e55e0e49fa9c4b Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 5 Oct 2012 18:08:06 -0400 Subject: [PATCH 25/26] MPI Distrib exp: generator of experiments and readme. --- eo/test/mpi/DISTRIB_XP_README.md | 21 +++++ eo/test/mpi/gen-xp.py | 134 +++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 eo/test/mpi/DISTRIB_XP_README.md create mode 100644 eo/test/mpi/gen-xp.py diff --git a/eo/test/mpi/DISTRIB_XP_README.md b/eo/test/mpi/DISTRIB_XP_README.md new file mode 100644 index 00000000..22b56b25 --- /dev/null +++ b/eo/test/mpi/DISTRIB_XP_README.md @@ -0,0 +1,21 @@ +README +------ + +To launch a set of experiments with t-mpi-distrib-exp: + +1) Generate the experiments, thanks to the script gen-xp.py +This script will guide you and ask you for all experiments. The prefix is used in the results filenames. +You may want to modify the name of the experiments file (default value: "experiments.json") or +the pattern of the results files. However, you have to ensure that the pattern is an one-to-one +function of the parameters, otherwise some results could be lost. + +2) Launch the t-mpi-distrib-exp program with mpirun: +For 4 cores (= 1 master + 3 workers) +mpirun -np 4 ./t-mpi-distrib-exp --use-experiment-file=1 --experiment-file=/home/eodev/eo/test/mpi/experiments.json + +For 16 cores (= 1 master + 15 workers) +mpirun -np 5 ./t-mpi-distrib-exp --use-experiment-file=1 --experiment-file=/home/eodev/eo/test/mpi/experiments.json + +3) The program will generate the results of the experiments, as txt files. There is one result file for each run of each +experiment. + diff --git a/eo/test/mpi/gen-xp.py b/eo/test/mpi/gen-xp.py new file mode 100644 index 00000000..d9b8242b --- /dev/null +++ b/eo/test/mpi/gen-xp.py @@ -0,0 +1,134 @@ +# -*- coding:utf-8 -*- +import json + +# Where will be saved the experiments? +EXPERIMENTS_FILENAME = "experiments.json" +# What will be the pattern for experiments filenames? +FILENAME_PATTERN = "%(prefix)s_%(distrib_name)s_%(size)s_%(packet_size)s_%(run)s.txt" + +def input_number_at_least( min ): + n = min - 1 + while n < min: + try: + n = int(raw_input("Enter a number greater or equal to %s: "% min)) + except Exception: + print "Please enter an integer." + return n + +def input_number_between( min, max ): + n = min - 1 + while n < min or n > max: + try: + n = int(raw_input("Enter a number between %s and %s: " % (min,max))) + except Exception: + print "Please enter a number." + return n + +def choose_continue(): + print """Do you want to continue? +0. No +1. Yes""" + return bool( input_number_between(0,1) ) + +def choose_distribution_uniform(): + distribution = {} + distribution["name"] = "uniform" + print "Enter the minimum value (in milliseconds): " + min = input_number_at_least( 0 ) + distribution["min"] = str(min) + print "Enter the maximum value (in milliseconds): " + distribution["max"] = str(input_number_at_least( min )) + return distribution + +def choose_distribution_normal(): + distribution = {} + distribution["name"] = "normal" + print "Enter the mean (in milliseconds): " + distribution["mean"] = str(input_number_at_least( 0 )) + print "Enter the standard deviation (in milliseconds): " + distribution["stddev"] = str(input_number_at_least( 0 )) + return distribution + +def choose_distribution_power(): + distribution = {} + distribution["name"] = "exponential" + print "Enter the mean (in milliseconds): " + distribution["mean"] = str(input_number_at_least( 0 )) + return distribution + +def choose_distribution(): + print """Choose your distribution: +0. Uniform +1. Normal +2. Exponential""" + choice = input_number_between( 0, 2 ) + choose_distrib_functions = [ choose_distribution_uniform, choose_distribution_normal, choose_distribution_power ] + return choose_distrib_functions[ choice ]() + +def choose_packet_size(): + print "Enter the size of a packet (group of elements):" + return str(input_number_at_least( 0 )) + +def choose_size(): + print "Enter the total size (size of population):" + return str(input_number_at_least( 0 )) + +def choose_worker_print(): + print """Should the workers print the time they sleep on stdout? +0. No +1. Yes""" + return str(input_number_between( 0, 1 )) + +def choose_nb_runs(): + print """How many runs should be launched for this configuration? Seeds will be automatically affected to the number +of run+1 (for instance, the first run has a seed of 1, the second has a seed of 2, etc.).""" + return input_number_at_least( 1 ) + +def choose_prefix(): + print """What is the name of the experiment? It will be used as the prefix of file names.""" + return raw_input("Enter the prefix name: ") + +def main(): + + prefix = choose_prefix() + exps = [] + + while True: + exp = {} + exp["distribution"] = choose_distribution() + + exp["size"] = choose_size() + exp["packet_size"] = choose_packet_size() + exp["worker_print_waiting_time"] = choose_worker_print() + runs = choose_nb_runs() + for i in range( runs ): + exp["seed"] = str(i+1) + + filename_map = exp.copy() + filename_map["run"] = exp["seed"] + filename_map["distrib_name"] = exp["distribution"]["name"] + filename_map["prefix"] = prefix + filename = FILENAME_PATTERN % filename_map + + exp["filename"] = filename + copy = exp.copy() + exps.append( copy ) + + if not choose_continue(): + break + + # Write the experiments in a file + f = file( EXPERIMENTS_FILENAME , 'wb') + f.write("""{"experiments":[""") + i = 0 + for exp in exps: + if i > 0: + f.write(",\n") + i += 1 + f.write( json.dumps(exp) ) + f.write("]}") + f.close() + +if __name__ == "__main__": + main() + From bef4f649cf59cf2751c0c39a36d10d34c73d5580 Mon Sep 17 00:00:00 2001 From: Benjamin BOUVIER Date: Fri, 5 Oct 2012 18:11:18 -0400 Subject: [PATCH 26/26] MPI Distrib exp: license for gen-xp --- eo/test/mpi/gen-xp.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/eo/test/mpi/gen-xp.py b/eo/test/mpi/gen-xp.py index d9b8242b..274e0243 100644 --- a/eo/test/mpi/gen-xp.py +++ b/eo/test/mpi/gen-xp.py @@ -1,4 +1,22 @@ # -*- coding:utf-8 -*- +""" +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Authors: + Benjamin Bouvier +""" import json # Where will be saved the experiments?