Don't you just NOT call MPI_thread_init? -d On 01/30/2008 10:43 AM, Rajeev Thakur wrote:
What is the switch to turn on/off this optimization?
Rajeev
------------------------------------------------------------------------ *From:* [email protected] [mailto:[email protected]] *On Behalf Of *William Gropp *Sent:* Wednesday, January 30, 2008 8:30 AM *To:* [email protected] *Subject:* Re: [mpich2-core] MPICH2 Threading Overhead
Cool, this is great! This means that there's lots of room for improvements :)
Is the one-thread case exploiting the "skip locks/nesting count through thread-specific storage" optimization? It would be good to run the 1-thread case with this optimization both on and off in any case, just to get a measure of the overhead.
Bill
On Jan 29, 2008, at 3:53 PM, Pavan Balaji wrote:
Hi all,
I've run some experiments for the MPICH2 threading overhead on BG/P and Breadboard (test program is attached). I don't remember if IBM reported that the performance stays constant with increasing threads or if it degrades, but I'm noticing a drop in performance (results below).
On BG/P, this is slight yet noticeable, but on breadboard it's drastic. Note that 1 thread refers to "no extra threads" -- the main process does a self send/recv.
This is not the fairest comparison since MPICH2-BG/P is based on MPICH2-1.0.4p1, while the breadboard runs are on MPICH2-trunk. I'll try out MPICH2-1.0.4p1 on breadboard as well. But as a longer term solution I'm trying to get MPICH2-BG/P ported to MPICH2-trunk so that I can easily try out any threading enhancements that might go in to trunk from here on.
Thanks.
-- Pavan
-------------------------------------------------------------------------
BG/P: ---- # MPI Message Rate Test with 1 Threads # Size Latency (us) Message Rate (MMPS) 0 1.38 0.725 # MPI Message Rate Test with 2 Threads # Size Latency (us) Message Rate (MMPS) 0 3.59 0.557 # MPI Message Rate Test with 3 Threads # Size Latency (us) Message Rate (MMPS) 0 5.75 0.522 # MPI Message Rate Test with 4 Threads # Size Latency (us) Message Rate (MMPS) 0 7.93 0.504
Breadboard: ---------- # MPI Message Rate Test with 1 Threads # Size Latency (us) Message Rate (MMPS) 0 0.17 5.774 # MPI Message Rate Test with 2 Threads # Size Latency (us) Message Rate (MMPS) 0 1.69 1.180 # MPI Message Rate Test with 3 Threads # Size Latency (us) Message Rate (MMPS) 0 7.54 0.398 # MPI Message Rate Test with 4 Threads # Size Latency (us) Message Rate (MMPS) 0 2.04 1.963
-- Pavan Balaji http://www.mcs.anl.gov/~balaji #include "mpi.h" #include <stdio.h> #include <stdlib.h> #include <math.h> #include <unistd.h> #include <pthread.h> #include <assert.h> #include <string.h>
#define REQUESTS 16 #define SKIP 100 #define LOOPS 1000000 #define MAX_THREADS 100
typedef struct { int nthreads; int size; } params_t;
char * s_buf, * r_buf; int skip = 10; int loop = 10000; double t_start = 0.0, t_end = 0.0; pthread_barrier_t pbarrier; params_t params;
void usage(char * progname) { fprintf(stderr, "Usage: %s -t [num_threads] -s [msg_size]\n", progname); MPI_Abort(MPI_COMM_WORLD, -1); }
void get_parameters(int orig_argc, char ** orig_argv) { int argc = orig_argc; char ** argv = orig_argv;
params.nthreads = -1; params.size = -1;
while (--argc && ++argv) { if (!strcmp(*argv, "-t")) { --argc; ++argv; params.nthreads = atoi(*argv); continue; } if (!strcmp(*argv, "-s")) { --argc; ++argv; params.size = atoi(*argv); continue; }
usage(orig_argv[0]); }
if ((params.nthreads <= 0) || (params.size < 0)) usage(orig_argv[0]); }
void * pingping_timed (void * arg) { int i, j; MPI_Request request[REQUESTS*2]; MPI_Status reqstat[REQUESTS*2];
pthread_barrier_wait(&pbarrier); for (i = 0; i < loop + skip; i++) { if (i == skip) t_start = MPI_Wtime();
for (j = 0; j < REQUESTS; j++) MPI_Irecv(r_buf, params.size, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &request[j+REQUESTS]);
for (j = 0; j < REQUESTS; j++) MPI_Isend(s_buf, params.size, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &request[j]);
MPI_Waitall(REQUESTS*2, request, reqstat); } pthread_barrier_wait(&pbarrier);
t_end = MPI_Wtime();
return NULL; }
int main(int argc, char ** argv) { int procs, i, pmode; pthread_t thread[MAX_THREADS]; void * retval[MAX_THREADS]; double latency, mrate;
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &pmode); if (pmode != MPI_THREAD_MULTIPLE) { fprintf(stderr, "Thread Multiple not supported by the MPI implementation\n"); MPI_Abort(MPI_COMM_WORLD, -1); }
get_parameters(argc, argv); pthread_barrier_init(&pbarrier, NULL, params.nthreads);
if (params.size) { s_buf = (void *) malloc(params.size); r_buf = (void *) malloc(params.size); for (i = 0; i < params.size; i++) { s_buf[i] = '0'; r_buf[i] = '0'; } }
MPI_Comm_size(MPI_COMM_WORLD, &procs); if (procs != 1) { fprintf(stderr, "This is a 1 process test\n"); MPI_Abort(MPI_COMM_WORLD, -1); } MPI_Barrier(MPI_COMM_WORLD);
for (i = 1; i < params.nthreads; i++) pthread_create(&thread[i], NULL, (void*) pingping_timed, (void *) &i);
pingping_timed(NULL);
for (i = 1; i < params.nthreads; i++) pthread_join(thread[i], &retval[i]);
printf("# MPI Message Rate Test with %d Threads\n", params.nthreads); printf("# Size\t\tLatency (us)\tMessage Rate (MMPS) \n");
/* Latency is the time to send and recieve a message divided by two. */ latency = (t_end - t_start) * 1.0e6 / (2*loop*REQUESTS); mrate = params.nthreads / latency; printf("%d\t\t%0.2f\t\t%0.3f\n", params.size, latency, mrate);
MPI_Finalize();
return 0; }
William Gropp Paul and Cynthia Saylor Professor of Computer Science University of Illinois Urbana-Champaign