Hi all,
I've run some experiments for the MPICH2 threading overhead on BG/P and Breadboard (test program is attached). I don't remember if IBM reported that the performance stays constant with increasing threads or if it degrades, but I'm noticing a drop in performance (results below).
On BG/P, this is slight yet noticeable, but on breadboard it's drastic. Note that 1 thread refers to "no extra threads" -- the main process does a self send/recv.
This is not the fairest comparison since MPICH2-BG/P is based on MPICH2-1.0.4p1, while the breadboard runs are on MPICH2-trunk. I'll try out MPICH2-1.0.4p1 on breadboard as well. But as a longer term solution I'm trying to get MPICH2-BG/P ported to MPICH2-trunk so that I can easily try out any threading enhancements that might go in to trunk from here on.
Thanks.
-- Pavan
-------------------------------------------------------------------------
BG/P:
----
# MPI Message Rate Test with 1 Threads
# Size Latency (us) Message Rate (MMPS)
0 1.38 0.725
# MPI Message Rate Test with 2 Threads
# Size Latency (us) Message Rate (MMPS)
0 3.59 0.557
# MPI Message Rate Test with 3 Threads
# Size Latency (us) Message Rate (MMPS)
0 5.75 0.522
# MPI Message Rate Test with 4 Threads
# Size Latency (us) Message Rate (MMPS)
0 7.93 0.504
Breadboard:
----------
# MPI Message Rate Test with 1 Threads
# Size Latency (us) Message Rate (MMPS)
0 0.17 5.774
# MPI Message Rate Test with 2 Threads
# Size Latency (us) Message Rate (MMPS)
0 1.69 1.180
# MPI Message Rate Test with 3 Threads
# Size Latency (us) Message Rate (MMPS)
0 7.54 0.398
# MPI Message Rate Test with 4 Threads
# Size Latency (us) Message Rate (MMPS)
0 2.04 1.963
--
Pavan Balaji
#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <unistd.h>
#include <pthread.h>
#include <assert.h>
#include <string.h>
#define REQUESTS 16
#define SKIP 100
#define LOOPS 1000000
#define MAX_THREADS 100
typedef struct {
int nthreads;
int size;
} params_t;
char * s_buf, * r_buf;
int skip = 10;
int loop = 10000;
double t_start = 0.0, t_end = 0.0;
pthread_barrier_t pbarrier;
params_t params;
void usage(char * progname)
{
fprintf(stderr, "Usage: %s -t [num_threads] -s [msg_size]\n", progname);
MPI_Abort(MPI_COMM_WORLD, -1);
}
void get_parameters(int orig_argc, char ** orig_argv)
{
int argc = orig_argc;
char ** argv = orig_argv;
params.nthreads = -1;
params.size = -1;
while (--argc && ++argv) {
if (!strcmp(*argv, "-t")) {
--argc; ++argv;
params.nthreads = atoi(*argv);
continue;
}
if (!strcmp(*argv, "-s")) {
--argc; ++argv;
params.size = atoi(*argv);
continue;
}
usage(orig_argv[0]);
}
if ((params.nthreads <= 0) || (params.size < 0))
usage(orig_argv[0]);
}
void * pingping_timed (void * arg)
{
int i, j;
MPI_Request request[REQUESTS*2];
MPI_Status reqstat[REQUESTS*2];
pthread_barrier_wait(&pbarrier);
for (i = 0; i < loop + skip; i++) {
if (i == skip)
t_start = MPI_Wtime();
for (j = 0; j < REQUESTS; j++)
MPI_Irecv(r_buf, params.size, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &request[j+REQUESTS]);
for (j = 0; j < REQUESTS; j++)
MPI_Isend(s_buf, params.size, MPI_CHAR, 0, 0, MPI_COMM_WORLD, &request[j]);
MPI_Waitall(REQUESTS*2, request, reqstat);
}
pthread_barrier_wait(&pbarrier);
t_end = MPI_Wtime();
return NULL;
}
int main(int argc, char ** argv)
{
int procs, i, pmode;
pthread_t thread[MAX_THREADS];
void * retval[MAX_THREADS];
double latency, mrate;
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &pmode);
if (pmode != MPI_THREAD_MULTIPLE) {
fprintf(stderr, "Thread Multiple not supported by the MPI implementation\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
get_parameters(argc, argv);
pthread_barrier_init(&pbarrier, NULL, params.nthreads);
if (params.size) {
s_buf = (void *) malloc(params.size);
r_buf = (void *) malloc(params.size);
for (i = 0; i < params.size; i++) {
s_buf[i] = '0';
r_buf[i] = '0';
}
}
MPI_Comm_size(MPI_COMM_WORLD, &procs);
if (procs != 1) {
fprintf(stderr, "This is a 1 process test\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
MPI_Barrier(MPI_COMM_WORLD);
for (i = 1; i < params.nthreads; i++)
pthread_create(&thread[i], NULL, (void*) pingping_timed, (void *) &i);
pingping_timed(NULL);
for (i = 1; i < params.nthreads; i++)
pthread_join(thread[i], &retval[i]);
printf("# MPI Message Rate Test with %d Threads\n", params.nthreads);
printf("# Size\t\tLatency (us)\tMessage Rate (MMPS) \n");
/* Latency is the time to send and recieve a message divided by two. */
latency = (t_end - t_start) * 1.0e6 / (2*loop*REQUESTS);
mrate = params.nthreads / latency;
printf("%d\t\t%0.2f\t\t%0.3f\n", params.size, latency, mrate);
MPI_Finalize();
return 0;
}