[svn-r4746]

Purpose:
	Feature Fix
Description:
	Changed so the "pio_perf" module handles creating and destroying the
	MPI Comm. Worked it so we get the minimum, maximum, and average times
	over a set of iterations.
Solution:
	Lots. Had to pull the MPI Comm code from the "pio_engine" module and
	place it in the "pio_perf" module. Then worked on a way to have all
	processes send their time output to process 0, who collects it and
	gives back the min, max, and avg times for the iterations.
Platforms tested:
	Linux. Doesn't work if you use more than 1 processor...*hrmph*

4 files changed, 346 insertions, 139 deletions

diff --git a/perform/pio_engine.c b/perform/pio_engine.c
index c2e7ac4..cebefe4 100644
--- a/perform/pio_engine.c
+++ b/perform/pio_engine.c
@@ -30,7 +30,7 @@
 /* Macro definitions */
 
 /* sizes of various items. these sizes won't change during program execution */
-#define ELMT_SIZE           sizeof(int)                 /* we're doing ints */
+#define ELMT_SIZE           ((int)sizeof(int))          /* we're doing ints */
 
 #define GOTOERROR(errcode)	{ ret_code = errcode; goto done; }
 #define GOTODONE		{ goto done; }
@@ -93,10 +93,6 @@ enum {
 };
 
 /* Global variables */
-MPI_Comm    pio_comm_g;         /* Communicator to run the PIO          */
-int         pio_mpi_rank_g;     /* MPI rank of pio_comm_g               */
-int	        pio_mpi_nprocs_g;   /* number of processes of pio_comm_g    */
-
 static int	clean_file_g = -1;	/*whether to cleanup temporary test     */
                                 /*files. -1 is not defined;             */
                                 /*0 is no cleanup; 1 is do cleanup      */
@@ -152,7 +148,6 @@ results
 do_pio(parameters param)
 {
     /* return codes */
-    int         rc;             /*routine return code                   */
     int         mrc;            /*MPI return code                       */
     herr_t      ret_code = 0;   /*return code                           */
     results     res;
@@ -164,18 +159,12 @@ do_pio(parameters param)
     int         maxprocs;
     int		nfiles, nf;
     long        ndsets, nelmts;
-    int         color;                  /*for communicator creation     */
     char        *buffer = NULL;         /*data buffer pointer           */
     long        buf_size;               /*data buffer size in bytes     */
 
     /* HDF5 variables */
     herr_t          hrc;                /*HDF5 return code              */
 
-    /* MPI variables */
-    int             myrank, nprocs = 1;
-
-    pio_comm_g = MPI_COMM_NULL;
-
     /* Sanity check parameters */
 
     /* IO type */
@@ -234,15 +223,6 @@ do_pio(parameters param)
         GOTOERROR(FAIL);
     }
 
-    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-
-    if (maxprocs > nprocs) {
-        fprintf(stderr,
-                "maximum number of process(%d) must be <= process in MPI_COMM_WORLD(%d)\n",
-                maxprocs, nprocs);
-        GOTOERROR(FAIL);
-    }
-
     if (buf_size <= 0 ){
         fprintf(stderr,
                 "buffer size must be > 0 (%ld)\n", buf_size);
@@ -264,33 +244,12 @@ buf_size=MIN(1024*1024, buf_size);
 /* DEBUG END */
 #endif
 
-    /* Create a sub communicator for this PIO run. Easier to use the first N
-     * processes. */
-    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-    color = (myrank < maxprocs);
-    mrc = MPI_Comm_split(MPI_COMM_WORLD, color, myrank, &pio_comm_g);
-
-    if (mrc != MPI_SUCCESS) {
-        fprintf(stderr, "MPI_Comm_split failed\n");
-        GOTOERROR(FAIL);
-    }
-
-    if (!color){
-        /* not involved in this run */
-        mrc = MPI_Comm_free(&pio_comm_g);
-        GOTODONE;
-    }
-
-    /* determine the mpi rank of in the PIO communicator */
-    MPI_Comm_size(pio_comm_g, &pio_mpi_nprocs_g);
-    MPI_Comm_rank(pio_comm_g, &pio_mpi_rank_g);
-
     /* allocate data buffer */
-    buffer = malloc(buf_size);
+    buffer = malloc((size_t)buf_size);
 
     if (buffer == NULL){
         fprintf(stderr, "malloc for data buffer size (%ld) failed\n",
-	    buf_size);
+  	    buf_size);
         GOTOERROR(FAIL);
     }
 
@@ -307,15 +266,17 @@ buf_size=MIN(1024*1024, buf_size);
 fprintf(stderr, "filename=%s\n", fname);
 #endif
 
+        set_time(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, START);
+
         set_time(res.timers, HDF5_FILE_OPENCLOSE, START);
         hrc = do_fopen(iot, fname, &fd, PIO_CREATE | PIO_WRITE);
         set_time(res.timers, HDF5_FILE_OPENCLOSE, STOP);
 
         VRFY((hrc == SUCCESS), "do_fopen failed");
 
-        set_time(res.timers, HDF5_WRITE_FIXED_DIMS, START);
+        set_time(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, START);
         hrc = do_write(&fd, iot, ndsets, nelmts, buf_size, buffer);
-        set_time(res.timers, HDF5_WRITE_FIXED_DIMS, STOP);
+        set_time(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, STOP);
 
         VRFY((hrc == SUCCESS), "do_write failed");
 
@@ -324,21 +285,25 @@ fprintf(stderr, "filename=%s\n", fname);
         hrc = do_fclose(iot, &fd);
         set_time(res.timers, HDF5_FILE_OPENCLOSE, STOP);
 
+        set_time(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, STOP);
+
         VRFY((hrc == SUCCESS), "do_fclose failed");
 
 	/*
 	 * Read performance measurement
 	 */
         /* Open file for read */
+        set_time(res.timers, HDF5_GROSS_READ_FIXED_DIMS, START);
+
         set_time(res.timers, HDF5_FILE_OPENCLOSE, START);
         hrc = do_fopen(iot, fname, &fd, PIO_READ);
         set_time(res.timers, HDF5_FILE_OPENCLOSE, STOP);
 
         VRFY((hrc == SUCCESS), "do_fopen failed");
 
-        set_time(res.timers, HDF5_READ_FIXED_DIMS, START);
+        set_time(res.timers, HDF5_FINE_READ_FIXED_DIMS, START);
         hrc = do_read(&fd, iot, ndsets, nelmts, buf_size, buffer);
-        set_time(res.timers, HDF5_READ_FIXED_DIMS, STOP);
+        set_time(res.timers, HDF5_FINE_READ_FIXED_DIMS, STOP);
 
         VRFY((hrc == SUCCESS), "do_read failed");
 
@@ -347,6 +312,8 @@ fprintf(stderr, "filename=%s\n", fname);
         hrc = do_fclose(iot, &fd);
         set_time(res.timers, HDF5_FILE_OPENCLOSE, STOP);
 
+        set_time(res.timers, HDF5_GROSS_READ_FIXED_DIMS, STOP);
+
         VRFY((hrc == SUCCESS), "do_fclose failed");
 
         do_cleanupfile(iot, fname);
@@ -373,16 +340,6 @@ done:
         break;
     }
 
-    /* release MPI resources */
-    if (pio_comm_g != MPI_COMM_NULL){
-        mrc = MPI_Comm_free(&pio_comm_g);
-
-        if (mrc != MPI_SUCCESS) {
-            fprintf(stderr, "MPI_Comm_free failed\n");
-            ret_code = FAIL;
-        }
-    }
-
     /* release generic resources */
     free(buffer);
     res.ret_code = ret_code;
@@ -637,8 +594,8 @@ fprintf(stderr, "proc %d: writes %ld bytes at file-offset %ld\n",
 
                 rc = RAWSEEK(fd->rawfd, file_offset);
                 VRFY((rc>=0), "RAWSEEK");
-                rc = RAWWRITE(fd->rawfd, buffer, nelmts_towrite*ELMT_SIZE);
-                VRFY((rc==(nelmts_towrite*ELMT_SIZE)), "RAWWRITE");
+                rc = RAWWRITE(fd->rawfd, buffer, (size_t)(nelmts_towrite * ELMT_SIZE));
+                VRFY((rc == (nelmts_towrite*ELMT_SIZE)), "RAWWRITE");
                 break;
 
             case MPIO:
@@ -735,7 +692,7 @@ done:
  */
 static herr_t
 do_read(file_descr *fd, iotype iot, long ndsets,
-         long nelmts, long buf_size, void *buffer /*out*/)
+        long nelmts, long buf_size, void *buffer /*out*/)
 {
     int         ret_code = SUCCESS;
     int         rc;             /*routine return code                   */
@@ -849,7 +806,7 @@ fprintf(stderr, "proc %d: read %ld bytes at file-offset %ld\n",
 
                 rc = RAWSEEK(fd->rawfd, file_offset);
                 VRFY((rc>=0), "RAWSEEK");
-                rc = RAWREAD(fd->rawfd, buffer, nelmts_toread*ELMT_SIZE);
+                rc = RAWREAD(fd->rawfd, buffer, (size_t)(nelmts_toread*ELMT_SIZE));
                 VRFY((rc==(nelmts_toread*ELMT_SIZE)), "RAWREAD");
                 break;
 
@@ -858,7 +815,7 @@ fprintf(stderr, "proc %d: read %ld bytes at file-offset %ld\n",
 
 #if AKCDEBUG
 fprintf(stderr, "proc %d: read %ld bytes at mpi-offset %ld\n",
-    pio_mpi_rank_g, nelmts_toread*ELMT_SIZE, mpi_offset);
+        pio_mpi_rank_g, nelmts_toread*ELMT_SIZE, mpi_offset);
 #endif
 
                 mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buffer,
@@ -992,7 +949,7 @@ do_fopen(iotype iot, char *fname, file_descr *fd /*out*/, int flags)
 
             /*since MPI_File_open with MPI_MODE_CREATE does not truncate  */
             /*filesize , set size to 0 explicitedly.	*/
-            mrc = MPI_File_set_size(fd->mpifd, 0);
+            mrc = MPI_File_set_size(fd->mpifd, (MPI_Offset)0);
 
             if (mrc != MPI_SUCCESS) {
                 fprintf(stderr, "MPI_File_set_size failed\n");
diff --git a/perform/pio_perf.c b/perform/pio_perf.c
index 8439f36..731df9b 100644
--- a/perform/pio_perf.c
+++ b/perform/pio_perf.c
@@ -50,6 +50,7 @@
  */
 
 /* system header files */
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -80,6 +81,12 @@
 #define MIN_HDF5_BUF_SIZE   (ONE_MB >> 1)
 #define MAX_HDF5_BUF_SIZE   (ONE_GB / 2)
 
+/* global variables */
+MPI_Comm    pio_comm_g;         /* Communicator to run the PIO          */
+int         pio_mpi_rank_g;     /* MPI rank of pio_comm_g               */
+int	        pio_mpi_nprocs_g;   /* number of processes of pio_comm_g    */
+
+
 /* local variables */
 static const char  *progname = "pio_perf";
 
@@ -196,11 +203,23 @@ struct options {
     long min_xfer_size;         /* minimum transfer buffer size         */
 };
 
+typedef struct _minmax {
+    double min;
+    double max;
+    double sum;
+    int num;
+} minmax;
+
 /* local functions */
 static long parse_size_directive(const char *size);
 static struct options *parse_command_line(int argc, char *argv[]);
 static void run_test_loop(FILE *output, struct options *options);
-static void run_test(FILE *output, iotype iot, parameters parms);
+static int run_test(FILE *output, iotype iot, parameters parms);
+static void get_minmax(minmax *mm);
+static minmax accumulate_minmax_stuff(minmax *mm, int count);
+static int create_comm_world(int num_procs);
+static int destroy_comm_world(void);
+static void output_report(FILE *output, const char *fmt, ...);
 static void print_indent(register FILE *output, register int indent);
 static void usage(const char *prog);
 
@@ -218,17 +237,7 @@ main(int argc, char **argv)
     int world_size, ret;
     int exit_value = EXIT_SUCCESS;
     FILE *output = stdout;
-    struct options *opts;
-
-    opts = parse_command_line(argc, argv);
-
-    if (opts->output_file) {
-        if ((output = fopen(opts->output_file, "w")) == NULL) {
-            fprintf(stderr, "%s: cannot open output file\n", progname);
-            perror(opts->output_file);
-            goto onions;
-        }
-    }
+    struct options *opts = NULL;
 
     /* initialize MPI and get the maximum num of processors we started with */
     MPI_Init(&argc, &argv);
@@ -243,15 +252,30 @@ main(int argc, char **argv)
             fprintf(stderr, "invalid argument\n");
 
         exit_value = EXIT_FAILURE;
-        goto cheese_and;
+        goto finish;
+    }
+
+    pio_comm_g = MPI_COMM_WORLD;
+
+    opts = parse_command_line(argc, argv);
+
+    if (!opts) {
+        exit_value = EXIT_FAILURE;
+        goto finish;
+    }
+
+    if (opts->output_file) {
+        if ((output = fopen(opts->output_file, "w")) == NULL) {
+            fprintf(stderr, "%s: cannot open output file\n", progname);
+            perror(opts->output_file);
+            goto finish;
+        }
     }
 
     run_test_loop(output, opts);
 
-cheese_and:
+finish:
     MPI_Finalize();
-
-onions:
     free(opts);
     return exit_value;
 }
@@ -301,24 +325,31 @@ run_test_loop(FILE *output, struct options *opts)
     parms.num_dsets = opts->num_dsets;
     parms.num_iters = opts->num_iters;
 
-    /* divide the maximum number of processors by 2 for each loop iter */
+    /* multiply the maximum number of processors by 2 for each loop iter */
     for (num_procs = opts->min_num_procs;
             num_procs <= opts->max_num_procs; num_procs <<= 1) {
-        register long j;
+        register long buf_size;
 
         parms.num_procs = num_procs;
-        fprintf(output, "Number of processors = %u\n", parms.num_procs);
 
-        for (j = opts->min_xfer_size; j <= opts->max_xfer_size; j <<= 1) {
-            parms.buf_size = j;
+        if (create_comm_world(parms.num_procs) != SUCCESS) {
+            /* do something harsh */
+        }
+
+        output_report(output, "Number of processors = %ld\n", parms.num_procs);
+
+        /* multiply the xfer buffer size by 2 for each loop iteration */
+        for (buf_size = opts->min_xfer_size;
+                buf_size <= opts->max_xfer_size; buf_size <<= 1) {
+            parms.buf_size = buf_size;
             parms.num_elmts = opts->file_size / (parms.num_dsets * sizeof(int));
 
             print_indent(output, TAB_SPACE * 1);
-            fprintf(output, "Transfer Buffer Size: %u\n", j);
+            output_report(output, "Transfer Buffer Size: %ld\n", buf_size);
             print_indent(output, TAB_SPACE * 1);
-            fprintf(output,
-                    "  # of files: %u, # of dsets: %lu, Elements per dset: %lu\n",
-                    parms.num_files, parms.num_dsets, parms.num_elmts);
+            output_report(output,
+                          "  # of files: %ld, # of dsets: %ld, # of elmts per dset: %ld\n",
+                          parms.num_files, parms.num_dsets, parms.num_elmts);
 
             if (io_runs & PIO_RAW)
                 run_test(output, RAW, parms);
@@ -329,6 +360,10 @@ run_test_loop(FILE *output, struct options *opts)
             if (io_runs & PIO_HDF5)
                 run_test(output, PHDF5, parms);
         }
+
+        if (destroy_comm_world() != SUCCESS) {
+            /* do something harsh */
+        }
     }
 }
 
@@ -339,41 +374,248 @@ run_test_loop(FILE *output, struct options *opts)
  * Programmer:  Bill Wendling, 18. December 2001
  * Modifications:
  */
-static void
+static int
 run_test(FILE *output, iotype iot, parameters parms)
 {
-    results res;
+    results         res;
+    register int    i, ret_value = SUCCESS;
+    int             comm_size;
+    minmax          total_mm;
+    minmax         *write_mm_table;
+    minmax         *read_mm_table;
+    minmax          write_mm = {0.0, 0.0, 0.0, 0};
+    minmax          read_mm = {0.0, 0.0, 0.0, 0};
 
     parms.io_type = iot;
     print_indent(output, TAB_SPACE * 2);
-    fprintf(output, "Type of IO = ");
+    output_report(output, "Type of IO = ");
 
     switch (iot) {
     case RAW:
-        fprintf(output, "Raw\n");
+        output_report(output, "Raw\n");
         break;
     case MPIO:
-        fprintf(output, "MPIO\n");
+        output_report(output, "MPIO\n");
         break;
     case PHDF5:
-        fprintf(output, "PHDF5\n");
+        output_report(output, "PHDF5\n");
         break;
     }
 
+    MPI_Comm_size(pio_comm_g, &comm_size);
+
+    write_mm_table = malloc(parms.num_iters * sizeof(minmax));
+    read_mm_table = malloc(parms.num_iters * sizeof(minmax));
+
+    for (i = 0; i < parms.num_iters; ++i) {
+        write_mm_table[i].min = 0.0;
+        write_mm_table[i].max = 0.0;
+        write_mm_table[i].sum = 0.0;
+        write_mm_table[i].num = 0;
+
+        read_mm_table[i].min = 0.0;
+        read_mm_table[i].max = 0.0;
+        read_mm_table[i].sum = 0.0;
+        read_mm_table[i].num = 0;
+    }
+
     /* call Albert's testing here */
-    res = do_pio(parms); 
+    for (i = 0; i < parms.num_iters; ++i) {
+        register int j;
+        double t;
+
+        MPI_Barrier(pio_comm_g);
+        res = do_pio(parms);
+
+        t = get_time(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS);
+        MPI_Send((void *)&t, 1, MPI_DOUBLE, 0, 0, pio_comm_g);
+
+        for (j = 0; j < comm_size; ++j)
+            get_minmax(&write_mm);
+
+        write_mm_table[i] = write_mm;
+
+        t = get_time(res.timers, HDF5_GROSS_READ_FIXED_DIMS);
+        MPI_Send((void *)&t, 1, MPI_DOUBLE, 0, 0, pio_comm_g);
+
+        for (j = 0; j < comm_size; ++j)
+            get_minmax(&read_mm);
+
+        read_mm_table[i] = read_mm;
+    }
+
+    total_mm = accumulate_minmax_stuff(write_mm_table, parms.num_iters);
+
+printf("write metrics: min: %f, max: %f, avg: %f\n", total_mm.min,
+       total_mm.max, total_mm.sum / total_mm.num);
+
+    total_mm = accumulate_minmax_stuff(read_mm_table, parms.num_iters);
+
+printf("read metrics: min: %f, max: %f, avg: %f\n", total_mm.min,
+       total_mm.max, total_mm.sum / total_mm.num);
 
     print_indent(output, TAB_SPACE * 3);
-    fprintf(output, "Write Results = %.2f MB/s\n",
-            MB_PER_SEC(parms.num_dsets * parms.num_elmts * sizeof(int),
-                       get_time(res.timers, HDF5_WRITE_FIXED_DIMS)));
+    output_report(output, "Write Results = %.2f MB/s\n",
+                  MB_PER_SEC(parms.num_dsets * parms.num_elmts * sizeof(int),
+                             get_time(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS)));
 
     print_indent(output, TAB_SPACE * 3);
-    fprintf(output, "Read Results = %.2f MB/s\n",
-            MB_PER_SEC(parms.num_dsets * parms.num_elmts * sizeof(int),
-                       get_time(res.timers, HDF5_READ_FIXED_DIMS)));
+    output_report(output, "Read Results = %.2f MB/s\n",
+                  MB_PER_SEC(parms.num_dsets * parms.num_elmts * sizeof(int),
+                             get_time(res.timers, HDF5_GROSS_READ_FIXED_DIMS)));
 
+    free(write_mm_table);
+    free(read_mm_table);
     pio_time_destroy(res.timers);
+    return ret_value;
+}
+
+static void
+get_minmax(minmax *mm)
+{
+    int myrank;
+
+    MPI_Comm_rank(pio_comm_g, &myrank);
+
+    if (myrank == 0) {
+        MPI_Status status = {0};
+        double t;
+
+        MPI_Recv((void *)&t, 1, MPI_DOUBLE, MPI_ANY_SOURCE,
+                 MPI_ANY_TAG, pio_comm_g, &status);
+
+        ++mm->num;
+        mm->sum += t;
+
+        if (t > mm->max)
+            mm->max = t;
+
+        if (t < mm->min || mm->min <= 0.0)
+            mm->min = t;
+    }
+}
+
+static minmax
+accumulate_minmax_stuff(minmax *mm, int count)
+{
+    register int i;
+    minmax total_mm = mm[0];
+
+    for (i = 1; i < count; ++i) {
+        total_mm.sum += mm[i].sum;
+        total_mm.num += mm[i].num;
+
+        if (mm[i].min < total_mm.min)
+            total_mm.min = mm[i].min;
+
+        if (mm[i].max > total_mm.max)
+            total_mm.max = mm[i].max;
+    }
+
+    return total_mm;
+}
+
+/*
+ * Function:    create_comm_world
+ * Purpose:     Create an MPI Comm world and store it in pio_comm_g, which
+ *              is a global variable.
+ * Return:      SUCCESS on success.
+ *              FAIL otherwise.
+ * Programmer:  Bill Wendling, 19. December 2001
+ * Modifications:
+ */
+static int
+create_comm_world(int num_procs)
+{
+    /* MPI variables */
+    int     mrc, ret_value;     /* return values                */
+    int     color;              /* for communicator creation    */
+    int     myrank, nprocs;
+
+    pio_comm_g = MPI_COMM_NULL;
+
+    /*
+     * Create a sub communicator for this PIO run. Easier to use the first N
+     * processes.
+     */
+    MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
+
+    if (num_procs > nprocs) {
+        fprintf(stderr,
+                "number of process(%d) must be <= number of processes in MPI_COMM_WORLD(%d)\n",
+                num_procs, nprocs);
+        goto error_done;
+    }
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+    color = (myrank < num_procs);
+    mrc = MPI_Comm_split(MPI_COMM_WORLD, color, myrank, &pio_comm_g);
+
+    if (mrc != MPI_SUCCESS) {
+        fprintf(stderr, "MPI_Comm_split failed\n");
+        goto error_done;
+    }
+
+    if (!color) {
+        /* not involved in this run */
+        mrc = destroy_comm_world();
+        goto done;
+    }
+
+    /* determine the MPI rank in the PIO communicator */
+    MPI_Comm_size(pio_comm_g, &pio_mpi_nprocs_g);
+    MPI_Comm_rank(pio_comm_g, &pio_mpi_rank_g);
+
+done:
+    return ret_value;
+
+error_done:
+    destroy_comm_world();
+    return FAIL;
+}
+
+/*
+ * Function:    destroy_comm_world
+ * Purpose:     Destroy the created MPI Comm world which is stored in the
+ *              pio_comm_g global variable.
+ * Return:      SUCCESS on success.
+ *              FAIL otherwise.
+ * Programmer:  Bill Wendling, 19. December 2001
+ * Modifications:
+ */
+static int
+destroy_comm_world(void)
+{
+    int     mrc = SUCCESS;      /* return code      */
+
+    /* release MPI resources */
+    if (pio_comm_g != MPI_COMM_NULL)
+        mrc = (MPI_Comm_free(&pio_comm_g) == MPI_SUCCESS ? SUCCESS : FAIL);
+
+    return mrc;
+}
+
+/*
+ * Function:    output_report
+ * Purpose:     Print a line of the report. Only do so if I'm the 0 process.
+ * Return:      Nothing
+ * Programmer:  Bill Wendling, 19. December 2001
+ * Modifications:
+ */
+static void
+output_report(FILE *output, const char *fmt, ...)
+{
+    int myrank;
+
+    MPI_Comm_rank(pio_comm_g, &myrank);
+
+    if (myrank == 0) {
+        va_list ap;
+
+        va_start(ap, fmt);
+        vfprintf(output, fmt, ap);
+        va_end(ap);
+    }
 }
 
 /*
@@ -465,13 +707,11 @@ parse_command_line(int argc, char *argv[])
             cl_opts->max_xfer_size = parse_size_directive(opt_arg);
             break;
         case 'h':
-            usage(progname);
-            exit(EXIT_SUCCESS);
         case '?':
         default:
-            /* there could be other command line options, such as MPI stuff 
-             * that gets passed to our program, for some reason */
-            break;
+            usage(progname);
+            free(cl_opts);
+            return NULL;
         }
     }
 
@@ -536,34 +776,40 @@ parse_size_directive(const char *size)
 static void
 usage(const char *prog)
 {
-    fflush(stdout);
-    fprintf(stdout, "usage: %s [OPTIONS]\n", prog);
-    fprintf(stdout, "  OPTIONS\n");
-    fprintf(stdout, "     -h, --help                  Print a usage message and exit\n");
-    fprintf(stdout, "     -d N, --num-dsets=N         Number of datasets per file [default:1]\n");
-    fprintf(stdout, "     -f S, --file-size=S         Size of a single file [default: 64M]\n");
-    fprintf(stdout, "     -F N, --num-files=N         Number of files [default: 1]\n");
-    fprintf(stdout, "     -H, --hdf5                  Run HDF5 performance test\n");
-    fprintf(stdout, "     -i, --num-iterations        Number of iterations to perform [default: 1]\n");
-    fprintf(stdout, "     -m, --mpiio                 Run MPI/IO performance test\n");
-    fprintf(stdout, "     -o F, --output=F            Output raw data into file F [default: none]\n");
-    fprintf(stdout, "     -P N, --max-num-processes=N Maximum number of processes to use [default: 1]\n");
-    fprintf(stdout, "     -p N, --min-num-processes=N Minimum number of processes to use [default: 1]\n");
-    fprintf(stdout, "     -r, --raw                   Run raw (UNIX) performance test\n");
-    fprintf(stdout, "     -X S, --max-xfer-size=S     Maximum transfer buffer size [default: 1M]\n");
-    fprintf(stdout, "     -x S, --min-xfer-size=S     Minimum transfer buffer size [default: 1K]\n");
-    fprintf(stdout, "\n");
-    fprintf(stdout, "  F - is a filename.\n");
-    fprintf(stdout, "  N - is an integer >=0.\n");
-    fprintf(stdout, "  S - is a size specifier, an integer >=0 followed by a size indicator:\n");
-    fprintf(stdout, "\n");
-    fprintf(stdout, "          K - Kilobyte\n");
-    fprintf(stdout, "          M - Megabyte\n");
-    fprintf(stdout, "          G - Gigabyte\n");
-    fprintf(stdout, "\n");
-    fprintf(stdout, "      Example: 37M = 37 Megabytes\n");
-    fprintf(stdout, "\n");
-    fflush(stdout);
+    int myrank;
+
+    MPI_Comm_rank(pio_comm_g, &myrank);
+
+    if (myrank == 0) {
+        fflush(stdout);
+        fprintf(stdout, "usage: %s [OPTIONS]\n", prog);
+        fprintf(stdout, "  OPTIONS\n");
+        fprintf(stdout, "     -h, --help                  Print a usage message and exit\n");
+        fprintf(stdout, "     -d N, --num-dsets=N         Number of datasets per file [default:1]\n");
+        fprintf(stdout, "     -f S, --file-size=S         Size of a single file [default: 64M]\n");
+        fprintf(stdout, "     -F N, --num-files=N         Number of files [default: 1]\n");
+        fprintf(stdout, "     -H, --hdf5                  Run HDF5 performance test\n");
+        fprintf(stdout, "     -i, --num-iterations        Number of iterations to perform [default: 1]\n");
+        fprintf(stdout, "     -m, --mpiio                 Run MPI/IO performance test\n");
+        fprintf(stdout, "     -o F, --output=F            Output raw data into file F [default: none]\n");
+        fprintf(stdout, "     -P N, --max-num-processes=N Maximum number of processes to use [default: 1]\n");
+        fprintf(stdout, "     -p N, --min-num-processes=N Minimum number of processes to use [default: 1]\n");
+        fprintf(stdout, "     -r, --raw                   Run raw (UNIX) performance test\n");
+        fprintf(stdout, "     -X S, --max-xfer-size=S     Maximum transfer buffer size [default: 1M]\n");
+        fprintf(stdout, "     -x S, --min-xfer-size=S     Minimum transfer buffer size [default: 1K]\n");
+        fprintf(stdout, "\n");
+        fprintf(stdout, "  F - is a filename.\n");
+        fprintf(stdout, "  N - is an integer >=0.\n");
+        fprintf(stdout, "  S - is a size specifier, an integer >=0 followed by a size indicator:\n");
+        fprintf(stdout, "\n");
+        fprintf(stdout, "          K - Kilobyte\n");
+        fprintf(stdout, "          M - Megabyte\n");
+        fprintf(stdout, "          G - Gigabyte\n");
+        fprintf(stdout, "\n");
+        fprintf(stdout, "      Example: 37M = 37 Megabytes\n");
+        fprintf(stdout, "\n");
+        fflush(stdout);
+    }
 }
 
 #else /* H5_HAVE_PARALLEL */
diff --git a/perform/pio_perf.h b/perform/pio_perf.h
index d954591..5ae3faa 100644
--- a/perform/pio_perf.h
+++ b/perform/pio_perf.h
@@ -39,6 +39,10 @@ typedef struct results_ {
 #define FAIL        -1
 #endif  /* !FAIL */
 
+extern MPI_Comm pio_comm_g;         /* Communicator to run the PIO          */
+extern int      pio_mpi_rank_g;     /* MPI rank of pio_comm_g               */
+extern int	    pio_mpi_nprocs_g;   /* number of processes of pio_comm_g    */
+
 #ifdef __cplusplus
 extern "C" {
 #endif  /* __cplusplus */
diff --git a/perform/pio_timer.h b/perform/pio_timer.h
index 258ecc9..dc723a0 100644
--- a/perform/pio_timer.h
+++ b/perform/pio_timer.h
@@ -20,12 +20,12 @@
 
 /* The different types of timers we can have */
 typedef enum timer_type_ {
-    HDF5_MPI_OVERHEAD,
     HDF5_FILE_OPENCLOSE,
-    HDF5_GROUP_CREATE,
     HDF5_DATASET_CREATE,
-    HDF5_WRITE_FIXED_DIMS,
-    HDF5_READ_FIXED_DIMS,
+    HDF5_FINE_WRITE_FIXED_DIMS,
+    HDF5_FINE_READ_FIXED_DIMS,
+    HDF5_GROSS_WRITE_FIXED_DIMS,
+    HDF5_GROSS_READ_FIXED_DIMS,
     NUM_TIMERS
 } timer_type;