diff options
| -rw-r--r-- | perform/pio_engine.c | 249 | ||||
| -rw-r--r-- | perform/pio_timer.c | 36 | ||||
| -rw-r--r-- | perform/pio_timer.h | 16 |
3 files changed, 223 insertions, 78 deletions
diff --git a/perform/pio_engine.c b/perform/pio_engine.c index cd2d134..abc1171 100644 --- a/perform/pio_engine.c +++ b/perform/pio_engine.c @@ -52,14 +52,6 @@ } \ } while(0) -#ifndef HDmalloc -#define HDmalloc(x) malloc(x) -#endif - -#ifndef HDfree -#define HDfree(x) free(x) -#endif - #ifndef HDopen #ifdef O_BINARY #define HDopen(S,F,M) open(S,F|_O_BINARY,M) @@ -98,6 +90,21 @@ enum { PIO_READ = 4 }; +/* + * In a parallel machine, the filesystem suitable for compiling is + * unlikely a parallel file system that is suitable for parallel I/O. + * There is no standard pathname for the parallel file system. /tmp + * is about the best guess. + */ +#ifndef HDF5_PARAPREFIX +# ifdef __PUMAGON__ + /* For the PFS of TFLOPS */ +# define HDF5_PARAPREFIX "pfs:/pfs_grande/multi/tmp_1" +# else +# define HDF5_PARAPREFIX "/tmp" +# endif /* __PUMAGON__ */ +#endif /* !HDF5_PARAPREFIX */ + /* the different types of file descriptors we can expect */ typedef union _file_descr { int rawfd; /* raw/Unix file */ @@ -106,8 +113,10 @@ typedef union _file_descr { } file_descr; /* local functions */ +static char *pio_create_filename(iotype iot, const char *base_name, + char *fullname, size_t size); static herr_t do_write(file_descr fd, iotype iot, unsigned long ndsets, - unsigned long nelmts, hid_t h5dset_space_id, char * buffer); + unsigned long nelmts, hid_t h5dset_space_id, char *buffer); static herr_t do_open(iotype iot, char *fname, file_descr fd /*out*/, int flags); static herr_t do_close(iotype iot, file_descr fd); @@ -122,7 +131,7 @@ do_pio(parameters param) file_descr fd; iotype iot; - char fname[256]; + char fname[FILENAME_MAX]; unsigned int maxprocs, nfiles, nf; unsigned long ndsets; unsigned long nelmts; @@ -144,21 +153,27 @@ do_pio(parameters param) MPI_Comm comm = MPI_COMM_NULL; int myrank, nprocs = 1; + pio_time *timer = NULL; + /* Sanity check parameters */ /* IO type */ iot = param.io_type; switch (iot) { - case RAW: case MPIO: + timer = pio_time_new(MPI_TIMER); + break; + + case RAW: case PHDF5: - /* nothing */ - break; + timer = pio_time_new(SYS_TIMER); + break; + default: - /* unknown request */ - fprintf(stderr, "Unknown IO type request (%d)\n", iot); - GOTOERROR(FAIL); + /* unknown request */ + fprintf(stderr, "Unknown IO type request (%d)\n", iot); + GOTOERROR(FAIL); } nfiles = param.num_files; /* number of files */ @@ -167,14 +182,14 @@ do_pio(parameters param) niters = param.num_iters; /* number of iterations of reads/writes */ maxprocs = param.max_num_procs; /* max number of mpi-processes to use */ - if (nelmts == 0 ){ + if (nelmts == 0 ) { fprintf(stderr, "number of elements per dataset must be > 0 (%lu)\n", nelmts); GOTOERROR(FAIL); } - if (maxprocs == 0 ){ + if (maxprocs == 0 ) { fprintf(stderr, "maximun number of process to use must be > 0 (%u)\n", maxprocs); @@ -183,7 +198,7 @@ do_pio(parameters param) MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - if (maxprocs > nprocs){ + if (maxprocs > nprocs) { fprintf(stderr, "maximun number of process(%d) must be <= process in MPI_COMM_WORLD(%d)\n", maxprocs, nprocs); @@ -203,7 +218,7 @@ nfiles=3; * processes. */ MPI_Comm_rank(comm, &myrank); color = (myrank < maxprocs); - mrc = MPI_Comm_split (MPI_COMM_WORLD, color, myrank, &comm); + mrc = MPI_Comm_split(MPI_COMM_WORLD, color, myrank, &comm); if (mrc != MPI_SUCCESS) { fprintf(stderr, "MPI_Comm_split failed\n"); @@ -221,7 +236,7 @@ nfiles=3; MPI_Comm_rank(comm, &myrank); /* allocate data buffer */ - buffer = HDmalloc(BUFFER_SIZE); + buffer = malloc(BUFFER_SIZE); if (buffer == NULL){ fprintf(stderr, "malloc for data buffer failed\n"); @@ -243,20 +258,11 @@ nfiles=3; for (nf = 1; nf <= nfiles; nf++) { /* Open file for write */ -MSG("creating file"); - sprintf(fname, "#pio_tmp_%u", nf); + char base_name[256]; - switch (iot) { - case RAW: - strcat(fname, ".raw"); - break; - case MPIO: - strcat(fname, ".mpio"); - break; - case PHDF5: - strcat(fname, ".h5"); - break; - } +MSG("creating file"); + sprintf(base_name, "#pio_tmp_%u", nf); + pio_create_filename(iot, base_name, fname, sizeof(fname)); rc = do_open(iot, fname, fd, PIO_CREATE | PIO_WRITE); VRFY((rc == SUCCESS), "do_open failed\n"); @@ -291,12 +297,13 @@ MSG("opening file to read"); MSG("closing read file"); rc = do_close(iot, fd); VRFY((rc == SUCCESS), "do_close failed\n"); + remove(fname); } done: /* clean up */ /* release HDF5 objects */ - if (h5dset_space_id != -1){ + if (h5dset_space_id != -1) { rc = H5Sclose(h5dset_space_id); if (rc < 0){ @@ -307,10 +314,10 @@ done: } } - if (h5mem_space_id != -1){ + if (h5mem_space_id != -1) { rc = H5Sclose(h5mem_space_id); - if (rc < 0){ + if (rc < 0) { fprintf(stderr, "HDF5 Memory Space Close failed\n"); ret_code = FAIL; } else { @@ -322,15 +329,127 @@ done: rc = do_close(iot, fd); /* release generic resources */ - HDfree(buffer); + free(buffer); + pio_time_destroy(timer); fprintf(stderr, "returning with ret_code=%d\n", ret_code); return ret_code; } /* + * Function: pio_create_filename + * Purpose: Create a new filename to write to. Determine the correct + * suffix to append to the filename by the type of I/O we're + * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if + * USER or LOGIN are specified in the environment. + * Return: Pointer to filename or NULL + * Programmer: Bill Wendling, 21. November 2001 + * Modifications: + */ +static char * +pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size) +{ + const char *prefix, *suffix; + char *ptr, last = '\0'; + size_t i, j; + + if (!base_name || !fullname || size < 1) + return NULL; + + memset(fullname, 0, size); + + switch (iot) { + case RAW: + suffix = ".raw"; + break; + case MPIO: + suffix = ".mpio"; + break; + case PHDF5: + suffix = ".h5"; + break; + } + + /* First use the environment variable and then try the constant */ + prefix = getenv("HDF5_PARAPREFIX"); + +#ifdef HDF5_PARAPREFIX + if (!prefix) + prefix = HDF5_PARAPREFIX; +#endif /* HDF5_PARAPREFIX */ + + /* Prepend the prefix value to the base name */ + if (prefix && *prefix) { + /* If the prefix specifies the HDF5_PARAPREFIX directory, then + * default to using the "/tmp/$USER" or "/tmp/$LOGIN" + * directory instead. */ + register char *user, *login, *subdir; + + user = getenv("USER"); + login = getenv("LOGIN"); + subdir = (user ? user : login); + + if (subdir) { + for (i = 0; i < size && prefix[i]; i++) + fullname[i] = prefix[i]; + + fullname[i++] = '/'; + + for (j = 0; i < size && subdir[j]; i++, j++) + fullname[i] = subdir[j]; + } else { + /* We didn't append the prefix yet */ + strncpy(fullname, prefix, MIN(strlen(prefix), size)); + } + + if ((strlen(fullname) + strlen(base_name) + 1) < size) { + /* Append the base_name with a slash first. Multiple slashes are + * handled below. */ + struct stat buf; + + if (stat(fullname, &buf) < 0) + /* The directory doesn't exist just yet */ + if (mkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) { + /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. + * Default to PREFIX's original prefix value. */ + strcpy(fullname, prefix); + } + + strcat(fullname, "/"); + strcat(fullname, base_name); + } else { + /* Buffer is too small */ + return NULL; + } + } else if (strlen(base_name) >= size) { + /* Buffer is too small */ + return NULL; + } else { + strcpy(fullname, base_name); + } + + /* Append a suffix */ + if (suffix) { + if (strlen(fullname) + strlen(suffix) >= size) + return NULL; + + strcat(fullname, suffix); + } + + /* Remove any double slashes in the filename */ + for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { + if (*ptr != '/' || last != '/') + fullname[j++] = *ptr; + + last = *ptr; + } + + return fullname; +} + +/* * Function: do_write - * Purpose: Write + * Purpose: Write the required amount of data to the file. * Return: SUCCESS or FAIL * Programmer: Bill Wendling, 14. November 2001 * Modifications: @@ -366,7 +485,12 @@ do_write(file_descr fd, iotype iot, unsigned long ndsets, sprintf(dname, "Dataset_%lu", ndset); h5ds_id = H5Dcreate(fd.h5fd, dname, H5T_NATIVE_INT, h5dset_space_id, H5P_DEFAULT); - VRFY((h5ds_id >= 0), "H5Dcreate"); + + if (h5ds_id < 0) { + fprintf(stderr, "HDF5 Dataset Create failed\n"); + GOTOERROR(FAIL); + } + break; } @@ -384,7 +508,7 @@ do_write(file_descr fd, iotype iot, unsigned long ndsets, /*Prepare write data*/ { - int *intptr = (int*)buffer; + int *intptr = (int *)buffer; register int i; for (i = 0; i < nelmts_towrite; ++i) @@ -401,13 +525,10 @@ do_write(file_descr fd, iotype iot, unsigned long ndsets, break; case MPIO: - break; - case PHDF5: break; } - nelmts_written += nelmts_towrite; fprintf(stderr, "wrote %lu elmts, %lu written\n", nelmts_towrite, nelmts_written); } @@ -418,7 +539,11 @@ fprintf(stderr, "wrote %lu elmts, %lu written\n", nelmts_towrite, nelmts_written if (iot == PHDF5){ herr_t hrc = H5Dclose(h5ds_id); - VRFY((hrc >= 0), "HDF5 Dataset Close failed\n"); + if (hrc < 0) { + fprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + h5ds_id = -1; } } @@ -427,6 +552,13 @@ done: return ret_code; } +/* + * Function: do_open + * Purpose: Open the specified file. + * Return: SUCCESS or FAIL + * Programmer: Bill Wendling, 14. November 2001 + * Modifications: + */ static herr_t do_open(iotype iot, char *fname, file_descr fd /*out*/, int flags) { @@ -439,7 +571,7 @@ do_open(iotype iot, char *fname, file_descr fd /*out*/, int flags) if ((flags | PIO_CREATE) || (flags | PIO_WRITE)) { fd.rawfd = RAWCREATE(fname); } else { - fd.rawfd = RAWOPEN(fname, O_RDONLY); + fd.rawfd = RAWOPEN(fname, O_RDONLY); } if (fd.rawfd < 0 ) { @@ -454,7 +586,7 @@ do_open(iotype iot, char *fname, file_descr fd /*out*/, int flags) mrc = MPI_File_open(comm, fname, MPI_MODE_CREATE | MPI_MODE_RDWR, MPI_INFO_NULL, &fd.mpifd); } else { - mrc = MPI_File_open(comm, fname, MPI_MODE_RDONLY, + mrc = MPI_File_open(comm, fname, MPI_MODE_RDONLY, MPI_INFO_NULL, &fd.mpifd); } @@ -467,26 +599,39 @@ do_open(iotype iot, char *fname, file_descr fd /*out*/, int flags) case PHDF5: acc_tpl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((acc_tpl >= 0), ""); + + if (acc_tpl < 0) { + fprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + hrc = H5Pset_fapl_mpio(acc_tpl, comm, MPI_INFO_NULL); - VRFY((hrc >= 0), ""); + + if (hrc < 0) { + fprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* create the parallel file */ if ((flags | PIO_CREATE) || (flags | PIO_WRITE)) { fd.h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); } else { - fd.h5fd = H5Fopen(fname, H5P_DEFAULT, acc_tpl); + fd.h5fd = H5Fopen(fname, H5P_DEFAULT, acc_tpl); } hrc = H5Pclose(acc_tpl); if (fd.h5fd < 0) { - fprintf(stderr, "HDF5 file Create failed(%s)\n", fname); + fprintf(stderr, "HDF5 File Create failed(%s)\n", fname); GOTOERROR(FAIL); } /* verifying the close of the acc_tpl */ - VRFY((hrc >= 0), "H5Pclose"); + if (hrc < 0) { + fprintf(stderr, "HDF5 Property List Close failed\n"); + GOTOERROR(FAIL); + } + break; } diff --git a/perform/pio_timer.c b/perform/pio_timer.c index 6fc078e..247b3c3 100644 --- a/perform/pio_timer.c +++ b/perform/pio_timer.c @@ -26,35 +26,35 @@ #define MILLISECOND 1000000.0 /* - * Function: perf_time_new + * Function: pio_time_new * Purpose: Build us a brand, spankin', new performance time object. * The object is a black box to the user. They just tell us * what type of timer they want (MPI_TIMER for MPI_Wtime or * SYS_TIMER for system time). - * Return: Pointer to perf_time object + * Return: Pointer to pio_time object * Programmer: Bill Wendling, 01. October 2001 * Modifications: */ -perf_time * -perf_time_new(unsigned int type) +pio_time * +pio_time_new(unsigned int type) { - perf_time *pt = (perf_time *)calloc(1, sizeof(struct perf_time_)); + pio_time *pt = (pio_time *)calloc(1, sizeof(struct pio_time_)); pt->type = type; return pt; } /* - * Function: perf_time_destroy - * Purpose: Remove the memory allocated for the perf_time object. Only - * need to call on a pointer allocated with the ``perf_time_new'' + * Function: pio_time_destroy + * Purpose: Remove the memory allocated for the pio_time object. Only + * need to call on a pointer allocated with the ``pio_time_new'' * function. * Return: Nothing * Programmer: Bill Wendling, 01. October 2001 * Modifications: */ void -perf_time_destroy(perf_time *pt) +pio_time_destroy(pio_time *pt) { free(pt); } @@ -63,13 +63,13 @@ perf_time_destroy(perf_time *pt) * Function: set_timer_type * Purpose: Set the type of the timer to either MPI_TIMER or SYS_TIMER. * This really only needs to be called if you didn't construct a - * timer with the perf_timer_new function (shame!). + * timer with the pio_timer_new function (shame!). * Return: Nothing * Programmer: Bill Wendling, 04. October 2001 * Modifications: */ void -set_timer_type(perf_time *pt, timer_type type) +set_timer_type(pio_time *pt, timer_type type) { pt->type = type; } @@ -82,20 +82,20 @@ set_timer_type(perf_time *pt, timer_type type) * Modifications: */ timer_type -get_timer_type(perf_time *pt) +get_timer_type(pio_time *pt) { return pt->type; } /* * Function: set_time - * Purpose: Set the time in a ``perf_time'' object. - * Return: Pointer to the passed in ``perf_time'' object. + * Purpose: Set the time in a ``pio_time'' object. + * Return: Pointer to the passed in ``pio_time'' object. * Programmer: Bill Wendling, 01. October 2001 * Modifications: */ -perf_time * -set_time(perf_time *pt, timer_type t, int start_stop) +pio_time * +set_time(pio_time *pt, timer_type t, int start_stop) { if (pt) { if (pt->type == MPI_TIMER) { @@ -122,13 +122,13 @@ set_time(perf_time *pt, timer_type t, int start_stop) /* * Function: get_time - * Purpose: Get the time from a ``perf_time'' object. + * Purpose: Get the time from a ``pio_time'' object. * Return: The number of seconds as a DOUBLE. * Programmer: Bill Wendling, 01. October 2001 * Modifications: */ double -get_time(perf_time *pt, timer_type t) +get_time(pio_time *pt, timer_type t) { return pt->total_time[t]; } diff --git a/perform/pio_timer.h b/perform/pio_timer.h index 620d0c4..258ecc9 100644 --- a/perform/pio_timer.h +++ b/perform/pio_timer.h @@ -39,23 +39,23 @@ enum { }; /* The performance time structure */ -typedef struct perf_time_ { +typedef struct pio_time_ { unsigned int type : 1; double total_time[NUM_TIMERS]; double mpi_timer[NUM_TIMERS]; struct timeval sys_timer[NUM_TIMERS]; -} perf_time; +} pio_time; /* External function declarations */ #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ -extern perf_time *perf_time_new(unsigned int); -extern void perf_time_destroy(perf_time *pt); -extern void set_timer_type(perf_time *pt, timer_type type); -extern timer_type get_timer_type(perf_time *pt); -extern perf_time *set_time(perf_time *pt, timer_type t, int start_stop); -extern double get_time(perf_time *pt, timer_type t); +extern pio_time *pio_time_new(unsigned int); +extern void pio_time_destroy(pio_time *pt); +extern void set_timer_type(pio_time *pt, timer_type type); +extern timer_type get_timer_type(pio_time *pt); +extern pio_time *set_time(pio_time *pt, timer_type t, int start_stop); +extern double get_time(pio_time *pt, timer_type t); #ifdef __cplusplus } #endif /* __cplusplus */ |