From 61126f5ce68c612fcb1f1938b1146985f263bde5 Mon Sep 17 00:00:00 2001 From: Richard Warren Date: Sun, 27 Sep 2020 19:31:33 -0400 Subject: Various edits to remove #if 0 blocks in an effort to make the code a bit cleaner --- SUBFILING_README.txt | 30 +++++++++++++++++ src/H5FDsubfile_mpi.c | 91 +++++--------------------------------------------- src/H5FDsubfiling.c | 17 +--------- test/h5subfiling_vol.c | 16 +-------- 4 files changed, 40 insertions(+), 114 deletions(-) diff --git a/SUBFILING_README.txt b/SUBFILING_README.txt index 71dd9e4..b09c592 100644 --- a/SUBFILING_README.txt +++ b/SUBFILING_README.txt @@ -42,10 +42,40 @@ that it supports Multithreading, but only MPI_THREAD_FUNNELED or potentially MPI_THREAD_SERIALIZED. The initial benchmarking efforts utilize OpenMPI to provide the necessary thread safe functionality. +At login, the modules preloaded by default are: + +rawarren@cori06:~> module list +Currently Loaded Modulefiles: + 1) modules/3.2.11.4 12) gni-headers/5.0.12.0-7.0.1.1_6.31__g3b1768f.ari + 2) altd/2.0 13) xpmem/2.2.20-7.0.1.1_4.13__g0475745.ari + 3) darshan/3.1.7 14) job/2.2.4-7.0.1.1_3.40__g36b56f4.ari + 4) intel/19.0.3.199 15) dvs/2.12_2.2.157-7.0.1.1_9.4__g083131db + 5) craype-network-aries 16) alps/6.6.58-7.0.1.1_6.8__g437d88db.ari + 6) craype/2.6.2 17) rca/2.2.20-7.0.1.1_4.51__g8e3fb5b.ari + 7) cray-libsci/19.06.1 18) atp/2.1.3 + 8) udreg/2.3.2-7.0.1.1_3.36__g8175d3d.ari 19) PrgEnv-intel/6.0.5 + 9) ugni/6.0.14.0-7.0.1.1_7.38__ge78e5b0.ari 20) craype-haswell + 10) pmi/5.0.14 21) cray-mpich/7.7.10 + 11) dmapp/7.1.1-7.0.1.1_4.52__g38cf134.ari 22) craype-hugepages2M +rawarren@cori06:~> + Interestingly, the default C compiler (or at least a compiler wrapper which calls the actual C compiler) appears to know about the default MPI and thus avoids the necessity of users providing an include path (-I) or a library link path (-L) when creating executables or shared objects (libraries). As a consequence, we unload these specific modules prior to selecting alternatives for compilation and for the MPI library implementation. +These defaults need to overridden, e.g. + +MODULE UNLOADS:: +rawarren@cori06:~> module unload cray-mpich/7.7.10 +rawarren@cori06:~> module unload craype/2.6.2 + +MODULE LOADS:: +rawarren@cori06:~> module load gcc/9.3.0 +rawarren@cori06:~> module load openmpi/4.0.2 + + + + diff --git a/src/H5FDsubfile_mpi.c b/src/H5FDsubfile_mpi.c index 57add71..5bffdad 100644 --- a/src/H5FDsubfile_mpi.c +++ b/src/H5FDsubfile_mpi.c @@ -1192,6 +1192,7 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset, int64_t ioc_read_datasize[n_io_concentrators]; int64_t ioc_read_offset[n_io_concentrators]; MPI_Datatype ioc_read_type[n_io_concentrators]; + useconds_t delay = 50; subfiling_context_t *sf_context = get_subfiling_object(context_id); assert(sf_context != NULL); @@ -1229,22 +1230,12 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset, #ifndef NDEBUG if (sf_verbose_flag) { -#if 0 - if (sf_logfile) { - fprintf(sf_logfile, - "[%d %s] Requesting %ld read bytes from IOC(%d): " - "sourceOffset=%ld subfile_offset=%ld\n", - sf_world_rank, __func__, msg[0], io_concentrator[ioc], - sourceOffset, msg[1]); - } -#else fprintf(stdout, "[%d %s] Requesting %ld read bytes from IOC(%d): " "sourceOffset=%ld subfile_offset=%ld\n", sf_world_rank, __func__, msg[0], io_concentrator[ioc], sourceOffset, msg[1]); fflush(stdout); -#endif } #endif @@ -1299,22 +1290,12 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset, for (i = 0; i < ready; i++) { #ifndef NDEBUG if (sf_verbose_flag) { -#if 0 - if (sf_logfile) { - fprintf(sf_logfile, - "[%d] READ bytes(%ld) of data from ioc_concentrator %d " - "complete\n", - sf_world_rank, ioc_read_datasize[indices[i]], - indices[i]); - } -#else fprintf(stdout, "[%d] READ bytes(%ld) of data from ioc_concentrator %d " "complete\n", sf_world_rank, ioc_read_datasize[indices[i]], indices[i]); fflush(stdout); -#endif } #endif if (ioc_read_type[indices[i]] != MPI_BYTE) { @@ -1322,6 +1303,8 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset, } n_waiting--; } + if (n_waiting) + usleep(delay); } return status; } @@ -1455,11 +1438,12 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset, int64_t ioc_write_datasize[n_io_concentrators]; int64_t ioc_write_offset[n_io_concentrators]; MPI_Datatype ioc_write_type[n_io_concentrators]; - + useconds_t delay = 50; subfiling_context_t *sf_context = get_subfiling_object(context_id); int i, target, ioc, n_waiting = 0, status = 0; int errors = 0; + io_concentrator = sf_context->topology->io_concentrator; if (sf_context->topology->rank_is_ioc) { @@ -1516,22 +1500,12 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset, #ifndef NDEBUG if (sf_verbose_flag) { -#if 0 - if (sf_logfile) { - fprintf(sf_logfile, - "[%d %s]: write_dest[ioc(%d), " - "sourceOffset=%ld, datasize=%ld, foffset=%ld]\n", - sf_world_rank, __func__, ioc, sourceOffset, - ioc_write_datasize[ioc], ioc_write_offset[ioc]); - } -#else fprintf(stdout, "[%d %s]: write_dest[ioc(%d), " "sourceOffset=%ld, datasize=%ld, foffset=%ld]\n", sf_world_rank, __func__, ioc, sourceOffset, ioc_write_datasize[ioc], ioc_write_offset[ioc]); fflush(stdout); -#endif } #endif @@ -1619,7 +1593,6 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset, fflush(stdout); errors++; } - for (i = 0; i < ready; i++) { /* One of the Issend calls has completed * If we used a derived type to send data, then should free @@ -1630,6 +1603,8 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset, } n_waiting--; } + if (n_waiting) + usleep(delay); } if (errors) return -1; @@ -2041,9 +2016,6 @@ sf_open_subfiles(hid_t fid, char *filename, char *prefix, int flags) puts("H5FDsubfiling_init failed!"); return -1; } -#if 0 - printf("[%d %s]\n", sf_world_rank, __func__); -#endif sf_context = get_subfiling_object(context_id); assert(sf_context != NULL); @@ -2194,11 +2166,6 @@ ioc_main(int64_t context_id) sf_close_file_count = 0; sf_ops_after_first_close = 0; -#if 0 - printf("Starting IOC! mpi_rank=%d\n", sf_world_rank); - fflush(stdout); -#endif - while (!sf_shutdown_flag || sf_work_pending) { flag = 0; ret = MPI_Iprobe( @@ -2220,29 +2187,6 @@ ioc_main(int64_t context_id) MPI_BYTE, source, tag, context->sf_msg_comm, &msg_status); } if (ret == MPI_SUCCESS) { -#if 0 - if (tag == OPEN_OP) { - sf_open_file_count++; - printf("source=%d: sf_open_file_count = %d\n", source, sf_open_file_count); - fflush(stdout); - } - else if (tag == CLOSE_OP) { - sf_close_file_count++; - printf("source=%d: sf_close_file_count = %d\n", source, sf_close_file_count); - fflush(stdout); - } - else { - printf("ioc(0): tag=%d\n", tag); - fflush(stdout); - if (sf_close_file_count) { - sf_ops_after_first_close++; - if (sf_close_file_count == sf_world_size) { - printf("op=%d from source(%d) after file close! sf_open_file_count=%d\n", tag, source, sf_open_file_count); - fflush(stdout); - } - } - } -#endif if (msg) { msg->source = source; msg->subfile_rank = subfile_rank; @@ -2620,14 +2564,9 @@ queue_file_open( } } #endif -#if 0 - printf("[ioc(%d) %s]\n", subfile_rank, __func__); - fflush(stdout); -#endif errors = subfiling_open_file(msg, sf_subfile_prefix, subfile_rank, flags); // open_count = atomic_load(&sf_file_refcount); -#if 1 ret = MPI_Send(&errors, 1, MPI_INT, source, COMPLETED, comm); if (ret != MPI_SUCCESS) { printf("[ioc(%d)] MPI_Send FILE_OPEN, COMPLETED to source(%d) FAILED\n", @@ -2635,21 +2574,7 @@ queue_file_open( fflush(stdout); errors++; } -#else - if (open_count == sf_world_size) { - int i, k = (sf_world_rank +1); - for (i=0; i < sf_world_size; i++, k++) { - source = k % sf_world_size; - ret = MPI_Send(&errors, 1, MPI_INT, source, COMPLETED, comm); - if (ret != MPI_SUCCESS) { - printf("[ioc(%d)] MPI_Send FILE_OPEN, COMPLETED to source(%d) FAILED\n", - subfile_rank, source); - fflush(stdout); - errors++; - } - } - } -#endif + if (errors) { #ifndef NDEBUG if (sf_verbose_flag) { diff --git a/src/H5FDsubfiling.c b/src/H5FDsubfiling.c index 7fbfdc7..99ea9ac 100644 --- a/src/H5FDsubfiling.c +++ b/src/H5FDsubfiling.c @@ -697,16 +697,8 @@ H5FD_subfiling_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t max * We can be a bit more efficient by having rank 0 broadcast * the stat buffer. */ -#if 0 - if (mpi_enabled && (my_rank == 0)) { - int sb_size = sizeof(sb); - - MPI_Bcast(&sb, sb_size, MPI_BYTE, 0, MPI_COMM_WORLD); - } -#else if(HDfstat(fd, &sb) < 0) HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file") -#endif /* Create the new file struct */ if(NULL == (file = H5FL_CALLOC(H5FD_subfiling_t))) @@ -929,9 +921,6 @@ H5FD_subfiling_query(const H5FD_t *_file, unsigned long *flags /* out */) *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ *flags |= H5FD_FEAT_POSIX_COMPAT_HANDLE; /* get_handle callback returns a POSIX file descriptor */ *flags |= H5FD_FEAT_SUPPORTS_SWMR_IO; /* VFD supports the single-writer/multiple-readers (SWMR) pattern */ -#if 0 - *flags |= H5FD_FEAT_HAS_MPI; /* FIXME:: for experimentation only... */ -#endif /* Check for flags that are set by h5repart */ if(file && file->fam_to_single) *flags |= H5FD_FEAT_IGNORE_DRVRINFO; /* Ignore the driver info when file is opened (which eliminates it) */ @@ -1755,13 +1744,9 @@ H5FD__dataset_write_contiguous(hid_t h5_file_id, haddr_t dataset_baseAddr, size_ { int status; haddr_t rank_baseAddr; -#if 0 - rank_baseAddr = get_data_offset(mpi_rank, mpi_size, dtype_extent, mem_space, file_space); - -#else rank_baseAddr = get_base_offset(mpi_rank, mpi_size, mem_space_id, file_space_id); rank_baseAddr += dataset_baseAddr; -#endif + // printf("[%d] H5S_SEL_HYPERSLABS, file_offset = %lld\n", mpi_rank, rank_baseAddr ); if ((status = H5Sis_regular_hyperslab(file_space_id)) < 0) { puts("H5Sis_regular_hyperslab returned an error"); diff --git a/test/h5subfiling_vol.c b/test/h5subfiling_vol.c index 25f1292..6db5aa2 100644 --- a/test/h5subfiling_vol.c +++ b/test/h5subfiling_vol.c @@ -1884,11 +1884,6 @@ H5VL_subfiling_file_create(const char *name, unsigned flags, hid_t fcpl_id, void *ret_value = NULL; void *under = NULL; -#if 0 - file_create_count++; - printf("%s: count=%d\n", __func__, file_create_count); - fflush(stdout); -#endif #ifdef ENABLE_EXT_PASSTHRU_LOGGING printf("------- SUBFILING VOL FILE Create\n"); #endif @@ -2008,11 +2003,6 @@ H5VL_subfiling_file_open(const char *name, unsigned flags, hid_t fapl_id, void *ret_value = NULL; void *under = NULL; -#if 0 - file_open_count++; - printf("%s: count=%d\n", __func__, file_open_count); - fflush(stdout); -#endif #ifdef ENABLE_EXT_PASSTHRU_LOGGING printf("------- SUBFILING VOL FILE Open\n"); #endif @@ -2328,11 +2318,7 @@ H5VL_subfiling_file_close(void *_file, hid_t dxpl_id, void **req) H5VL_subfiling_file_t *subfiling_file = (H5VL_subfiling_file_t *)o->obj.item.file; int mpi_enabled = 0; herr_t ret_value; -#if 0 - file_close_count++; - printf("%s: count=%d\n", __func__, file_close_count); - fflush(stdout); -#endif + if (file_create_count > 0) { if (file_create_count != file_close_count) puts("mismatched file_create_count and file_close_count"); -- cgit v0.12