Various edits to remove #if 0 blocks in an effort to make the code a bit cleaner

4 files changed, 40 insertions, 114 deletions

diff --git a/SUBFILING_README.txt b/SUBFILING_README.txt
index 71dd9e4..b09c592 100644
--- a/SUBFILING_README.txt
+++ b/SUBFILING_README.txt
@@ -42,10 +42,40 @@ that it supports Multithreading, but only MPI_THREAD_FUNNELED or potentially
 MPI_THREAD_SERIALIZED. The initial benchmarking efforts utilize OpenMPI to
 provide the necessary thread safe functionality.
 
+At login, the modules preloaded by default are:
+
+rawarren@cori06:~> module list
+Currently Loaded Modulefiles:
+  1) modules/3.2.11.4                                 12) gni-headers/5.0.12.0-7.0.1.1_6.31__g3b1768f.ari
+  2) altd/2.0                                         13) xpmem/2.2.20-7.0.1.1_4.13__g0475745.ari
+  3) darshan/3.1.7                                    14) job/2.2.4-7.0.1.1_3.40__g36b56f4.ari
+  4) intel/19.0.3.199                                 15) dvs/2.12_2.2.157-7.0.1.1_9.4__g083131db
+  5) craype-network-aries                             16) alps/6.6.58-7.0.1.1_6.8__g437d88db.ari
+  6) craype/2.6.2                                     17) rca/2.2.20-7.0.1.1_4.51__g8e3fb5b.ari
+  7) cray-libsci/19.06.1                              18) atp/2.1.3
+  8) udreg/2.3.2-7.0.1.1_3.36__g8175d3d.ari           19) PrgEnv-intel/6.0.5
+  9) ugni/6.0.14.0-7.0.1.1_7.38__ge78e5b0.ari         20) craype-haswell
+ 10) pmi/5.0.14                                       21) cray-mpich/7.7.10
+ 11) dmapp/7.1.1-7.0.1.1_4.52__g38cf134.ari           22) craype-hugepages2M
+rawarren@cori06:~> 
+
 Interestingly, the default C compiler (or at least a compiler wrapper which
 calls the actual C compiler) appears to know about the default MPI and thus
 avoids the necessity of users providing an include path (-I<path>) or a library
 link path (-L<path>) when creating executables or shared objects (libraries).
 As a consequence, we unload these specific modules prior to selecting
 alternatives for compilation and for the MPI library implementation.
+These defaults need to overridden, e.g.
+
+MODULE UNLOADS::
+rawarren@cori06:~> module unload cray-mpich/7.7.10
+rawarren@cori06:~> module unload craype/2.6.2
+
+MODULE LOADS::
+rawarren@cori06:~> module load gcc/9.3.0
+rawarren@cori06:~> module load openmpi/4.0.2
+
+
+
+
 
diff --git a/src/H5FDsubfile_mpi.c b/src/H5FDsubfile_mpi.c
index 57add71..5bffdad 100644
--- a/src/H5FDsubfile_mpi.c
+++ b/src/H5FDsubfile_mpi.c
@@ -1192,6 +1192,7 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
     int64_t      ioc_read_datasize[n_io_concentrators];
     int64_t      ioc_read_offset[n_io_concentrators];
     MPI_Datatype ioc_read_type[n_io_concentrators];
+	useconds_t   delay = 50;
 
     subfiling_context_t *sf_context = get_subfiling_object(context_id);
     assert(sf_context != NULL);
@@ -1229,22 +1230,12 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
 
 #ifndef NDEBUG
         if (sf_verbose_flag) {
-#if 0
-            if (sf_logfile) {
-                fprintf(sf_logfile,
-                    "[%d %s] Requesting %ld read bytes from IOC(%d): "
-                    "sourceOffset=%ld subfile_offset=%ld\n",
-                    sf_world_rank, __func__, msg[0], io_concentrator[ioc],
-						sourceOffset, msg[1]);
-            }
-#else
 			fprintf(stdout,
                     "[%d %s] Requesting %ld read bytes from IOC(%d): "
                     "sourceOffset=%ld subfile_offset=%ld\n",
                     sf_world_rank, __func__, msg[0], io_concentrator[ioc],
 					sourceOffset, msg[1]);
 			fflush(stdout);
-#endif
         }
 #endif
 
@@ -1299,22 +1290,12 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
         for (i = 0; i < ready; i++) {
 #ifndef NDEBUG
             if (sf_verbose_flag) {
-#if 0
-                if (sf_logfile) {
-                    fprintf(sf_logfile,
-                        "[%d] READ bytes(%ld) of data from ioc_concentrator %d "
-                        "complete\n",
-                        sf_world_rank, ioc_read_datasize[indices[i]],
-                        indices[i]);
-                }
-#else
 				fprintf(stdout,
                         "[%d] READ bytes(%ld) of data from ioc_concentrator %d "
                         "complete\n",
                         sf_world_rank, ioc_read_datasize[indices[i]],
                         indices[i]);
 				fflush(stdout);
-#endif
             }
 #endif
             if (ioc_read_type[indices[i]] != MPI_BYTE) {
@@ -1322,6 +1303,8 @@ read__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
             }
             n_waiting--;
         }
+		if (n_waiting)
+			usleep(delay);
     }
     return status;
 }
@@ -1455,11 +1438,12 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
     int64_t      ioc_write_datasize[n_io_concentrators];
     int64_t      ioc_write_offset[n_io_concentrators];
     MPI_Datatype ioc_write_type[n_io_concentrators];
-
+	useconds_t   delay = 50;
     subfiling_context_t *sf_context = get_subfiling_object(context_id);
     int                  i, target, ioc, n_waiting = 0, status = 0;
     int                  errors = 0;
 
+
     io_concentrator = sf_context->topology->io_concentrator;
 
     if (sf_context->topology->rank_is_ioc) {
@@ -1516,22 +1500,12 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
 #ifndef NDEBUG
 		if (sf_verbose_flag)
 		{
-#if 0
-			if (sf_logfile) {
-				fprintf(sf_logfile,
-						"[%d %s]: write_dest[ioc(%d), "
-						"sourceOffset=%ld, datasize=%ld, foffset=%ld]\n",
-						sf_world_rank, __func__, ioc, sourceOffset,
-						ioc_write_datasize[ioc], ioc_write_offset[ioc]);
-			}
-#else
 			fprintf(stdout,
 					"[%d %s]: write_dest[ioc(%d), "
 					"sourceOffset=%ld, datasize=%ld, foffset=%ld]\n",
 					sf_world_rank, __func__, ioc, sourceOffset,
 					ioc_write_datasize[ioc], ioc_write_offset[ioc]);
 			fflush(stdout);
-#endif
 		}
 #endif
 
@@ -1619,7 +1593,6 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
             fflush(stdout);
             errors++;
         }
-
         for (i = 0; i < ready; i++) {
             /* One of the Issend calls has completed
              * If we used a derived type to send data, then should free
@@ -1630,6 +1603,8 @@ write__independent(int n_io_concentrators, hid_t context_id, int64_t offset,
             }
             n_waiting--;
         }
+		if (n_waiting)
+			usleep(delay);
     }
     if (errors)
         return -1;
@@ -2041,9 +2016,6 @@ sf_open_subfiles(hid_t fid, char *filename, char *prefix, int flags)
         puts("H5FDsubfiling_init failed!");
         return -1;
     }
-#if 0
-	printf("[%d %s]\n", sf_world_rank, __func__);
-#endif
 
     sf_context = get_subfiling_object(context_id);
     assert(sf_context != NULL);
@@ -2194,11 +2166,6 @@ ioc_main(int64_t context_id)
 	sf_close_file_count = 0;  
 	sf_ops_after_first_close = 0;
 
-#if 0
-	printf("Starting IOC! mpi_rank=%d\n", sf_world_rank);
-	fflush(stdout);
-#endif
-
     while (!sf_shutdown_flag || sf_work_pending) {
         flag = 0;
         ret = MPI_Iprobe(
@@ -2220,29 +2187,6 @@ ioc_main(int64_t context_id)
                     MPI_BYTE, source, tag, context->sf_msg_comm, &msg_status);
             }
             if (ret == MPI_SUCCESS) {
-#if 0
-				if (tag == OPEN_OP) {
-					sf_open_file_count++;
-					printf("source=%d: sf_open_file_count = %d\n", source, sf_open_file_count);
-					fflush(stdout);
-				}
-				else if (tag == CLOSE_OP) {
-					sf_close_file_count++;
-					printf("source=%d: sf_close_file_count = %d\n", source, sf_close_file_count);
-					fflush(stdout);
-				}
-				else {
-					printf("ioc(0): tag=%d\n", tag);
-					fflush(stdout);
-					if (sf_close_file_count) {
-						sf_ops_after_first_close++;
-						if (sf_close_file_count == sf_world_size) {
-							printf("op=%d from source(%d) after file close! sf_open_file_count=%d\n", tag, source, sf_open_file_count);
-							fflush(stdout);
-						}
-					}
-				}
-#endif
                 if (msg) {
                     msg->source = source;
                     msg->subfile_rank = subfile_rank;
@@ -2620,14 +2564,9 @@ queue_file_open(
         }
     }
 #endif
-#if 0
-	printf("[ioc(%d) %s]\n", subfile_rank, __func__);
-	fflush(stdout);
-#endif
     errors = subfiling_open_file(msg, sf_subfile_prefix, subfile_rank, flags);
 	// open_count = atomic_load(&sf_file_refcount);
 
-#if 1
 	ret = MPI_Send(&errors, 1, MPI_INT, source, COMPLETED, comm);
 	if (ret != MPI_SUCCESS) {
 		printf("[ioc(%d)] MPI_Send FILE_OPEN, COMPLETED to source(%d) FAILED\n",
@@ -2635,21 +2574,7 @@ queue_file_open(
 		fflush(stdout);
 		errors++;
 	}
-#else
-	if (open_count == sf_world_size) {
-		int i, k = (sf_world_rank +1);
-		for (i=0; i < sf_world_size; i++, k++) {
-			source = k % sf_world_size;
-			ret = MPI_Send(&errors, 1, MPI_INT, source, COMPLETED, comm);
-			if (ret != MPI_SUCCESS) {
-				printf("[ioc(%d)] MPI_Send FILE_OPEN, COMPLETED to source(%d) FAILED\n",
-					   subfile_rank, source);
-				fflush(stdout);
-				errors++;
-			}
-		}	
-    }
-#endif
+
     if (errors) {
 #ifndef NDEBUG
         if (sf_verbose_flag) {
diff --git a/src/H5FDsubfiling.c b/src/H5FDsubfiling.c
index 7fbfdc7..99ea9ac 100644
--- a/src/H5FDsubfiling.c
+++ b/src/H5FDsubfiling.c
@@ -697,16 +697,8 @@ H5FD_subfiling_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t max
 	 * We can be a bit more efficient by having rank 0 broadcast
 	 * the stat buffer.
 	 */
-#if 0
-	if (mpi_enabled && (my_rank == 0)) {
-		int sb_size = sizeof(sb);
-
-		MPI_Bcast(&sb, sb_size, MPI_BYTE, 0, MPI_COMM_WORLD);
-	}				
-#else
 	if(HDfstat(fd, &sb) < 0)
 		HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file")
-#endif
 
     /* Create the new file struct */
     if(NULL == (file = H5FL_CALLOC(H5FD_subfiling_t)))
@@ -929,9 +921,6 @@ H5FD_subfiling_query(const H5FD_t *_file, unsigned long *flags /* out */)
         *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA;    /* OK to aggregate "small" raw data allocations                     */
         *flags |= H5FD_FEAT_POSIX_COMPAT_HANDLE;    /* get_handle callback returns a POSIX file descriptor              */
         *flags |= H5FD_FEAT_SUPPORTS_SWMR_IO;       /* VFD supports the single-writer/multiple-readers (SWMR) pattern   */
-#if 0
-		*flags |= H5FD_FEAT_HAS_MPI;				/* FIXME:: for experimentation only... */
-#endif
         /* Check for flags that are set by h5repart */
         if(file && file->fam_to_single)
             *flags |= H5FD_FEAT_IGNORE_DRVRINFO; /* Ignore the driver info when file is opened (which eliminates it) */
@@ -1755,13 +1744,9 @@ H5FD__dataset_write_contiguous(hid_t h5_file_id, haddr_t dataset_baseAddr, size_
 		{
 			int status;
 			haddr_t rank_baseAddr;
-#if 0
-			rank_baseAddr = get_data_offset(mpi_rank, mpi_size, dtype_extent, mem_space, file_space);
-
-#else		
 			rank_baseAddr = get_base_offset(mpi_rank, mpi_size, mem_space_id, file_space_id);
 			rank_baseAddr += dataset_baseAddr;
-#endif
+
 			// printf("[%d] H5S_SEL_HYPERSLABS, file_offset = %lld\n", mpi_rank, rank_baseAddr );
 			if ((status = H5Sis_regular_hyperslab(file_space_id)) < 0) {
 				puts("H5Sis_regular_hyperslab returned an error");
diff --git a/test/h5subfiling_vol.c b/test/h5subfiling_vol.c
index 25f1292..6db5aa2 100644
--- a/test/h5subfiling_vol.c
+++ b/test/h5subfiling_vol.c
@@ -1884,11 +1884,6 @@ H5VL_subfiling_file_create(const char *name, unsigned flags, hid_t fcpl_id,
 	void *ret_value = NULL;
     void *under = NULL;
 
-#if 0
-	file_create_count++;
-	printf("%s: count=%d\n", __func__, file_create_count);
-	fflush(stdout);
-#endif
 #ifdef ENABLE_EXT_PASSTHRU_LOGGING
     printf("------- SUBFILING VOL FILE Create\n");
 #endif
@@ -2008,11 +2003,6 @@ H5VL_subfiling_file_open(const char *name, unsigned flags, hid_t fapl_id,
 	void *ret_value = NULL;
     void *under = NULL;
 	
-#if 0
-	file_open_count++;
-	printf("%s: count=%d\n", __func__, file_open_count);
-	fflush(stdout);
-#endif	
 #ifdef ENABLE_EXT_PASSTHRU_LOGGING
     printf("------- SUBFILING VOL FILE Open\n");
 #endif
@@ -2328,11 +2318,7 @@ H5VL_subfiling_file_close(void *_file, hid_t dxpl_id, void **req)
     H5VL_subfiling_file_t *subfiling_file = (H5VL_subfiling_file_t *)o->obj.item.file;
     int mpi_enabled = 0;
     herr_t ret_value;
-#if 0
-	file_close_count++;
-	printf("%s: count=%d\n", __func__, file_close_count);
-	fflush(stdout);
-#endif
+
 	if (file_create_count > 0) {
 		if (file_create_count != file_close_count)
 			puts("mismatched file_create_count and file_close_count");