Make the initial bug fixes to allow >2GB writes with Independent IO

4 files changed, 47 insertions, 26 deletions

diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index 71e9fe1..458d190 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -22,16 +22,17 @@
 #include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */
 
 
-#include "H5private.h"		/* Generic Functions			*/
+#include "H5private.h"          /* Generic Functions                    */
 #include "H5CXprivate.h"        /* API Contexts                         */
-#include "H5Dprivate.h"		/* Dataset functions			*/
-#include "H5Eprivate.h"		/* Error handling		  	*/
-#include "H5Fprivate.h"		/* File access				*/
-#include "H5FDprivate.h"	/* File drivers				*/
-#include "H5FDmpi.h"            /* MPI-based file drivers		*/
-#include "H5Iprivate.h"		/* IDs			  		*/
-#include "H5MMprivate.h"	/* Memory management			*/
+#include "H5Dprivate.h"         /* Dataset functions                    */
+#include "H5Eprivate.h"         /* Error handling                       */
+#include "H5Fprivate.h"         /* File access                          */
+#include "H5FDprivate.h"        /* File drivers                         */
+#include "H5FDmpi.h"            /* MPI-based file drivers               */
+#include "H5Iprivate.h"         /* IDs                                  */
+#include "H5MMprivate.h"        /* Memory management                    */
 #include "H5Pprivate.h"         /* Property lists                       */
+#include "H5Sprivate.h"         /* Large MPIO data type creation        */
 
 #ifdef H5_HAVE_PARALLEL
 
@@ -1324,6 +1325,7 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__mpio_read() */
 
+
 
 /*-------------------------------------------------------------------------
  * Function:    H5FD__mpio_write
@@ -1366,6 +1368,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
 #endif
     int                         size_i;
     hbool_t                     use_view_this_time = FALSE;
+    hbool_t                     derived_type = FALSE;
     H5FD_mpio_xfer_t            xfer_mode;   /* I/O transfer mode */
     herr_t                      ret_value = SUCCEED;
 
@@ -1391,8 +1394,6 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
     if(H5FD_mpi_haddr_to_MPIOff(addr, &mpi_off) < 0)
         HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off")
     size_i = (int)size;
-    if((hsize_t)size_i != size)
-        HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from size to size_i")
 
 #ifdef H5FDmpio_DEBUG
     if(H5FD_mpio_Debug[(int)'w'])
@@ -1430,6 +1431,20 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
          */
         mpi_off = 0;
     } /* end if */
+    else if(size != (hsize_t)size_i) {
+        /* If HERE, then we need to work around the integer size limit
+         * of 2GB. The input size_t size variable cannot fit into an integer,
+	 * but we can get around that limitation by creating a different datatype
+	 * and then setting the integer size (or element count) to 1 when using
+	 * the derived_type.
+	 */
+
+        if (H5S_mpio_create_large_type(size, 0, MPI_BYTE, &buf_type) < 0)
+            HGOTO_ERROR(H5E_INTERNAL, H5E_CANTGET, FAIL, "can't create MPI-I/O datatype")
+
+        derived_type = TRUE;
+        size_i = 1;
+    }
 
     /* Write the data. */
     if(use_view_this_time) {
@@ -1506,6 +1521,9 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id,
         file->local_eof = addr + (haddr_t)bytes_written;
 
 done:
+    if(derived_type) {
+        MPI_Type_free(&buf_type);
+    }
 #ifdef H5FDmpio_DEBUG
     if(H5FD_mpio_Debug[(int)'t'])
         HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value );
diff --git a/src/H5Smpio.c b/src/H5Smpio.c
index aeec566..287bde1 100644
--- a/src/H5Smpio.c
+++ b/src/H5Smpio.c
@@ -34,6 +34,7 @@
 #include "H5FLprivate.h"	/* Free Lists				*/
 #include "H5MMprivate.h"        /* Memory management                    */
 #include "H5Spkg.h"		/* Dataspaces 				*/
+#include "H5Sprivate.h"		/* H5S prototypes, etc.                 */
 #include "H5VMprivate.h"        /* Vector and array functions		*/
 
 #ifdef H5_HAVE_PARALLEL
@@ -44,7 +45,7 @@
 #define H5S_MPIO_INITIAL_ALLOC_COUNT    256
 #define TWO_GIG_LIMIT                   2147483648
 #ifndef H5S_MAX_MPI_COUNT
-#define H5S_MAX_MPI_COUNT               536870911  /* (2^29)-1  */
+#define H5S_MAX_MPI_COUNT               1073741824
 #endif
 
 
@@ -88,8 +89,6 @@ static herr_t H5S__release_datatype(H5S_mpio_mpitype_list_t *type_list);
 static herr_t H5S__obtain_datatype(H5S_hyper_span_info_t *spans, const hsize_t *down,
     size_t elmt_size, const MPI_Datatype *elmt_type, MPI_Datatype *span_type,
     H5S_mpio_mpitype_list_t *type_list, uint64_t op_gen);
-static herr_t H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
-    MPI_Datatype old_type, MPI_Datatype *new_type);
 
 
 /*****************************/
@@ -183,7 +182,7 @@ H5S__mpio_all_type(const H5S_t *space, size_t elmt_size,
     } /* end if */
     else {
         /* Create a LARGE derived datatype for this transfer */
-        if(H5S__mpio_create_large_type(total_bytes, 0, MPI_BYTE, new_type) < 0)
+        if(H5S_mpio_create_large_type(total_bytes, 0, MPI_BYTE, new_type) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large datatype from the all selection")
         *count = 1;
         *is_derived_type = TRUE;
@@ -824,7 +823,7 @@ if(H5DEBUG(S)) {
     } /* end if */
     else
         /* Create the compound datatype for this operation (> 2GB) */
-        if(H5S__mpio_create_large_type(elmt_size, 0, MPI_BYTE, &inner_type) < 0)
+        if(H5S_mpio_create_large_type(elmt_size, 0, MPI_BYTE, &inner_type) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large inner datatype in hyper selection")
 
 /*******************************************************
@@ -878,7 +877,7 @@ if(H5DEBUG(S))
              * Again we need to check that the number of BLOCKS can fit into
              * a 32 bit integer */
             if(bigio_count < d[i].block) {
-                if(H5S__mpio_create_large_type(d[i].block, 0, inner_type, &block_type) < 0)
+                if(H5S_mpio_create_large_type(d[i].block, 0, inner_type, &block_type) < 0)
                     HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large block datatype in hyper selection")
             } /* end if */
             else
@@ -899,7 +898,7 @@ if(H5DEBUG(S))
              * we call the large type creation function to handle that
              */
             if(bigio_count < d[i].count) {
-                if(H5S__mpio_create_large_type(d[i].count, stride_in_bytes, block_type, &outer_type) < 0)
+                if(H5S_mpio_create_large_type(d[i].count, stride_in_bytes, block_type, &outer_type) < 0)
                     HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large outer datatype in hyper selection")
             } /* end if */
             /* otherwise a regular create_hvector will do */
@@ -1020,7 +1019,7 @@ H5S__mpio_span_hyper_type(const H5S_t *space, size_t elmt_size,
             HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code)
     } /* end if */
     else
-        if(H5S__mpio_create_large_type(elmt_size, 0, MPI_BYTE, &elmt_type) < 0)
+        if(H5S_mpio_create_large_type(elmt_size, 0, MPI_BYTE, &elmt_type) < 0)
             HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large element datatype in span_hyper selection")
     elmt_type_is_derived = TRUE;
 
@@ -1203,7 +1202,7 @@ H5S__obtain_datatype(H5S_hyper_span_info_t *spans, const hsize_t *down,
 
                     /* create the block type from elmt_type while checking the 32 bit int limit */
                     if(blocklen[u] > bigio_count) {
-                        if(H5S__mpio_create_large_type(blocklen[u], 0, *elmt_type, &temp_type) < 0)
+                        if(H5S_mpio_create_large_type(blocklen[u], 0, *elmt_type, &temp_type) < 0)
                             HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "couldn't create a large element datatype in span_hyper selection")
                     } /* end if */
                     else
@@ -1455,7 +1454,7 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:    H5S__mpio_create_large_type
+ * Function:    H5S_mpio_create_large_type
  *
  * Purpose:     Create a large datatype of size larger than what a 32 bit integer
  *              can hold.
@@ -1468,8 +1467,8 @@ done:
  *
  *-------------------------------------------------------------------------
  */
-static herr_t
-H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
+herr_t
+H5S_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
     MPI_Datatype old_type, MPI_Datatype *new_type)
 {
     int           num_big_types;          /* num times the 2G datatype will be repeated */
@@ -1481,7 +1480,7 @@ H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
     MPI_Aint      disp[2], old_extent;
     herr_t        ret_value = SUCCEED;    /* Return value */
 
-    FUNC_ENTER_STATIC
+    FUNC_ENTER_NOAPI_NOINIT
 
     /* Calculate how many Big MPI datatypes are needed to represent the buffer */
     num_big_types = (int)(num_elements/bigio_count);
@@ -1559,7 +1558,7 @@ H5S__mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes,
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5S__mpio_create_large_type() */
+} /* end H5S_mpio_create_large_type() */
 
 #endif  /* H5_HAVE_PARALLEL */
 
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index 0a9d2e7..831c454 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -308,6 +308,11 @@ H5_DLL herr_t H5S_sel_iter_close(H5S_sel_iter_t *sel_iter);
 
 #ifdef H5_HAVE_PARALLEL
 H5_DLL hsize_t H5S_mpio_set_bigio_count(hsize_t new_count);
+H5_DLL herr_t H5S_mpio_create_large_type(hsize_t num_elements,
+		MPI_Aint stride_bytes,
+		MPI_Datatype old_type,
+		MPI_Datatype *new_type);
+
 H5_DLL herr_t H5S_mpio_space_type(const H5S_t *space, size_t elmt_size,
     /* out: */  MPI_Datatype *new_type,
                 int *count,
diff --git a/testpar/t_file.c b/testpar/t_file.c
index 204095b..e5a4f8a 100644
--- a/testpar/t_file.c
+++ b/testpar/t_file.c
@@ -145,7 +145,7 @@ test_page_buffer_access(void)
 
     ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0);
     VRFY((ret == 0), "");
-    ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*100);
+    ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*128);
     VRFY((ret == 0), "");
     ret = H5Pset_page_buffer_size(fapl, sizeof(int)*100000, 0, 0);
     VRFY((ret == 0), "");
@@ -180,7 +180,6 @@ test_page_buffer_access(void)
         data[i] = -1;
     if(MAINPROCESS) {
         hid_t fapl_self = H5I_INVALID_HID;
-
         fapl_self = create_faccess_plist(MPI_COMM_SELF, MPI_INFO_NULL, facc_type);
 
         ret = H5Pset_page_buffer_size(fapl_self, sizeof(int)*1000, 0, 0);