[svn-r15015] Description:

    Detect chunks that are >4GB before dataset gets created and return error
to application.

    Tweak lots of internal variables that hold the chunk size/dimensions to
use an 'uint32_t', instead of a 'size_t', so that the integer size is constant.

    Correct a number of our tests which were creating datasets with chunks
that were >4GB and add some specific tests for >4GB chunk size detection.

    Minor whitespace & other code cleanups.

Tested on:
    Mac OS X/32 10.5.2 (amazon)
    Forthcoming testing on other platforms...

4 files changed, 150 insertions, 21 deletions

diff --git a/test/Makefile.am b/test/Makefile.am
index 9dbb0cf..a3f8660 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -104,7 +104,7 @@ flush2.chkexe_: flush1.chkexe_
 # the temporary file name in ways that the makefile is not aware of.
 CHECK_CLEANFILES+=cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offset.h5 \
             max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \
-            extend.h5 istore.h5 \
+            huge_chunks.h5 extend.h5 istore.h5 \
             tfile[1-4].h5 th5s[1-3].h5 lheap.h5 fheap.h5 ohdr.h5 stab.h5 \
             extern_[1-3].h5 extern_[1-4][ab].raw gheap[0-4].h5 dt_arith[1-2] \
             links.h5 links[0-6]*.h5 extlinks[0-15].h5 tmp \
diff --git a/test/dsets.c b/test/dsets.c
index e5fa5b5..8edc2d2 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -42,6 +42,7 @@ const char *FILENAME[] = {
     "simple",
     "set_local",
     "random_chunks",
+    "huge_chunks",
     NULL
 };
 #define FILENAME_BUF_SIZE       1024
@@ -145,6 +146,22 @@ const char *FILENAME[] = {
 /* Names for random chunks test */
 #define NPOINTS         50
 
+/* Parameters for huge chunks test */
+#define HUGE_DATASET            "Dataset"
+#define HUGE_DIM                ((hsize_t)16 * 1024 * 1024 * 1024)
+#define HUGE_CHUNK_DIM          ((hsize_t)2 * 1024 * 1024 * 1024)
+#define TOO_HUGE_CHUNK_DIM      ((hsize_t)4 * 1024 * 1024 * 1024)
+#define HUGE_DATASET2           "Dataset2"
+#define HUGE_DIM2_0             ((hsize_t)16 * 1024)
+#define HUGE_DIM2_1             ((hsize_t)16 * 1024)
+#define HUGE_DIM2_2             ((hsize_t)16 * 1024)
+#define HUGE_CHUNK_DIM2_0       ((hsize_t)2 * 1024)
+#define HUGE_CHUNK_DIM2_1       ((hsize_t)1024)
+#define HUGE_CHUNK_DIM2_2       ((hsize_t)1024)
+#define TOO_HUGE_CHUNK_DIM2_0   ((hsize_t)4 * 1024)
+#define TOO_HUGE_CHUNK_DIM2_1   ((hsize_t)1024)
+#define TOO_HUGE_CHUNK_DIM2_2   ((hsize_t)1024)
+
 /* Shared global arrays */
 #define DSET_DIM1       100
 #define DSET_DIM2       200
@@ -1941,8 +1958,6 @@ UNUSED
     data_corrupt[1] = 33;
     data_corrupt[2] = 27;
 
-    /* Temporarily disable this test because the changes in chunk caching conflicts with
-     * the way this test is conducted. -slu 2007/7/20 */
     if(H5Zregister (H5Z_CORRUPT) < 0) goto error;
     if(H5Pset_filter(dc, H5Z_FILTER_CORRUPT, 0, (size_t)3, data_corrupt) < 0) goto error;
     if(test_filter_internal(file,DSET_FLETCHER32_NAME_3,dc,DISABLE_FLETCHER32,DATA_CORRUPTED,&fletcher32_size) < 0) goto error;
@@ -6359,6 +6374,118 @@ test_deprec(hid_t file)
 
 
 /*-------------------------------------------------------------------------
+ * Function: test_huge_chunks
+ *
+ * Purpose: Tests that datasets with chunks >4GB can't be created.
+ *
+ * Return:      Success: 0
+ *              Failure: -1
+ *
+ * Programmer:  Quincey Koziol
+ *              Thursday, May  1, 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_huge_chunks(hid_t fapl)
+{
+    char        filename[FILENAME_BUF_SIZE];
+    hid_t       fid = -1;       /* File ID */
+    hid_t       dcpl = -1;      /* Dataset creation property list ID */
+    hid_t       sid = -1;       /* Dataspace ID */
+    hid_t       dsid = -1;      /* Dataset ID */
+    hsize_t     dim, chunk_dim; /* Dataset and chunk dimensions */
+    hsize_t     dim2[3], chunk_dim2[3];  /* Dataset and chunk dimensions */
+    herr_t      ret;            /* Generic return value */
+
+    TESTING("creating dataset with >4GB chunks");
+
+    h5_fixname(FILENAME[7], fapl, filename, sizeof filename);
+
+    /* Create file */
+    if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR
+
+    /* Create dataset creation property list */
+    if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR
+
+    /* Try to set too large of a chunk for 1-D dataset (# of elements) */
+    chunk_dim = TOO_HUGE_CHUNK_DIM;
+    H5E_BEGIN_TRY {
+        ret = H5Pset_chunk(dcpl, 1, &chunk_dim);
+    } H5E_END_TRY;
+    if(ret >= 0)
+	FAIL_PUTS_ERROR("    Set chunk size with too large of chunk dimensions.")
+
+    /* Try to set too large of a chunk for n-D dataset (# of elements) */
+    chunk_dim2[0] = TOO_HUGE_CHUNK_DIM2_0;
+    chunk_dim2[1] = TOO_HUGE_CHUNK_DIM2_1;
+    chunk_dim2[2] = TOO_HUGE_CHUNK_DIM2_2;
+    H5E_BEGIN_TRY {
+        ret = H5Pset_chunk(dcpl, 3, chunk_dim2);
+    } H5E_END_TRY;
+    if(ret >= 0)
+	FAIL_PUTS_ERROR("    Set chunk size with too large of chunk dimensions.")
+
+    /* Set 1-D chunk size */
+    chunk_dim = HUGE_CHUNK_DIM;
+    if(H5Pset_chunk(dcpl, 1, &chunk_dim) < 0) FAIL_STACK_ERROR
+
+    /* Create 1-D dataspace */
+    dim = HUGE_DIM;
+    if((sid = H5Screate_simple(1, &dim, NULL)) < 0) FAIL_STACK_ERROR
+
+    /* Try to create dataset */
+    H5E_BEGIN_TRY {
+        dsid = H5Dcreate2(fid, HUGE_DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT);
+    } H5E_END_TRY;
+    if(dsid >= 0)
+	FAIL_PUTS_ERROR("    1-D Dataset with too large of chunk dimensions created.")
+
+    /* Close 1-D dataspace */
+    if(H5Sclose(sid) < 0) FAIL_STACK_ERROR
+
+
+    /* Set n-D chunk size */
+    chunk_dim2[0] = HUGE_CHUNK_DIM2_0;
+    chunk_dim2[1] = HUGE_CHUNK_DIM2_1;
+    chunk_dim2[2] = HUGE_CHUNK_DIM2_2;
+    if(H5Pset_chunk(dcpl, 3, chunk_dim2) < 0) FAIL_STACK_ERROR
+
+    /* Create n-D dataspace */
+    dim2[0] = HUGE_DIM2_0;
+    dim2[1] = HUGE_DIM2_1;
+    dim2[2] = HUGE_DIM2_2;
+    if((sid = H5Screate_simple(3, dim2, NULL)) < 0) FAIL_STACK_ERROR
+
+    /* Try to create dataset */
+    H5E_BEGIN_TRY {
+        dsid = H5Dcreate2(fid, HUGE_DATASET2, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT);
+    } H5E_END_TRY;
+    if(dsid >= 0)
+	FAIL_PUTS_ERROR("    n-D Dataset with too large of chunk dimensions created.")
+
+    /* Close n-D dataspace */
+    if(H5Sclose(sid) < 0) FAIL_STACK_ERROR
+
+    /* Close everything else */
+    if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR
+    if(H5Fclose(fid) < 0) FAIL_STACK_ERROR
+
+    PASSED();
+    return 0;
+
+error:
+    H5E_BEGIN_TRY {
+        H5Pclose(dcpl);
+        H5Dclose(dsid);
+        H5Sclose(sid);
+        H5Fclose(fid);
+    } H5E_END_TRY;
+    return -1;
+} /* end test_huge_chunks() */
+
+
+/*-------------------------------------------------------------------------
  * Function:	main
  *
  * Purpose:	Tests the dataset interface (H5D)
@@ -6393,12 +6520,19 @@ main(void)
         size_t rdcc_nbytes;
         double rdcc_w0;
 
+	/* Set the random # seed */
+	HDsrandom((unsigned long)HDtime(NULL));
+
 	/* Testing setup */
 	h5_reset();
 	fapl = h5_fileaccess();
 
-	/* Set the random # seed */
-	HDsrandom((unsigned long)HDtime(NULL));
+        /* Turn off the chunk cache, so all the chunks are immediately written to disk */
+        if(H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0)
+            goto error;
+        rdcc_nbytes = 0;
+        if(H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0)
+            goto error;
 
         /* Copy the file access property list */
         if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR
@@ -6422,13 +6556,6 @@ main(void)
                 my_fapl = fapl;
             } /* end else */
 
-            /* Turn off the chunk cache, so all the chunks are immediately written to disk */
-            if(H5Pget_cache(my_fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0)
-                goto error;
-            rdcc_nbytes = 0;
-            if(H5Pset_cache(my_fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0)
-                goto error;
-
             /* Create the file for this test */
             if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0)
                 goto error;
@@ -6478,6 +6605,7 @@ main(void)
 #ifndef H5_NO_DEPRECATED_SYMBOLS
             nerrors += (test_deprec(file) < 0			? 1 : 0);
 #endif /* H5_NO_DEPRECATED_SYMBOLS */
+            nerrors += (test_huge_chunks(my_fapl) < 0		? 1 : 0);
             
             if(H5Fclose(file) < 0)
                 goto error;
diff --git a/test/istore.c b/test/istore.c
index dfe5c41..e0d2e82 100644
--- a/test/istore.c
+++ b/test/istore.c
@@ -171,19 +171,20 @@ test_create(hid_t f, const char *prefix)
 {
     hid_t       dataset;        /* Dataset ID */
     hsize_t     dims[H5O_LAYOUT_NDIMS+1]; /* Dimensions of dataset */
+    hsize_t     my_chunk_dims[H5O_LAYOUT_NDIMS+1]; /* Dimensions of chunks */
     char        name[256];      /* Dataset name */
     unsigned    u;              /* Local index variable */
 
     TESTING("istore create");
 
-    dims[0]=TEST_CHUNK_SIZE;
+    dims[0] = my_chunk_dims[0] = 1;
     for (u = 1; u <= H5S_MAX_RANK; u++) {
         /* Initialize the dimension size in this new dimension */
-        dims[u]=TEST_CHUNK_SIZE;
+        dims[u] = my_chunk_dims[u] = 2;
 
         /* Create chunked dataset of this dimensionality */
 	HDsnprintf(name, sizeof name, "%s_%02u", prefix, u);
-	if ((dataset=new_object(f, name, (int)u, dims, chunk_dims)) < 0)
+	if ((dataset=new_object(f, name, (int)u, dims, my_chunk_dims)) < 0)
 	    return FAIL;
 
         /* Close dataset created */
diff --git a/test/tsohm.c b/test/tsohm.c
index abdbe77..0fc94f2 100644
--- a/test/tsohm.c
+++ b/test/tsohm.c
@@ -121,9 +121,9 @@ const int ENUM_VAL[] = {
     20480,  10,
     -1001,    -10
 };
-#define SIZE2_RANK1 10
-#define SIZE2_RANK2 20
-#define SIZE2_DIMS {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
+#define SIZE2_RANK1 6
+#define SIZE2_RANK2 10
+#define SIZE2_DIMS {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
 
 #define LONG_STRING "00 index.  A long string used for testing. To create new strings, set the first two characters to be some ASCII number other than 00, such as 01."
 
@@ -1376,7 +1376,7 @@ size2_helper(hid_t fcpl_id, int test_file_closing, size2_helper_struct *ret_size
     /* Constants used in this function */
     const int rank1 = SIZE2_RANK1;
     const int rank2 = SIZE2_RANK2;
-    const hsize_t dims[20] = SIZE2_DIMS;
+    const hsize_t dims[SIZE2_RANK2] = SIZE2_DIMS;
     dtype1_struct fill1;
     char fill2[DTYPE2_SIZE];
 
@@ -1745,8 +1745,8 @@ static void size2_verify(void)
     char attr_correct_string[NAME_BUF_SIZE];
     char attr_name[NAME_BUF_SIZE];
     int ndims;
-    hsize_t dims[20];
-    hsize_t correct_dims[20] = SIZE2_DIMS;
+    hsize_t dims[SIZE2_RANK2];
+    hsize_t correct_dims[SIZE2_RANK2] = SIZE2_DIMS;
 
     file_id = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT);
     CHECK_I(file_id, "H5Fopen");