[svn-r8590] Purpose:

    Code optimization & bug fix

Description:
    When dimension information is being stored in the storage layout message
on disk, it is stored as 32-bit quantities, possibly truncating the dimension
information, if a dimension is greater than 32-bits in size.

Solution:
    Fix the storage layout message problem by revising file format to not store
dimension information, since it is already available in the dataspace.

    Also revise the storage layout data structures to be more compartmentalized
for the information for contiguous, chunked and compact storage.

Platforms tested:
    FreeBSD 4.9 (sleipnir) w/parallel
    Solaris 2.7 (arabica)
    h5committest

4 files changed, 286 insertions, 0 deletions

diff --git a/test/dsets.c b/test/dsets.c
index deaba61..79ff80b 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -578,6 +578,8 @@ test_compact_io(hid_t fapl)
                  H5_FAILED();
                  printf("    Read different values than written.\n");
                  printf("    At index %d,%d\n", i, j);
+                 printf("    wbuf[%d][%d]=%d\n", i, j, wbuf[i][j]);
+                 printf("    rbuf[%d][%d]=%d\n", i, j, rbuf[i][j]);
                  goto error;
              }
          }
diff --git a/test/gen_old_layout.c b/test/gen_old_layout.c
new file mode 100644
index 0000000..473b3d3
--- /dev/null
+++ b/test/gen_old_layout.c
@@ -0,0 +1,95 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by the Board of Trustees of the University of Illinois.         *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the files COPYING and Copyright.html.  COPYING can be found at the root   *
+ * of the source code distribution tree; Copyright.html can be found at the  *
+ * root level of an installed copy of the electronic HDF5 document set and   *
+ * is linked from the top-level documents page.  It can also be found at     *
+ * http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html.  If you do not have     *
+ * access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer:  Quincey Koziol <koziol@ncsa.uiuc.edu>
+ *              Thursday, May 27, 2004
+ *
+ * Purpose:	Create two datasets (one for version 1 and one for version 2 of
+ *      the layout message), which should have dimensions too large to
+ *      represent in version 1 & 2 of the storage layout message.
+ *		This program is used to create the test file `tlayouto.h5' which
+ *      has truncated dimension information and can be used to verify that the
+ *      library has fixed up the storage size correctly.
+ *		To build the test file, this program MUST be compiled and linked
+ *      with version hdf5-1.6.2 or _earlier_ libraries and the generated test
+ *      file must be put into the 'test' directory in the 1.7+ (or 1.6+) branch
+ *      of the library.
+ */
+
+#include "hdf5.h"
+
+#define TESTFILE   "tlayouto.h5"
+#define SPACE_RANK       2
+#define SPACE_DIM0       (8*1024*1024*1024ULL)
+#define SPACE_DIM1       ((4*1024*1024*1024ULL)+1ULL)
+
+
+/*-------------------------------------------------------------------------
+ * Function:	main
+ *
+ * Purpose:	
+ *
+ * Return:	Success:	
+ *
+ *		Failure:	
+ *
+ * Programmer:	Quincey Koziol
+ *              Friday, January  3, 2003
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+    hid_t	file, space, dset, dcpl;
+    herr_t      ret;
+    unsigned rank=SPACE_RANK;    /* Rank of dataspace */
+    hsize_t big_dims[SPACE_RANK]={SPACE_DIM0,SPACE_DIM1};      /* Large dimensions */
+
+    /* Create the file */
+    file = H5Fcreate(TESTFILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    if(file<0)
+        printf("file<0!\n");
+
+    /* Create the dataspace (for dataset) */
+    space = H5Screate_simple(rank,big_dims,NULL);
+    if(space<0)
+        printf("space<0!\n");
+
+    /* Create a dataset creation property list */
+    dcpl = H5Pcreate(H5P_DATASET_CREATE);
+    if(dcpl<0)
+        printf("dcpl<0!\n");
+
+    /* Make certain that the dataset's storage doesn't get allocated :-) */
+    ret = H5Pset_alloc_time(dcpl,H5D_ALLOC_TIME_LATE);
+    if(ret<0)
+        printf("H5Pset_alloc_time() failed!\n");
+
+    /* Create the dataset with deferred storage allocation */
+    dset = H5Dcreate(file, "Dataset", H5T_NATIVE_INT, space, dcpl);
+    if(dset<0)
+        printf("dset<0!\n");
+
+    H5Dclose(dset);
+    H5Sclose(space);
+    H5Pclose(dcpl);
+    H5Fclose(file);
+
+    return 0;
+}
+
diff --git a/test/tlayouto.h5 b/test/tlayouto.h5
new file mode 100644
index 0000000..5dba556
--- /dev/null
+++ b/test/tlayouto.h5
diff --git a/test/tmisc.c b/test/tmisc.c
index 87534fa..b1d1691 100644
--- a/test/tmisc.c
+++ b/test/tmisc.c
@@ -22,8 +22,14 @@
 *
 *************************************************************/
 
+#define H5D_PACKAGE		/*suppress error about including H5Dpkg	  */
+
+/* Define this macro to indicate that the testing APIs should be available */
+#define H5D_TESTING
+
 #include "hdf5.h"
 #include "testhdf5.h"
+#include "H5Dpkg.h"		/* Datasets 				*/
 
 /* Definitions for misc. test #1 */
 #define MISC1_FILE	"tmisc1.h5"
@@ -217,6 +223,17 @@ unsigned m13_rdata[MISC13_DIM1][MISC13_DIM2];          /* Data read from dataset
 #define MISC19_ATTR_NAME        "Attribute"
 #define MISC19_GROUP_NAME       "Group"
 
+/* Definitions for misc. test #20 */
+#define MISC20_FILE             "tmisc20.h5"
+#define MISC20_FILE_OLD         "tlayouto.h5"
+#define MISC20_DSET_NAME        "Dataset"
+#define MISC20_DSET2_NAME       "Dataset2"
+#define MISC20_SPACE_RANK       2
+#define MISC20_SPACE_DIM0       (8*1024*1024*1024ULL)
+#define MISC20_SPACE_DIM1       ((4*1024*1024*1024ULL)+1ULL)
+#define MISC20_SPACE2_DIM0      8
+#define MISC20_SPACE2_DIM1      4
+
 /****************************************************************
 **
 **  test_misc1(): test unlinking a dataset from a group and immediately
@@ -3288,6 +3305,176 @@ test_misc19(void)
     
 /****************************************************************
 **
+**  test_misc20(): Test problems with version 2 of storage layout
+**                      message truncating dimensions
+**
+****************************************************************/
+static void
+test_misc20(void)
+{
+    hid_t fid;          /* File ID */
+    hid_t sid;          /* 'Space ID */
+    hid_t did;          /* Dataset ID */
+    hid_t dcpl;         /* Dataset creation property list ID */
+    unsigned rank=MISC20_SPACE_RANK;    /* Rank of dataspace */
+    hsize_t big_dims[MISC20_SPACE_RANK]={MISC20_SPACE_DIM0,MISC20_SPACE_DIM1};      /* Large dimensions */
+    hsize_t small_dims[MISC20_SPACE_RANK]={MISC20_SPACE2_DIM0,MISC20_SPACE2_DIM1};      /* Small dimensions */
+    unsigned version;   /* Version of storage layout info */
+    hsize_t contig_size;        /* Size of contiguous storage size from layout into */
+    char testfile[512]="";          /* Character buffer for corrected test file name */
+    char *srcdir = HDgetenv("srcdir");    /* Pointer to the directory the source code is located within */
+    herr_t ret;         /* Generic return value */
+
+    /* Output message about test being performed */
+    MESSAGE(5, ("Testing large dimension truncation fix\n"));
+
+    /* Verify that chunks with dimensions that are too large get rejected */
+
+    /* Create a dataset creation property list */
+    dcpl = H5Pcreate(H5P_DATASET_CREATE);
+    CHECK(dcpl, FAIL, "H5Pcreate"); 
+
+    /* Use chunked storage for this dataset */
+    ret = H5Pset_chunk(dcpl,rank,big_dims);
+    VERIFY(ret, FAIL, "H5Pset_chunk"); 
+
+    /* Verify that the storage for the dataset is the correct size and hasn't
+     * been truncated.
+     */
+
+    /* Create the file */
+    fid = H5Fcreate(MISC20_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    CHECK(fid, FAIL, "H5Fcreate");
+
+    /* Create dataspace with _really_ big dimensions */
+    sid = H5Screate_simple(rank,big_dims,NULL);
+    CHECK(sid, FAIL, "H5Screate_simple");
+
+    /* Make certain that the dataset's storage doesn't get allocated :-) */
+    ret = H5Pset_alloc_time(dcpl,H5D_ALLOC_TIME_LATE);
+    CHECK(ret, FAIL, "H5Pset_alloc_time"); 
+
+    /* Create dataset with big dataspace */
+    did = H5Dcreate(fid, MISC20_DSET_NAME, H5T_NATIVE_INT, sid, dcpl);
+    CHECK(did, FAIL, "H5Dcreate");
+
+    /* Close datasset */
+    ret=H5Dclose(did);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Close dataspace */
+    ret=H5Sclose(sid);
+    CHECK(ret, FAIL, "H5Sclose");
+
+    /* Create dataspace with small dimensions */
+    sid = H5Screate_simple(rank,small_dims,NULL);
+    CHECK(sid, FAIL, "H5Screate_simple");
+
+    /* Create dataset with big dataspace */
+    did = H5Dcreate(fid, MISC20_DSET2_NAME, H5T_NATIVE_INT, sid, dcpl);
+    CHECK(did, FAIL, "H5Dcreate");
+
+    /* Close datasset */
+    ret=H5Dclose(did);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Close dataspace */
+    ret=H5Sclose(sid);
+    CHECK(ret, FAIL, "H5Sclose");
+
+    /* Close dataset creation property list */
+    ret=H5Pclose(dcpl);
+    CHECK(ret, FAIL, "H5Pclose");
+
+    /* Close file */
+    ret = H5Fclose(fid);
+    CHECK(ret, FAIL, "H5Fclose");
+
+    /* Re-open the file */
+    fid = H5Fopen(MISC20_FILE, H5F_ACC_RDONLY, H5P_DEFAULT);
+    CHECK(fid, FAIL, "H5Fopen");
+
+    /* Open dataset with big dimensions */
+    did = H5Dopen(fid, MISC20_DSET_NAME);
+    CHECK(did, FAIL, "H5Dopen");
+
+    /* Get the layout version */
+    ret = H5D_layout_version_test(did,&version);
+    CHECK(ret, FAIL, "H5D_layout_version_test");
+    VERIFY(version,3,"H5D_layout_version_test");
+
+    /* Get the layout contiguous storage size */
+    ret = H5D_layout_contig_size_test(did,&contig_size);
+    CHECK(ret, FAIL, "H5D_layout_contig_size_test");
+    VERIFY(contig_size, MISC20_SPACE_DIM0*MISC20_SPACE_DIM1*H5Tget_size(H5T_NATIVE_INT), "H5D_layout_contig_size_test");
+
+    /* Close datasset */
+    ret=H5Dclose(did);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Open dataset with small dimensions */
+    did = H5Dopen(fid, MISC20_DSET2_NAME);
+    CHECK(did, FAIL, "H5Dopen");
+
+    /* Get the layout version */
+    ret = H5D_layout_version_test(did,&version);
+    CHECK(ret, FAIL, "H5D_layout_version_test");
+    VERIFY(version,3,"H5D_layout_version_test");
+
+    /* Get the layout contiguous storage size */
+    ret = H5D_layout_contig_size_test(did,&contig_size);
+    CHECK(ret, FAIL, "H5D_layout_contig_size_test");
+    VERIFY(contig_size, MISC20_SPACE2_DIM0*MISC20_SPACE2_DIM1*H5Tget_size(H5T_NATIVE_INT), "H5D_layout_contig_size_test");
+
+    /* Close datasset */
+    ret=H5Dclose(did);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Close file */
+    ret = H5Fclose(fid);
+    CHECK(ret, FAIL, "H5Fclose");
+
+    /* Verify that the storage size is computed correctly for older versions of layout info */
+
+    /* Generate the correct name for the test file, by prepending the source path */
+    if (srcdir && ((HDstrlen(srcdir) + HDstrlen(MISC20_FILE_OLD) + 1) < sizeof(testfile))) {
+        HDstrcpy(testfile, srcdir);
+        HDstrcat(testfile, "/");
+    }
+    HDstrcat(testfile, MISC20_FILE_OLD);
+
+    /*
+     * Open the old file and the dataset and get old settings.
+     */
+    fid =    H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT);
+    CHECK(fid, FAIL, "H5Fopen");
+
+    /* Open dataset with small dimensions */
+    did = H5Dopen(fid, MISC20_DSET_NAME);
+    CHECK(did, FAIL, "H5Dopen");
+
+    /* Get the layout version */
+    ret = H5D_layout_version_test(did,&version);
+    CHECK(ret, FAIL, "H5D_layout_version_test");
+    VERIFY(version,2,"H5D_layout_version_test");
+
+    /* Get the layout contiguous storage size */
+    ret = H5D_layout_contig_size_test(did,&contig_size);
+    CHECK(ret, FAIL, "H5D_layout_contig_size_test");
+    VERIFY(contig_size, MISC20_SPACE_DIM0*MISC20_SPACE_DIM1*H5Tget_size(H5T_NATIVE_INT), "H5D_layout_contig_size_test");
+
+    /* Close datasset */
+    ret=H5Dclose(did);
+    CHECK(ret, FAIL, "H5Dclose");
+
+    /* Close file */
+    ret = H5Fclose(fid);
+    CHECK(ret, FAIL, "H5Fclose");
+
+} /* end test_misc20() */
+    
+/****************************************************************
+**
 **  test_misc(): Main misc. test routine.
 ** 
 ****************************************************************/
@@ -3316,6 +3503,7 @@ test_misc(void)
     test_misc17();      /* Test array of ASCII character */
     test_misc18();      /* Test new object header information in H5G_stat_t struct */
     test_misc19();      /* Test incrementing & decrementing ref count on IDs */
+    test_misc20();      /* Test problems with truncated dimensions in version 2 of storage layout message */
 
 } /* test_misc() */
 
@@ -3359,4 +3547,5 @@ cleanup_misc(void)
     HDremove(MISC17_FILE);
     HDremove(MISC18_FILE);
     HDremove(MISC19_FILE);
+    HDremove(MISC20_FILE);
 }