[svn-r16641] merge from trunk revs 16614, 16629

1.	#1501  (B1) tools bug if dataset is larger than H5TOOLS_BUFSIZE limit. 
ISSUE : the tools use the following formula to read by hyperslabs: hyperslab_size[i] = MIN( dim_size[i], H5TOOLS_BUFSIZE / datum_size) where H5TOOLS_BUFSIZE is a constant defined of 1024K. This is OK as long as the datum_size does not exceed 1024K, otherwise we have a hyperslab size of 0 (since 1024K/(greater than 1024K) = 0). This affects h5dump. h5repack, h5diff
SOLUTION: add a check for a 0 size and define as 1 if so. 
TEST FOR H5DUMP: Defined a case in the h5dump test generator program of such a type (an array type of doubles with a large array dimension, that was the case the user reported).  Since the written file commited in svn would be around 1024K, opted for not writing the data (the part of the code where the hyperslab is defined is executed, since h5dump always reads the files). Defined a macro WRITE_ARRAY to enable such writing if needed. Added a run on the h5dump shell script. Added 2 new files to svn: tools/testfiles/tarray8.ddl, tools/testfiles/tarray8.h5. NOTE: while doing this I thought of adding this dataset case to an existing file, but that would add the large array output to those files (the ddls). The issue is that the file list is increasing.
TEST FOR H5DIFF: for h5diff the check for reading by hyperslabs is H5TOOLS_MALLOCSIZE (128 * H5TOOLS_BUFSIZE) or 128 Mb. This makes it not possible to add such a file to svn, so used the same method as h5dump (only write the dataset if WRITE_ARRAY is defined). As opposed to h5dump, the hyperslab code is NOT executed when the dataset is empty (dataset is not read). Added the new dataset to existing files and shell run (tools/h5diff/testfiles/h5diff_dset1.h5 and tools/h5diff/testfiles/h5diff_dset2.h5 and output in tools/h5diff/testfiles/h5diff_80.txt).
TEST FOR H5REPACK: similar issue as h5diff with the difference that the hyperslab code is run. Added a run to the shell script (with a filter, otherwise the code uses H5Ocopy). 
FURTHER ISSUES: the type in question ("double") has a different output cross platforms (e.g on liberty some garbage number is printed at some array locations)
SOLUTION: defined an "int" type for this test. However the printing of such an array has a bogus output at least in one platform (FreeBsd), so eliminated the test run altogether and filed a bug report on this

13 files changed, 182 insertions, 25 deletions

diff --git a/tools/h5diff/h5diffgentest.c b/tools/h5diff/h5diffgentest.c
index 181e242..92e5493 100644
--- a/tools/h5diff/h5diffgentest.c
+++ b/tools/h5diff/h5diffgentest.c
@@ -17,6 +17,8 @@
 #include <stdlib.h>
 #include "hdf5.h"
 #include "H5private.h"
+#include "h5tools.h"
+
 
 /*-------------------------------------------------------------------------
  * Program: h5diffgentest
@@ -153,12 +155,12 @@ int test_basic(const char *fname1, const char *fname2, const char *fname3)
         goto out;
     if (( fid2 = H5Fcreate (fname2, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0 )
         goto out;
-    
-        /*-------------------------------------------------------------------------
-        * create groups
-        *-------------------------------------------------------------------------
+
+    /*-------------------------------------------------------------------------
+    * create groups
+    *-------------------------------------------------------------------------
     */
-    
+
     gid1 = H5Gcreate2(fid1, "g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
     gid2 = H5Gcreate2(fid2, "g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
     gid3 = H5Gcreate2(fid2, "g2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
@@ -600,42 +602,42 @@ int test_datatypes(const char *fname)
     write_dset(fid1,2,dims,"dset0b",H5T_STD_I32LE,buf3b);
     
     /*-------------------------------------------------------------------------
-    * Check H5T_NATIVE_CHAR
+    * H5T_NATIVE_CHAR
     *-------------------------------------------------------------------------
     */
     write_dset(fid1,2,dims,"dset1a",H5T_NATIVE_CHAR,buf1a);
     write_dset(fid1,2,dims,"dset1b",H5T_NATIVE_CHAR,buf1b);
     
     /*-------------------------------------------------------------------------
-    * Check H5T_NATIVE_SHORT
+    * H5T_NATIVE_SHORT
     *-------------------------------------------------------------------------
     */
     write_dset(fid1,2,dims,"dset2a",H5T_NATIVE_SHORT,buf2a);
     write_dset(fid1,2,dims,"dset2b",H5T_NATIVE_SHORT,buf2b);
     
     /*-------------------------------------------------------------------------
-    * Check H5T_NATIVE_INT
+    * H5T_NATIVE_INT
     *-------------------------------------------------------------------------
     */
     write_dset(fid1,2,dims,"dset3a",H5T_NATIVE_INT,buf3a);
     write_dset(fid1,2,dims,"dset3b",H5T_NATIVE_INT,buf3b);
     
     /*-------------------------------------------------------------------------
-    * Check H5T_NATIVE_LONG
+    * H5T_NATIVE_LONG
     *-------------------------------------------------------------------------
     */
     write_dset(fid1,2,dims,"dset4a",H5T_NATIVE_LONG,buf4a);
     write_dset(fid1,2,dims,"dset4b",H5T_NATIVE_LONG,buf4b);
     
     /*-------------------------------------------------------------------------
-    * Check H5T_NATIVE_FLOAT
+    * H5T_NATIVE_FLOAT
     *-------------------------------------------------------------------------
     */
     write_dset(fid1,2,dims,"dset5a",H5T_NATIVE_FLOAT,buf5a);
     write_dset(fid1,2,dims,"dset5b",H5T_NATIVE_FLOAT,buf5b);
     
     /*-------------------------------------------------------------------------
-    * Check H5T_NATIVE_DOUBLE
+    * H5T_NATIVE_DOUBLE
     *-------------------------------------------------------------------------
     */
     
@@ -1909,7 +1911,7 @@ void write_dset_in(hid_t loc_id,
     typedef enum
     {
         RED,
-            GREEN
+        GREEN
     } e_t;
     
     hid_t   did;
@@ -2155,7 +2157,47 @@ void write_dset_in(hid_t loc_id,
     tid = H5Tarray_create2(H5T_NATIVE_INT, 1, dimarray);
     write_dset(loc_id, 1, dims, "array", tid, buf6);
     status = H5Tclose(tid);
-    
+
+    {
+
+        double  *dbuf;                           /* information to write */
+        hid_t   did;                             /* dataset ID   */
+        hid_t   sid;                             /* dataspace ID   */
+        hid_t   tid;                             /* datatype ID   */
+        size_t  size;
+        hsize_t sdims[] = {1};
+        hsize_t tdims[] = {H5TOOLS_MALLOCSIZE / sizeof(double) + 1};
+        int     j;
+
+        /* allocate and initialize array data to write */
+        size = ( H5TOOLS_MALLOCSIZE / sizeof(double) + 1 ) * sizeof(double);
+        dbuf = malloc( size );
+
+        for( j = 0; j < H5TOOLS_MALLOCSIZE / sizeof(double) + 1; j++)
+            dbuf[j] = j;
+
+        if (make_diffs)
+        {
+            dbuf[5] = 0;
+            dbuf[6] = 0;
+        }
+
+        /* create a type larger than H5TOOLS_MALLOCSIZE */
+        tid = H5Tarray_create2(H5T_NATIVE_DOUBLE, 1, tdims);
+        size = H5Tget_size(tid);
+        sid = H5Screate_simple(1, sdims, NULL);
+        did = H5Dcreate2(loc_id, "arrayd", tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+#if defined(WRITE_ARRAY)
+        H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dbuf);
+#endif
+
+        /* close */
+        H5Dclose(did);
+        H5Tclose(tid);
+        H5Sclose(sid);
+        free( dbuf );
+    }
+
     /*-------------------------------------------------------------------------
     * H5T_INTEGER and H5T_FLOAT
     *-------------------------------------------------------------------------
diff --git a/tools/h5diff/testfiles/h5diff_80.txt b/tools/h5diff/testfiles/h5diff_80.txt
index dcabf10..a52b24e 100644
--- a/tools/h5diff/testfiles/h5diff_80.txt
+++ b/tools/h5diff/testfiles/h5diff_80.txt
@@ -11,6 +11,7 @@ file1     file2
     x      x    /g1/array      
     x      x    /g1/array2D    
     x      x    /g1/array3D    
+    x      x    /g1/arrayd     
     x      x    /g1/bitfield   
     x      x    /g1/bitfield2D 
     x      x    /g1/bitfield3D 
@@ -243,6 +244,9 @@ position        array3D         array3D         difference
 [ 3 2 1 ]          71              0               71             
 [ 3 2 1 ]          72              0               72             
 72 differences found
+dataset: </g1/arrayd> and </g1/arrayd>
+</g1/arrayd> or </g1/arrayd> are empty datasets
+0 differences found
 dataset: </g1/bitfield> and </g1/bitfield>
 size:           [2]           [2]
 position        bitfield        bitfield        difference          
diff --git a/tools/h5diff/testfiles/h5diff_dset1.h5 b/tools/h5diff/testfiles/h5diff_dset1.h5
index e2699bc..240e55c 100644
--- a/tools/h5diff/testfiles/h5diff_dset1.h5
+++ b/tools/h5diff/testfiles/h5diff_dset1.h5
diff --git a/tools/h5diff/testfiles/h5diff_dset2.h5 b/tools/h5diff/testfiles/h5diff_dset2.h5
index 3687fc4..10d8599 100644
--- a/tools/h5diff/testfiles/h5diff_dset2.h5
+++ b/tools/h5diff/testfiles/h5diff_dset2.h5
diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c
index fc75681..b8a3753 100644
--- a/tools/h5dump/h5dumpgentest.c
+++ b/tools/h5dump/h5dumpgentest.c
@@ -91,6 +91,7 @@
 #define FILE61  "textlinksrc.h5"
 #define FILE62  "textlinktar.h5"
 #define FILE63  "textlinkfar.h5"
+#define FILE64  "tarray8.h5"
 
 
 
@@ -2812,6 +2813,50 @@ static void gent_array7(void)
     assert(ret >= 0);
 }
 
+static void gent_array8(void)
+{
+    int     *buf;                            /* information to write */
+    hid_t   fid;                             /* HDF5 File ID  */
+    hid_t   did;                             /* dataset ID   */
+    hid_t   sid;                             /* dataspace ID   */
+    hid_t   tid;                             /* datatype ID   */
+    size_t  size;
+    hsize_t sdims[] = {1};
+    hsize_t tdims[] = {H5TOOLS_BUFSIZE / sizeof(int) + 1};
+    int     i;                               
+    herr_t  ret;                             
+
+    size = ( H5TOOLS_BUFSIZE / sizeof(int) + 1 ) * sizeof(int);
+    buf = malloc( size );
+
+    for( i = 0; i < H5TOOLS_BUFSIZE / sizeof(int) + 1; i++)
+        buf[i] = i;
+
+    /* create file */
+    fid = H5Fcreate(FILE64, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+
+    /* create a type larger than H5TOOLS_BUFSIZE */
+    tid = H5Tarray_create2(H5T_NATIVE_INT, 1, tdims);
+    size = H5Tget_size(tid);
+    sid = H5Screate_simple(1, sdims, NULL);
+    did = H5Dcreate2(fid, "dset", tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+    ret = H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf);
+    assert(ret >= 0);
+
+
+    /* close */
+    ret = H5Dclose(did);
+    assert(ret >= 0);
+    ret = H5Tclose(tid);
+    assert(ret >= 0);
+    ret = H5Sclose(sid);
+    assert(ret >= 0);
+
+    ret = H5Fclose(fid);
+    assert(ret >= 0);
+    free( buf );
+}
+
 static void gent_empty(void)
 {
     typedef struct {
diff --git a/tools/h5dump/testh5dump.sh.in b/tools/h5dump/testh5dump.sh.in
index 990ef2a..748b165 100644
--- a/tools/h5dump/testh5dump.sh.in
+++ b/tools/h5dump/testh5dump.sh.in
@@ -325,8 +325,7 @@ TOOLTEST tall-2B.ddl -A -r tall.h5
 TOOLTEST tall-4s.ddl --dataset=/g1/g1.1/dset1.1.1 --start=1,1 --stride=2,3 --count=3,2 --block=1,1 tall.h5
 TOOLTEST tall-5s.ddl -d "/g1/g1.1/dset1.1.2[0;2;10;]" tall.h5
 TOOLTEST tdset-3s.ddl -d "/dset1[1,1;;;]" tdset.h5
-# block
-# TOOLTEST tdset2-1s.ddl -d "/dset1[;3,2;4,4;1,4]" tdset2.h5
+
 
 # test printing characters in ASCII instead of decimal
 TOOLTEST tchar1.ddl -r tchar.h5
@@ -504,6 +503,7 @@ TOOLTEST torderattr2.ddl -H --sort_by=name --sort_order=descending torderattr.h5
 TOOLTEST torderattr3.ddl -H --sort_by=creation_order --sort_order=ascending torderattr.h5
 TOOLTEST torderattr4.ddl -H --sort_by=creation_order --sort_order=descending torderattr.h5
 
+
 # tests for floating point user defined printf format
 TOOLTEST tfpformat.ddl -m %.7f tfpformat.h5
 
@@ -511,6 +511,9 @@ TOOLTEST tfpformat.ddl -m %.7f tfpformat.h5
 TOOLTEST textlinksrc.ddl textlinksrc.h5
 TOOLTEST textlinkfar.ddl textlinkfar.h5
 
+
+
+
 if test $nerrors -eq 0 ; then
    echo "All $DUMPER tests passed."
 fi
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index ba3bdfe..4a48c8a 100755
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -504,6 +504,9 @@ TOOLTEST $arg
 #       to new version and be readable, etc.)
 TOOLTEST $FILE14
 
+# test for datum size > H5TOOLS_MALLOCSIZE
+TOOLTEST $FILE1 -f GZIP=1
+
 if test $nerrors -eq 0 ; then
    echo "All $H5REPACK tests passed."
 fi
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index b165dc7..4e2e036 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -650,7 +650,7 @@ int do_copy_objects(hid_t fidin,
                 {
                     
                     int      j;
-                    
+                   
                     if((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
                         goto error;
                     if((f_space_id = H5Dget_space(dset_in)) < 0)
@@ -790,8 +790,12 @@ int do_copy_objects(hid_t fidin,
                                      */
                                     sm_nbytes = p_type_nbytes;
 
-                                    for (k = rank; k > 0; --k) {
-                                        sm_size[k - 1] = MIN(dims[k - 1], H5TOOLS_BUFSIZE / sm_nbytes);
+                                    for (k = rank; k > 0; --k) 
+                                    {
+                                        hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
+                                        if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
+                                            size = 1;
+                                        sm_size[k - 1] = MIN(dims[k - 1], size);
                                         sm_nbytes *= sm_size[k - 1];
                                         assert(sm_nbytes > 0);
                                     }
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index 341803f..331243a 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -296,14 +296,19 @@ int apply_filters(const char* name,    /* object name from traverse list */
             * determine the strip mine size. The strip mine is
             * a hyperslab whose size is manageable.
             */
+
+            
             
             sm_nbytes = msize;
             for ( i = rank; i > 0; --i) 
             {
-                sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes);
+                hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
+                if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
+                    size = 1;
+                sm_size[i - 1] = MIN(dims[i - 1], size);
                 sm_nbytes *= sm_size[i - 1];
                 assert(sm_nbytes > 0);
-              
+
             }
 
             for ( i = 0; i < rank; i++)
diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c
index 1a9422e..c6afce1 100644
--- a/tools/h5repack/h5repacktst.c
+++ b/tools/h5repack/h5repacktst.c
@@ -3337,6 +3337,46 @@ void write_dset_in(hid_t loc_id,
     type_id = H5Tarray_create2(H5T_NATIVE_INT, 1, dimarray);
     write_dset(loc_id, 1, dims, "array", type_id, buf6);
     status = H5Tclose(type_id);
+
+    {
+
+        double  *dbuf;                           /* information to write */
+        hid_t   did;                             /* dataset ID   */
+        hid_t   sid;                             /* dataspace ID   */
+        hid_t   tid;                             /* datatype ID   */
+        size_t  size;
+        hsize_t sdims[] = {1};
+        hsize_t tdims[] = {H5TOOLS_MALLOCSIZE / sizeof(double) + 1};
+        int     j;
+
+        /* allocate and initialize array data to write */
+        size = ( H5TOOLS_MALLOCSIZE / sizeof(double) + 1 ) * sizeof(double);
+        dbuf = malloc( size );
+
+        for( j = 0; j < H5TOOLS_MALLOCSIZE / sizeof(double) + 1; j++)
+            dbuf[j] = j;
+
+        if (make_diffs)
+        {
+            dbuf[5] = 0;
+            dbuf[6] = 0;
+        }
+
+        /* create a type larger than H5TOOLS_MALLOCSIZE */
+        tid = H5Tarray_create2(H5T_NATIVE_DOUBLE, 1, tdims);
+        size = H5Tget_size(tid);
+        sid = H5Screate_simple(1, sdims, NULL);
+        did = H5Dcreate2(loc_id, "arrayd", tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+#if defined(WRITE_ARRAY)
+        H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dbuf);
+#endif
+
+        /* close */
+        H5Dclose(did);
+        H5Tclose(tid);
+        H5Sclose(sid);
+        free( dbuf );
+    }
     
     /*-------------------------------------------------------------------------
     * H5T_INTEGER and H5T_FLOAT
diff --git a/tools/h5repack/testfiles/h5repack_objs.h5 b/tools/h5repack/testfiles/h5repack_objs.h5
index 0a0c041..061b1cd 100644
--- a/tools/h5repack/testfiles/h5repack_objs.h5
+++ b/tools/h5repack/testfiles/h5repack_objs.h5
diff --git a/tools/lib/h5diff_dset.c b/tools/lib/h5diff_dset.c
index 4826a7f..a295585 100644
--- a/tools/lib/h5diff_dset.c
+++ b/tools/lib/h5diff_dset.c
@@ -452,8 +452,12 @@ hsize_t diff_datasetid( hid_t did1,
             */
             sm_nbytes = p_type_nbytes;
             
-            for (i = rank1; i > 0; --i) {
-                sm_size[i - 1] = MIN(dims1[i - 1], H5TOOLS_BUFSIZE / sm_nbytes);
+            for (i = rank1; i > 0; --i) 
+            {
+                hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
+                if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
+                    size = 1;
+                sm_size[i - 1] = MIN(dims1[i - 1], size);
                 sm_nbytes *= sm_size[i - 1];
                 assert(sm_nbytes > 0);
             }
diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c
index 8f2672e..c238063 100644
--- a/tools/lib/h5tools.c
+++ b/tools/lib/h5tools.c
@@ -941,8 +941,12 @@ h5tools_dump_simple_subset(FILE *stream, const h5tool_format_t *info, hid_t dset
             sm_nbytes = p_type_nbytes = H5Tget_size(p_type);
 
             if (ctx.ndims > 0)
-                for (i = ctx.ndims; i > 0; --i) {
-                    sm_size[i - 1] = MIN(total_size[i - 1], H5TOOLS_BUFSIZE / sm_nbytes);
+                for (i = ctx.ndims; i > 0; --i) 
+                {
+                    hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
+                    if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
+                        size = 1;
+                    sm_size[i - 1] = MIN(total_size[i - 1], size);
                     sm_nbytes *= sm_size[i - 1];
                     assert(sm_nbytes > 0);
                 }
@@ -1167,7 +1171,10 @@ h5tools_dump_simple_dset(FILE *stream, const h5tool_format_t *info, hid_t dset,
 
     if (ctx.ndims > 0) {
         for (i = ctx.ndims; i > 0; --i) {
-            sm_size[i - 1] = MIN(total_size[i - 1], H5TOOLS_BUFSIZE / sm_nbytes);
+            hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
+            if ( size == 0) /* datum size > H5TOOLS_BUFSIZE */
+                size = 1;
+            sm_size[i - 1] = MIN(total_size[i - 1], size);
             sm_nbytes *= sm_size[i - 1];
             assert(sm_nbytes > 0);
         }