[svn-r19391] Purpose:

Fix for Bug1896 h5repack - changing layout to COMPACT does not work

Description:
Merged from hdf5 trunk (r19389)
Make h5repack be able to convert a layout to COMPACT for small size dataset as default.  Also add verifying layout changes in our test script.

Tested:
 jam, amani

4 files changed, 261 insertions, 22 deletions

diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 94bdf04..f5cc615 100755
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -35,7 +35,12 @@ H5REPACK=h5repack               # The tool name
 H5REPACK_BIN=`pwd`/$H5REPACK    # The path of the tool binary
 
 H5DIFF=../h5diff/h5diff         # The h5diff tool name 
-H5DIFF_BIN=`pwd`/$H5DIFF        # The path of the h5diff  tool binary
+H5DIFF_BIN=`pwd`/$H5DIFF        # The path of the h5diff tool binary
+
+H5DUMP=../h5dump/h5dump         # The h5dump tool name
+H5DUMP_BIN=`pwd`/$H5DUMP        # The path of the h5dump tool binary
+
+GREP='grep'
 
 H5DETECTSZIP=testh5repack_detect_szip              
 H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP    
@@ -59,6 +64,7 @@ FILE14=h5repack_layouto.h5      # A file with an older version of the layout mes
                                 # (copy of test/tlayouto.h5)
 FILE15=h5repack_named_dtypes.h5
 FILE16=tfamily%05d.h5           # located in common testfiles folder
+FILE18=h5repack_layout2.h5
 FILE_REF=h5repack_refs.h5
 
 
@@ -114,8 +120,21 @@ DIFFTEST()
           
 # Call h5repack
 #
+
+
+# call TOOLTEST_MAIN and delete $output file
 TOOLTEST() 
 {
+    echo $@
+    TOOLTEST_MAIN $@
+    path=`pwd`
+    outfile=$path/out.$1
+    rm -f $outfile
+}
+
+# TOOLTEST main function, doesn't delete $output file
+TOOLTEST_MAIN() 
+{
     # Run test.
     TESTING $H5REPACK $@
 
@@ -132,7 +151,98 @@ TOOLTEST()
 	echo " PASSED"
 	DIFFTEST $infile $outfile
     fi
+}
+
+#------------------------------------------
+# Verifying layouts of a dataset 
+VERIFY_LAYOUT_DSET()
+{
+    path=`pwd`
+    outfile=$path/out.$1
+    layoutfile=$path/layout.$1
+    dset=$2
+    expectlayout=$3
+    
+    #---------------------------------
+    # check the layout from a dataset
+    VERIFY  "Layout"
+    $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile
+    $GREP $expectlayout $layoutfile > /dev/null
+    if [ $? -eq 0 ]; then
+        echo " PASSED"
+    else
+        echo " FAILED"
+    fi
+
+    # clean up tmp files
     rm -f $outfile
+    rm -f $layoutfile
+}
+
+#----------------------------------------
+# Verifying layouts from entire file
+VERIFY_LAYOUT_ALL()
+{
+    path=`pwd`
+    outfile=$path/out.$1
+    layoutfile=$path/layout.$1
+    expectlayout=$2
+    
+    #---------------------------------
+    # check the layout from a dataset
+    # check if the other layouts still exsit
+    VERIFY  "Layout "
+        # if CONTIGUOUS
+        if [ $expectlayout = "CONTIGUOUS" ]; then
+            $H5DUMP_BIN -pH $outfile > $layoutfile
+            $GREP "COMPACT" $layoutfile  > /dev/null
+            if [ $? -eq 0 ]; then
+                echo " FAILED"
+            else
+                $GREP "CHUNKED" $layoutfile  > /dev/null
+                if [ $? -eq 0 ]; then
+                    echo " FAILED"
+                else
+                    echo " PASSED"
+                fi
+            fi
+        else    
+            # if COMPACT
+            if [ $expectlayout = "COMPACT" ]; then
+                $H5DUMP_BIN -pH $outfile > $layoutfile
+                $GREP "CHUNKED" $layoutfile  > /dev/null
+                if [ $? -eq 0 ]; then
+                    echo " FAILED"
+                else
+                    $GREP "CONTIGUOUS" $layoutfile  > /dev/null
+                    if [ $? -eq 0 ]; then
+                        echo " FAILED"
+                    else
+                        echo " PASSED"
+                    fi
+                fi
+            else
+                # if CHUNKED
+                if [ $expectlayout = "CHUNKED" ]; then
+                    $H5DUMP_BIN -pH $outfile > $layoutfile
+                    $GREP "CONTIGUOUS" $layoutfile  > /dev/null
+                    if [ $? -eq 0 ]; then
+                        echo " FAILED"
+                    else
+                        $GREP "COMPACT" $layoutfile  > /dev/null
+                        if [ $? -eq 0 ]; then
+                            echo " FAILED"
+                        else
+                            echo " PASSED"
+                        fi
+                    fi
+                fi
+           fi
+        fi
+
+    # clean up tmp files
+    rm -f $outfile
+    rm -f $layoutfile
 }
 
 # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
@@ -493,32 +603,64 @@ else
  TOOLTEST $arg 
 fi
 
-
 #########################################################
 # layout options (these files have no filters)
 #########################################################
+TOOLTEST_MAIN $FILE4 --layout dset2:CHUNK=20x10
+VERIFY_LAYOUT_DSET $FILE4 dset2 CHUNKED
 
-TOOLTEST $FILE4 --layout=dset2:CHUNK=20x10
-TOOLTEST $FILE4 -l CHUNK=20x10
-TOOLTEST $FILE4 -l dset2:CONTI
-TOOLTEST $FILE4 -l CONTI
-TOOLTEST $FILE4 -l dset2:COMPA
-TOOLTEST $FILE4 -l COMPA
+TOOLTEST_MAIN $FILE4 -l CHUNK=20x10
+VERIFY_LAYOUT_ALL $FILE4 CHUNKED
 
+TOOLTEST_MAIN $FILE4 -l dset2:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset2 CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l CONTI
+VERIFY_LAYOUT_ALL $FILE4 CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset2:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset2 COMPACT
+
+TOOLTEST_MAIN $FILE4 -l COMPA
+VERIFY_LAYOUT_ALL $FILE4 COMPACT
 
 ################################################################
 # layout conversions (file has no filters)
 ###############################################################
 
-TOOLTEST $FILE4 -l dset_compact:CONTI
-TOOLTEST $FILE4 -l dset_compact:CHUNK=2x5
-TOOLTEST $FILE4 -l dset_compact:COMPA
-TOOLTEST $FILE4 -l dset_contiguous:COMPA
-TOOLTEST $FILE4 -l dset_contiguous:CHUNK=3x6
-TOOLTEST $FILE4 -l dset_contiguous:CONTI
-TOOLTEST $FILE4 -l dset_chunk:COMPA
-TOOLTEST $FILE4 -l dset_chunk:CONTI
-TOOLTEST $FILE4 -l dset_chunk:CHUNK=18x13
+TOOLTEST_MAIN $FILE4 -l dset_compact:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset_compact CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset_compact:CHUNK=2x5
+VERIFY_LAYOUT_DSET $FILE4 dset_compact CHUNKED
+
+TOOLTEST_MAIN $FILE4 -l dset_compact:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset_compact COMPACT
+
+TOOLTEST_MAIN $FILE4 -l dset_contiguous:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset_contiguous COMPACT
+
+TOOLTEST_MAIN $FILE4 -l dset_contiguous:CHUNK=3x6
+VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CHUNKED
+
+TOOLTEST_MAIN $FILE4 -l dset_contiguous:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset_chunk:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset_chunk COMPACT
+
+TOOLTEST_MAIN $FILE4 -l dset_chunk:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset_chunk CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset_chunk:CHUNK=18x13
+VERIFY_LAYOUT_DSET $FILE4 dset_chunk CHUNKED
+
+# test convert small size dataset ( < 1k) to compact layout without -m
+TOOLTEST_MAIN $FILE18 -l contig_small:COMPA
+VERIFY_LAYOUT_DSET $FILE18 contig_small COMPACT
+
+TOOLTEST_MAIN $FILE18 -l chunked_small_fixed:COMPA
+VERIFY_LAYOUT_DSET $FILE18 chunked_small_fixed COMPACT
 
 # Native option
 # Do not use FILE1, as the named dtype will be converted to native, and h5diff will
@@ -532,7 +674,8 @@ arg="$FILE4 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest
 if test $USE_FILTER_DEFLATE != "yes" ; then
  SKIP $arg
 else
- TOOLTEST $arg
+ TOOLTEST_MAIN $arg
+ VERIFY_LAYOUT_ALL $FILE4 CHUNKED
 fi
 
 # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
@@ -540,7 +683,8 @@ arg="$FILE4 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
 if test $USE_FILTER_DEFLATE != "yes" ; then
  SKIP $arg
 else
- TOOLTEST $arg
+ TOOLTEST_MAIN $arg
+ VERIFY_LAYOUT_ALL $FILE4 CHUNKED
 fi
 
 # several global filters
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 6853297..df49b3c 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -745,9 +745,15 @@ int do_copy_objects(hid_t fidin,
                         /* get the storage size of the input dataset */
                         dsize_in=H5Dget_storage_size(dset_in);
 
-                        /* check for datasets too small */
-                        if (nelmts*msize < options->min_comp )
-                            apply_s=0;
+                        /* check for small size datasets (less than 1k) except 
+                         * changing to COMPACT. For the reference, COMPACT is limited
+                         * by size 64K by library.
+                         */
+                        if (options->layout_g != H5D_COMPACT)
+                        {
+                            if ( nelmts*msize < options->min_comp )
+                                apply_s=0;
+                        }
 
                         /* apply the filter */
                         if (apply_s)
diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c
index 57fb1ce..8eddeca 100644
--- a/tools/h5repack/h5repacktst.c
+++ b/tools/h5repack/h5repacktst.c
@@ -76,6 +76,9 @@
 #define FNAME17     "h5repack_named_dtypes.h5"
 #define FNAME17OUT  "h5repack_named_dtypes_out.h5"
 
+#define FNAME18     "h5repack_layout2.h5"
+#define FNAME18OUT  "h5repack_layout2_out.h5"
+
 #define FNAME_UB   "ublock.bin"
 
 /* obj and region references */
@@ -117,6 +120,7 @@ static int make_attributes(hid_t loc_id);
 static int make_hlinks(hid_t loc_id);
 static int make_early(void);
 static int make_layout(hid_t loc_id);
+static int make_layout2(hid_t loc_id);
 #ifdef H5_HAVE_FILTER_SZIP
 static int make_szip(hid_t loc_id);
 #endif /* H5_HAVE_FILTER_SZIP */
@@ -1630,6 +1634,19 @@ int make_testfiles(void)
         return -1;
 
     /*-------------------------------------------------------------------------
+    * create a file for layout conversion test
+    *-------------------------------------------------------------------------
+    */
+    if((fid = H5Fcreate(FNAME18, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        return -1;
+
+    if(make_layout2(fid) < 0)
+        goto out;
+
+    if(H5Fclose(fid) < 0)
+        return -1;
+
+    /*-------------------------------------------------------------------------
     * create a file for the H5D_ALLOC_TIME_EARLY test
     *-------------------------------------------------------------------------
     */
@@ -2910,6 +2927,78 @@ out:
 }
 
 /*-------------------------------------------------------------------------
+ * Function: make_layout2
+ *
+ * Purpose: create datasets with contiguous and chunked layouts:
+ *
+ *	contig_small: < 1k, fixed dims datspace
+ *	chunked_small_fixed: < 1k, fixed dims dataspace
+ *
+ *-------------------------------------------------------------------------
+ */
+#define S_DIM1	4
+#define S_DIM2	10
+#define CONTIG_S	"contig_small"
+#define CHUNKED_S_FIX	"chunked_small_fixed"
+
+static
+int make_layout2(hid_t loc_id)
+{
+
+    hid_t    contig_dcpl = -1; 	/* dataset creation property list */
+    hid_t    chunked_dcpl = -1; /* dataset creation property list */
+
+    int      i, j, n;		/* Local index variables */
+    int	     ret_value = -1;	/* Return value */
+    hid_t    s_sid = -1;	/* dataspace ID */
+
+    hsize_t  s_dims[RANK] = {S_DIM1,S_DIM2};	/* Dataspace (< 1 k) */
+    hsize_t  chunk_dims[RANK] = {S_DIM1/2, S_DIM2/2};	/* Dimension sizes for chunks */
+
+    int      s_buf[S_DIM1][S_DIM2];	/* Temporary buffer */
+
+    for(i = n = 0; i < S_DIM1; i++) {
+        for (j = 0; j < S_DIM2; j++) {
+            s_buf[i][j] = n++;
+        }
+    }
+
+    /* Create dataspaces */
+    if((s_sid = H5Screate_simple(RANK, s_dims, NULL)) < 0)
+        goto out;
+
+    /* Create contiguous datasets */
+    if((contig_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+        goto out;
+    if(H5Pset_layout(contig_dcpl, H5D_CONTIGUOUS) < 0)
+        goto out;
+    if(make_dset(loc_id, CONTIG_S, s_sid, contig_dcpl, s_buf) < 0)
+        goto out;
+
+    /* Create chunked datasets */
+    if((chunked_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+        goto out;
+    if(H5Pset_chunk(chunked_dcpl, RANK, chunk_dims) < 0)
+        goto out;
+    if(make_dset(loc_id, CHUNKED_S_FIX, s_sid, chunked_dcpl, s_buf) < 0)
+        goto out;
+
+    ret_value = 0;
+
+out:
+    H5E_BEGIN_TRY {
+        H5Pclose(contig_dcpl);
+        H5Pclose(chunked_dcpl);
+    
+        H5Sclose(s_sid);
+
+    } H5E_END_TRY;
+
+    return(ret_value);
+
+} /* make_layout2() */
+
+/*-------------------------------------------------------------------------
 * Function: make a file with an integer dataset with a fill value
 *
 * Purpose: test copy of fill values
diff --git a/tools/h5repack/testfiles/h5repack_layout2.h5 b/tools/h5repack/testfiles/h5repack_layout2.h5
new file mode 100644
index 0000000..6e77e41
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_layout2.h5