[svn-r28002] merge from trunk.inactive/avoid_truncate

2 files changed, 158 insertions, 36 deletions

diff --git a/tools/h5repack/CMakeTests.cmake b/tools/h5repack/CMakeTests.cmake
index dee97ed..caad5c1 100644
--- a/tools/h5repack/CMakeTests.cmake
+++ b/tools/h5repack/CMakeTests.cmake
@@ -14,14 +14,14 @@
         multi
         family
     )
-  
+
     if (DIRECT_VFD)
       set (VFD_LIST ${VFD_LIST} direct)
     endif (DIRECT_VFD)
 
     MACRO (ADD_VFD_TEST vfdname resultcode)
       add_test (
-        NAME H5REPACK-VFD-${vfdname}-h5repacktest 
+        NAME H5REPACK-VFD-${vfdname}-h5repacktest
         COMMAND "${CMAKE_COMMAND}"
             -D "TEST_PROGRAM=$<TARGET_FILE:h5repacktest>"
             -D "TEST_ARGS:STRING="
@@ -37,7 +37,7 @@
       set (last_test "H5REPACK-VFD-${vfdname}-h5repacktest")
     ENDMACRO (ADD_VFD_TEST)
   endif (HDF5_TEST_VFD)
-    
+
   # --------------------------------------------------------------------
   # Copy all the HDF5 files from the source directory into the test directory
   # --------------------------------------------------------------------
@@ -77,6 +77,30 @@
       ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00008.h5
       ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00009.h5
       ${HDF5_TOOLS_SRC_DIR}/testfiles/tfamily00010.h5
+      # tools/testfiles/vds
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/1_a.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/1_b.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/1_c.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/1_d.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/1_e.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/1_f.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/1_vds.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/2_a.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/2_b.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/2_c.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/2_d.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/2_e.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/2_vds.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/3_1_vds.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/3_2_vds.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/4_0.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/4_1.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/4_2.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/4_vds.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/5_a.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/5_b.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/5_c.h5
+      ${HDF5_TOOLS_SRC_DIR}/testfiles/vds/5_vds.h5
   )
 
   set (LIST_OTHER_TEST_FILES
@@ -346,7 +370,7 @@
       add_test (
           NAME H5REPACK_UD-${testname}-clearall-objects
           COMMAND    ${CMAKE_COMMAND}
-              -E remove 
+              -E remove
               testfiles/out-${testname}.${resultfile}
               testfiles/${testname}.${resultfile}.out
               testfiles/${testname}.${resultfile}.out.err
@@ -392,7 +416,7 @@
 ##############################################################################
 
   # --------------------------------------------------------------------
-  # test file names 
+  # test file names
   # --------------------------------------------------------------------
   set (INFO_FILE testfiles/h5repack.info)
 
@@ -415,12 +439,18 @@
   set (FILE18 h5repack_layout2.h5)
   set (FILE_REF h5repack_refs.h5)
   set (FILE_ATTR_REF h5repack_attr_refs.h5)
-  
+  set (FILEV1 1_vds.h5)
+  set (FILEV2 2_vds.h5)
+  set (FILEV3_1 3_1_vds.h5)
+  set (FILEV3_2 3_2_vds.h5)
+  set (FILEV4 4_vds.h5)
+  set (FILEV5 5_vds.h5)
+
   # Remove any output file left over from previous test run
   add_test (
       NAME H5REPACK-clearall-objects
       COMMAND    ${CMAKE_COMMAND}
-          -E remove 
+          -E remove
          ./testfiles/h5dump-help.out
          ./testfiles/h5repack_filters.h5-gzip_verbose_filters.out
          ./testfiles/h5repack_filters.h5-gzip_verbose_filters.out.err
@@ -555,12 +585,41 @@
          ./testfiles/out-meta_short_N.meta_short.h5
          ./testfiles/out-meta_long_M.meta_long.h5
          ./testfiles/out-meta_long_N.meta_long.h5
-         # from the h5repacktst
+         ./testfiles/1_vds.h5-vds_dset_compa-v.out
+         ./testfiles/1_vds.h5-vds_dset_compa-v.out.err
+         ./testfiles/1_vds.h5-vds_dset_conti-v.out
+         ./testfiles/1_vds.h5-vds_dset_conti-v.out.err
+         ./testfiles/2_vds.h5-vds_null_compa-v.out
+         ./testfiles/2_vds.h5-vds_null_compa-v.out.err
+         ./testfiles/2_vds.h5-vds_null_conti-v.out
+         ./testfiles/2_vds.h5-vds_null_conti-v.out.err
+         ./testfiles/4_vds.h5-vds_compa-v.out
+         ./testfiles/4_vds.h5-vds_compa-v.out.err
+         ./testfiles/4_vds.h5-vds_compa_compa-v.out
+         ./testfiles/4_vds.h5-vds_compa_compa-v.out.err
+         ./testfiles/4_vds.h5-vds_compa_conti-v.out
+         ./testfiles/4_vds.h5-vds_compa_conti-v.out.err
+         ./testfiles/4_vds.h5-vds_conti-v.out
+         ./testfiles/4_vds.h5-vds_conti-v.out.err
+         ./testfiles/4_vds.h5-vds_conti_compa-v.out
+         ./testfiles/4_vds.h5-vds_conti_compa-v.out.err
+         ./testfiles/4_vds.h5-vds_conti_conti-v.out
+         ./testfiles/4_vds.h5-vds_conti_conti-v.out.err
+         ./testfiles/out-vds_compa.4_vds.h5
+         ./testfiles/out-vds_compa_compa.4_vds.h5
+         ./testfiles/out-vds_compa_conti.4_vds.h5
+         ./testfiles/out-vds_conti.4_vds.h5
+         ./testfiles/out-vds_conti_compa.4_vds.h5
+         ./testfiles/out-vds_conti_conti.4_vds.h5
+         ./testfiles/out-vds_dset_compa.1_vds.h5
+         ./testfiles/out-vds_dset_conti.1_vds.h5
+         ./testfiles/out-vds_null_compa.2_vds.h5
+         ./testfiles/out-vds_null_conti.2_vds.h5
          h5repack_attr.h5
          h5repack_attr_out.h5
          h5repack_attr_refs.h5
          h5repack_big.h5
-         h5repack_deflate.h5 
+         h5repack_deflate.h5
          h5repack_deflate_out.h5
          h5repack_early2.h5
          h5repack_early.h5
@@ -588,7 +647,7 @@
          h5repack_refs.h5
          h5repack_shuffle.h5
          h5repack_shuffle_out.h5
-         h5repack_soffset.h5 
+         h5repack_soffset.h5
          h5repack_soffset_out.h5
          h5repack_szip.h5
          h5repack_szip_out.h5
@@ -628,14 +687,14 @@
 # Each run generates "<file>.out.h5" and the tool h5diff is used to
 # compare the input and output files
 #
-# the tests are the same as the program h5repacktst, but run from the CLI 
+# the tests are the same as the program h5repacktst, but run from the CLI
 #
 
 # See which filters are usable (and skip tests for filters we
 # don't have).  Do this by searching H5pubconf.h to see which
 # filters are defined.
 
-# detect whether the encoder is present. 
+# detect whether the encoder is present.
   set (USE_FILTER_SZIP_ENCODER "no")
   if (HDF5_ENABLE_SZIP_ENCODING)
     set (USE_FILTER_SZIP_ENCODER ${testh5repack_detect_szip})
@@ -649,7 +708,7 @@
     set (USE_FILTER_SZIP "true")
   endif (H5_HAVE_FILTER_SZIP)
 
-# copy files (these files have no filters) 
+# copy files (these files have no filters)
   ADD_H5_TEST (fill "TEST" ${FILE0})
   ADD_H5_TEST (objs "TEST" ${FILE1})
   ADD_H5_TEST (attr "TEST" ${FILE2})
@@ -666,8 +725,8 @@
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_DEFLATE)
   ADD_H5_TEST (gzip_individual ${TESTTYPE} ${arg})
-  
-# gzip for all 
+
+# gzip for all
   set (arg ${FILE4} -f GZIP=1)
   set (TESTTYPE "TEST")
   if (NOT USE_FILTER_DEFLATE)
@@ -681,7 +740,7 @@
   if (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP)
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP)
-  ADD_H5_TEST (szip_individual ${TESTTYPE} ${arg}) 
+  ADD_H5_TEST (szip_individual ${TESTTYPE} ${arg})
 
 # szip for all
   set (arg ${FILE4} -f SZIP=8,NN)
@@ -689,16 +748,16 @@
   if (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP)
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP)
-  ADD_H5_TEST (szip_all ${TESTTYPE} ${arg}) 
+  ADD_H5_TEST (szip_all ${TESTTYPE} ${arg})
 
 # shuffle with individual object
   set (arg ${FILE4} -f dset2:SHUF  -l dset2:CHUNK=20x10)
-  ADD_H5_TEST (shuffle_individual "TEST" ${arg}) 
+  ADD_H5_TEST (shuffle_individual "TEST" ${arg})
 
 # shuffle for all
   set (arg ${FILE4} -f SHUF)
   ADD_H5_TEST (shuffle_all "TEST" ${arg})
-  
+
 # fletcher32  with individual object
   set (arg ${FILE4} -f dset2:FLET  -l dset2:CHUNK=20x10)
   ADD_H5_TEST (fletcher_individual "TEST" ${arg})
@@ -722,7 +781,7 @@
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_DEFLATE)
   ADD_H5_CMP_TEST (gzip_verbose_filters "O?...ing file[^\n]+\n" ${TESTTYPE} 0 ${arg})
-  
+
 ###########################################################
 # the following tests assume the input files have filters
 ###########################################################
@@ -734,7 +793,7 @@
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP)
   ADD_H5_TEST (szip_copy ${TESTTYPE} ${arg})
-  
+
 # szip remove
   set (arg ${FILE7} --filter=dset_szip:NONE)
   set (TESTTYPE "TEST")
@@ -742,7 +801,7 @@
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP)
   ADD_H5_TEST (szip_remove ${TESTTYPE} ${arg})
-  
+
 # deflate copy
   set (arg ${FILE8})
   set (TESTTYPE "TEST")
@@ -758,7 +817,7 @@
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_DEFLATE)
   ADD_H5_TEST (deflate_remove ${TESTTYPE} ${arg})
-    
+
 # shuffle copy
   set (arg ${FILE9})
   ADD_H5_TEST (shuffle_copy "TEST" ${arg})
@@ -813,14 +872,14 @@
   if (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_DEFLATE)
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_SZIP OR NOT USE_FILTER_DEFLATE)
-  ADD_H5_TEST (deflate_convert ${TESTTYPE} ${arg}) 
+  ADD_H5_TEST (deflate_convert ${TESTTYPE} ${arg})
 
   set (arg ${FILE7} -f dset_szip:GZIP=1)
   set (TESTTYPE "TEST")
   if (NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_DEFLATE)
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_SZIP OR NOT USE_FILTER_SZIP_ENCODER OR NOT USE_FILTER_DEFLATE)
-  ADD_H5_TEST (szip_convert ${TESTTYPE} ${arg}) 
+  ADD_H5_TEST (szip_convert ${TESTTYPE} ${arg})
 
 #limit
   set (arg ${FILE4} -f GZIP=1 -m 1024)
@@ -836,7 +895,7 @@
   if (NOT USE_FILTER_DEFLATE)
     set (TESTTYPE "SKIP")
   endif (NOT USE_FILTER_DEFLATE)
-  ADD_H5_TEST (deflate_file ${TESTTYPE} ${arg}) 
+  ADD_H5_TEST (deflate_file ${TESTTYPE} ${arg})
 
 #########################################################
 # layout options (these files have no filters)
@@ -865,18 +924,18 @@
   ADD_H5_VERIFY_TEST (contig_small_compa "TEST" 0 ${FILE18} contig_small COMPACT -l contig_small:COMPA)
   ADD_H5_VERIFY_TEST (contig_small_fixed_compa "TEST" 0 ${FILE18} chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA)
 
-#--------------------------------------------------------------------------- 
-# Test file contains chunked datasets (need multiple dsets) with 
+#---------------------------------------------------------------------------
+# Test file contains chunked datasets (need multiple dsets) with
 # unlimited max dims.   (HDFFV-7933)
 # Use first dset to test.
 #---------------------------------------------------------------------------
 # chunk to chunk - specify chunk dim bigger than any current dim
   ADD_H5_VERIFY_TEST (chunk2chunk "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:CHUNK=100x300)
 
-# chunk to contiguous 
+# chunk to contiguous
   ADD_H5_VERIFY_TEST (chunk2conti "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CONTI -l chunk_unlimit1:CONTI)
 
-# chunk to compact - convert big dataset (should be > 64k) for this purpose, 
+# chunk to compact - convert big dataset (should be > 64k) for this purpose,
 # should remain as original layout (chunk)
   ADD_H5_VERIFY_TEST (chunk2compa "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk_unlimit1:COMPA)
 
@@ -884,7 +943,7 @@
 # Test -f for some specific cases. Chunked dataset with unlimited max dims.
 # (HDFFV-8012)
 #--------------------------------------------------------------------------
-# - should not fail 
+# - should not fail
 # - should not change max dims from unlimit
 
 # chunk dim is bigger than dataset dim. ( dset size < 64k )
@@ -900,13 +959,13 @@
   ADD_H5_TEST (error4 "TEST" h5repack_layout3.h5 -f NONE)
 
 #--------------------------------------------------------------------------
-# Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset 
+# Test base: Convert CHUNK to CONTI for a chunked dataset with small dataset
 # (dset size < 64K) and with unlimited max dims on a condition as follow.
 # (HDFFV-8214)
 #--------------------------------------------------------------------------
 # chunk dim is bigger than dataset dim. should succeed.
   ADD_H5_VERIFY_TEST (ckdim_biger "TEST" 0 h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI)
-# chunk dim is smaller than dataset dim. should succeed.  
+# chunk dim is smaller than dataset dim. should succeed.
   ADD_H5_VERIFY_TEST (ckdim_smaller "TEST" 0 h5repack_layout3.h5 chunk_unlimit3 CONTI -l chunk_unlimit3:CONTI)
 
 
@@ -975,11 +1034,11 @@
   ADD_H5_TEST (bug1814 "TEST" ${FILE_REF})
 
 # test attribute with various references (bug1797 / HDFFV-5932)
-# the references in attribute of compund or vlen datatype 
+# the references in attribute of compund or vlen datatype
   ADD_H5_TEST (HDFFV-5932 "TEST" ${FILE_ATTR_REF})
 
-# Add test for memory leak in attirbute. This test is verified by CTEST.   
-# 1. leak from vlen string  
+# Add test for memory leak in attirbute. This test is verified by CTEST.
+# 1. leak from vlen string
 # 2. leak from compound type without reference member
 # (HDFFV-7840, )
 # Note: this test is experimental for sharing test file among tools
@@ -989,6 +1048,26 @@
   ADD_H5_TEST_META (meta_short h5repack_layout.h5 -M 8192)
   ADD_H5_TEST_META (meta_long h5repack_layout.h5 --metadata_block_size=8192)
 
+# VDS tests
+
+#########################################################
+# layout options
+#########################################################
+  ADD_H5_VERIFY_TEST (vds_dset_conti "TEST" 0 ${FILEV1} vds_dset CONTIGUOUS -l vds_dset:CONTI)
+  ADD_H5_VERIFY_TEST (vds_null_conti "TEST" 1 ${FILEV2} null CONTIGUOUS -l CONTI)
+  ADD_H5_VERIFY_TEST (vds_dset_compa "TEST" 0 ${FILEV1} vds_dset COMPACT -l vds_dset:COMPA)
+  ADD_H5_VERIFY_TEST (vds_null_compa "TEST" 1 ${FILEV2} null COMPACT -l COMPA)
+
+################################################################
+# layout conversions
+###############################################################
+  ADD_H5_VERIFY_TEST (vds_compa_conti "TEST" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI)
+  ADD_H5_VERIFY_TEST (vds_compa_compa "TEST" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA)
+  ADD_H5_VERIFY_TEST (vds_conti_compa "TEST" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA)
+  ADD_H5_VERIFY_TEST (vds_conti_conti "TEST" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI)
+  ADD_H5_VERIFY_TEST (vds_compa "TEST" 0 ${FILEV4} vds_dset COMPACT -l vds_dset:COMPA)
+  ADD_H5_VERIFY_TEST (vds_conti "TEST" 0 ${FILEV4} vds_dset CONTIGUOUS -l vds_dset:CONTI)
+
 ##############################################################################
 ###    P L U G I N  T E S T S
 ##############################################################################
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 1796151..24298d0 100644
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -113,6 +113,29 @@ $SRC_TOOLS_TESTFILES/tfamily00007.h5
 $SRC_TOOLS_TESTFILES/tfamily00008.h5
 $SRC_TOOLS_TESTFILES/tfamily00009.h5
 $SRC_TOOLS_TESTFILES/tfamily00010.h5
+$SRC_TOOLS_TESTFILES/vds/1_a.h5
+$SRC_TOOLS_TESTFILES/vds/1_b.h5
+$SRC_TOOLS_TESTFILES/vds/1_c.h5
+$SRC_TOOLS_TESTFILES/vds/1_d.h5
+$SRC_TOOLS_TESTFILES/vds/1_e.h5
+$SRC_TOOLS_TESTFILES/vds/1_f.h5
+$SRC_TOOLS_TESTFILES/vds/1_vds.h5
+$SRC_TOOLS_TESTFILES/vds/2_a.h5
+$SRC_TOOLS_TESTFILES/vds/2_b.h5
+$SRC_TOOLS_TESTFILES/vds/2_c.h5
+$SRC_TOOLS_TESTFILES/vds/2_d.h5
+$SRC_TOOLS_TESTFILES/vds/2_e.h5
+$SRC_TOOLS_TESTFILES/vds/2_vds.h5
+$SRC_TOOLS_TESTFILES/vds/3_1_vds.h5
+$SRC_TOOLS_TESTFILES/vds/3_2_vds.h5
+$SRC_TOOLS_TESTFILES/vds/4_0.h5
+$SRC_TOOLS_TESTFILES/vds/4_1.h5
+$SRC_TOOLS_TESTFILES/vds/4_2.h5
+$SRC_TOOLS_TESTFILES/vds/4_vds.h5
+$SRC_TOOLS_TESTFILES/vds/5_a.h5
+$SRC_TOOLS_TESTFILES/vds/5_b.h5
+$SRC_TOOLS_TESTFILES/vds/5_c.h5
+$SRC_TOOLS_TESTFILES/vds/5_vds.h5
 "
 
 LIST_OTHER_TEST_FILES="
@@ -1063,6 +1086,26 @@ TOOLTEST HDFFV-7840 h5diff_attr1.h5
 TOOLTEST_META meta_short h5repack_layout.h5 -M 8192
 TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192
 
+# VDS tests
+
+#########################################################
+# layout options
+#########################################################
+VERIFY_LAYOUT_DSET vds_dset_conti 1_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI
+VERIFY_LAYOUT_ALL vds_null_conti 2_vds.h5 CONTIGUOUS -l CONTI
+VERIFY_LAYOUT_DSET vds_dset_compa 1_vds.h5 vds_dset COMPACT -l vds_dset:COMPA
+VERIFY_LAYOUT_ALL vds_null_compa 2_vds.h5 COMPACT -l COMPA
+
+################################################################
+# layout conversions
+###############################################################
+VERIFY_LAYOUT_DSET vds_compa_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI
+VERIFY_LAYOUT_DSET vds_compa_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA
+VERIFY_LAYOUT_DSET vds_conti_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA
+VERIFY_LAYOUT_DSET vds_conti_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI
+VERIFY_LAYOUT_DSET vds_compa 4_vds.h5 vds_dset COMPACT -l vds_dset:COMPA
+VERIFY_LAYOUT_DSET vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI
+
 # Clean up temporary files/directories
 CLEAN_TESTFILES_AND_TESTDIR