From d7bfd458c4eadcfb338e9856256ecd98aa47aa10 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Fri, 9 Jun 2017 14:01:07 -0500 Subject: Changes made based on RFC review comments Test the changes in a branch via daily testing. --- src/H5Fprivate.h | 4 +++- src/H5MF.c | 2 +- src/H5MFsection.c | 32 ++++++++++++++++---------------- src/H5Pfcpl.c | 3 +++ test/tfile.c | 17 +++++++++++++++++ 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 6f68a62..eba48de 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -549,9 +549,11 @@ #define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096 /* For paged aggregation: minimum value for file space page size */ #define H5F_FILE_SPACE_PAGE_SIZE_MIN 512 +/* For paged aggregation: maxiumum value for file space page size: 1 gigabyte */ +#define H5F_FILE_SPACE_PAGE_SIZE_MAX 1024*1024*1024 /* For paged aggregation: drop free-space with size <= this threshold for small meta section */ -#define H5F_FILE_SPACE_PGEND_META_THRES 10 +#define H5F_FILE_SPACE_PGEND_META_THRES 0 /* Default for threshold for alignment (can be set via H5Pset_alignment()) */ #define H5F_ALIGN_DEF 1 diff --git a/src/H5MF.c b/src/H5MF.c index e54d809..d7af56a 100644 --- a/src/H5MF.c +++ b/src/H5MF.c @@ -1519,7 +1519,7 @@ H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ H5F_mem_page_t fs_type; /* Free space type */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ - htri_t ret_value = FAIL; /* Return value */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG diff --git a/src/H5MFsection.c b/src/H5MFsection.c index 02fc2d9..14e0ad1 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -130,27 +130,27 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Class info for "small" free space sections */ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{ /* Class variables */ - H5MF_FSPACE_SECT_SMALL, /* Section type */ - 0, /* Extra serialized size */ - H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ - NULL, /* Class private info */ + H5MF_FSPACE_SECT_SMALL, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ /* Class methods */ - NULL, /* Initialize section class */ - NULL, /* Terminate section class */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ /* Object methods */ - H5MF_sect_small_add, /* Add section */ - NULL, /* Serialize section */ + H5MF_sect_small_add, /* Add section */ + NULL, /* Serialize section */ H5MF_sect_deserialize, /* Deserialize section */ - H5MF_sect_small_can_merge, /* Can sections merge? */ - H5MF_sect_small_merge, /* Merge sections */ - H5MF_sect_small_can_shrink, /* Can section shrink container?*/ - H5MF_sect_small_shrink, /* Shrink container w/section */ - H5MF_sect_free, /* Free section */ - H5MF_sect_valid, /* Check validity of section */ - H5MF_sect_split, /* Split section node for alignment */ - NULL, /* Dump debugging for section */ + H5MF_sect_small_can_merge, /* Can sections merge? */ + H5MF_sect_small_merge, /* Merge sections */ + NULL, /* Can section shrink container?*/ + NULL, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ }}; /* Class info for "large" free space sections */ diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c index 5383aae..6b0d2c0 100644 --- a/src/H5Pfcpl.c +++ b/src/H5Pfcpl.c @@ -1472,6 +1472,9 @@ H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size) if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512") + if(fsp_size > H5F_FILE_SPACE_PAGE_SIZE_MAX) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to more than 1GB") + /* Set the value*/ if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size") diff --git a/test/tfile.c b/test/tfile.c index c59e756..70508cd 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -109,6 +109,7 @@ #define TEST_THRESHOLD10 10 /* Free space section threshold */ #define FSP_SIZE_DEF 4096 /* File space page size default */ #define FSP_SIZE512 512 /* File space page size */ +#define FSP_SIZE1G 1024*1024*1024 /* File space page size */ /* Declaration for test_libver_macros2() */ #define FILE6 "tfile6.h5" /* Test file */ @@ -3581,6 +3582,9 @@ test_filespace_info(const char *env_h5_drvr) * Setting value less than 512 will return an error; * --setting file space page size to 0 * --setting file space page size to 511 + * + * File space page size has a maximum size of 1 gigabyte. + * Setting value greater than 1 gigabyte will return an error. */ /* Create file creation property list template */ fcpl = H5Pcreate(H5P_FILE_CREATE); @@ -3598,6 +3602,12 @@ test_filespace_info(const char *env_h5_drvr) } H5E_END_TRY; VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 1GB+1: should fail */ + H5E_BEGIN_TRY { + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G+1); + } H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 512: should succeed */ ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512); CHECK(ret, FAIL, "H5Pset_file_space_page_size"); @@ -3605,6 +3615,13 @@ test_filespace_info(const char *env_h5_drvr) CHECK(ret, FAIL, "H5Pget_file_space_page_size"); VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + /* Setting to 1GB: should succeed */ + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE1G, "H5Pget_file_space_page_size"); + /* Close property list */ H5Pclose(fcpl); -- cgit v0.12 From d4e75f6a50d028cae250f764fe118a5f0c2257f6 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Mon, 10 Jul 2017 02:40:59 -0500 Subject: Skip test in test/fheap.c when: a) multi/split drivers and b) persisting free-space or using paged aggregation strategy because the library will fail file creation (temporary) for the above conditions. --- src/H5MFsection.c | 2 +- test/fheap.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/H5MFsection.c b/src/H5MFsection.c index 14e0ad1..02e3218 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -674,7 +674,7 @@ HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info. HDfprintf(stderr, "%s: section is dropped\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ - /* Adjust the section if it is not at page end but its size + pgend threshold is at page end */ + /* Adjust the section if it is not at page end but its size + prem is at page end */ else if(prem <= H5F_PGEND_META_THRES(udata->f)) { (*sect)->sect_info.size += prem; diff --git a/test/fheap.c b/test/fheap.c index 4be6cb9..6c3a8ac 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -16377,6 +16377,16 @@ main(void) unsigned nerrors = 0; /* Cumulative error count */ unsigned num_pb_fs = 1; /* The number of settings to test for page buffering and file space handling */ int ExpressMode; /* Express testing level */ + const char *envval; /* Environment variable */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + + /* Don't run this test using certain file drivers */ + envval = HDgetenv("HDF5_DRIVER"); + if(envval == NULL) + envval = "nomatch"; + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") && HDstrcmp(envval, "multi")); /* Reset library */ h5_reset(); @@ -16428,6 +16438,12 @@ main(void) shared_wobj_g[u] = (unsigned char)u; for(v = 0; v < num_pb_fs; v++) { + /* Skip test when: + a) multi/split drivers and + b) persisting free-space or using paged aggregation strategy + because the library will fail file creation (temporary) for the above conditions */ + if(!contig_addr_vfd && v) + break; if((fcpl = H5Pcopy(def_fcpl)) < 0) TEST_ERROR -- cgit v0.12 From aa0ad22a941e6d7fb4f31718c47d045931f8b317 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Sat, 29 Jul 2017 01:10:25 -0500 Subject: Add h5repack tests for paged aggregation Add tests to h5repack.sh.in to verify options added for paged aggregation work as expected. --- MANIFEST | 12 +++ tools/test/h5repack/h5repack.sh.in | 103 +++++++++++++++++++++ .../testfiles/GS.h5repack_paged_nopersist.h5.ddl | 15 +++ .../testfiles/S.h5repack_fsm_aggr_persist.h5.ddl | 15 +++ .../SP.h5repack_fsm_aggr_nopersist.h5.ddl | 16 ++++ .../testfiles/SP.h5repack_paged_persist.h5.ddl | 15 +++ .../h5repack/testfiles/SPT.h5repack_aggr.h5.ddl | 16 ++++ .../h5repack/testfiles/STG.h5repack_none.h5.ddl | 15 +++ tools/test/h5repack/testfiles/h5repack_aggr.h5 | Bin 0 -> 2448 bytes .../testfiles/h5repack_fsm_aggr_nopersist.h5 | Bin 0 -> 2448 bytes .../testfiles/h5repack_fsm_aggr_persist.h5 | Bin 0 -> 2565 bytes tools/test/h5repack/testfiles/h5repack_none.h5 | Bin 0 -> 1808 bytes .../h5repack/testfiles/h5repack_paged_nopersist.h5 | Bin 0 -> 8192 bytes .../h5repack/testfiles/h5repack_paged_persist.h5 | Bin 0 -> 16384 bytes 14 files changed, 207 insertions(+) create mode 100644 tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl create mode 100644 tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl create mode 100644 tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl create mode 100644 tools/test/h5repack/testfiles/h5repack_aggr.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_none.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_paged_persist.h5 diff --git a/MANIFEST b/MANIFEST index 6decb7d..8e33cfb 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2407,6 +2407,7 @@ #test files for h5repack ./tools/test/h5repack/testfiles/README +./tools/test/h5repack/testfiles/h5repack_aggr.h5 ./tools/test/h5repack/testfiles/h5repack_attr.h5 ./tools/test/h5repack/testfiles/h5repack_attr_refs.h5 ./tools/test/h5repack/testfiles/h5repack_deflate.h5 @@ -2416,6 +2417,8 @@ ./tools/test/h5repack/testfiles/h5repack_fill.h5 ./tools/test/h5repack/testfiles/h5repack_filters.h5 ./tools/test/h5repack/testfiles/h5repack_fletcher.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 ./tools/test/h5repack/testfiles/h5repack_hlink.h5 ./tools/test/h5repack/testfiles/h5repack.info ./tools/test/h5repack/testfiles/h5repack_layout.h5 @@ -2427,7 +2430,10 @@ ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum_deflated.h5 ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum.h5 ./tools/test/h5repack/testfiles/h5repack_nbit.h5 +./tools/test/h5repack/testfiles/h5repack_none.h5 ./tools/test/h5repack/testfiles/h5repack_objs.h5 +./tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_paged_persist.h5 ./tools/test/h5repack/testfiles/h5repack_refs.h5 ./tools/test/h5repack/testfiles/h5repack_shuffle.h5 ./tools/test/h5repack/testfiles/h5repack_soffset.h5 @@ -2446,6 +2452,12 @@ ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.tst +./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl +./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl +./tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl +./tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl ./tools/test/h5repack/testfiles/1_vds.h5-vds_dset_chunk20x10x5-v.ddl ./tools/test/h5repack/testfiles/2_vds.h5-vds_chunk3x6x9-v.ddl ./tools/test/h5repack/testfiles/3_1_vds.h5-vds_chunk2x5x8-v.ddl diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index bc6b527..e474bc3 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -36,6 +36,9 @@ H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary H5DUMP=../../src/h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary +H5STAT=../../src/h5stat/h5stat # The h5stat tool name +H5STAT_BIN=`pwd`/$H5STAT # The path of the h5stat tool binary + RM='rm -rf' CMP='cmp' DIFF='diff -c' @@ -101,6 +104,12 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 $SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 $SRC_H5REPACK_TESTFILES/h5repack_szip.h5 +$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_none.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 $SRC_TOOLS_TESTFILES/tfamily00000.h5 $SRC_TOOLS_TESTFILES/tfamily00001.h5 @@ -156,6 +165,12 @@ $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl $SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl +$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl " # @@ -707,6 +722,58 @@ TOOLTEST_DUMP() rm -f $outfile } +# This is similar to TOOLTEST_DUMP(). +# Test h5repack with options added for paged aggregation. +# h5stat is used on the repacked file and the expected output +# is compared for correctness. +# +TOOLTEST_STAT() +{ + infile=$2 + outfile=out-$1.$2 + expect="$TESTDIR/$1.$2.ddl" + actual="$TESTDIR/out-$1.$2.out" + actual_err="$TESTDIR/out-$1.$2.err" + + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5stat output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5STAT_BIN -S -s $outfile + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + RET=$? + + fi + + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + # TOOLTEST_META: # Test metadata block size option. # Reason to create a function here is to localize all special steps related to @@ -1057,6 +1124,42 @@ fi arg="tordergr.h5 -L" TOOLTEST_DUMP crtorder $arg +################################################################################################### +# Testing paged aggregation related options: +# -G pagesize +# -P 1 or 0 +# -S strategy +# -T threshold +# +# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": +# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ +# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ +# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ +# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ +# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ +# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ +# +##################################################################################################### +# +arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_fsm_aggr_persist.h5 -S AGGR" +TOOLTEST_STAT S $arg +# +arg="h5repack_none.h5 -S PAGE -T 10 -G 2048" +TOOLTEST_STAT STG $arg +# +arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR" +TOOLTEST_STAT GS $arg +# +arg="h5repack_paged_persist.h5 -S NONE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5" +TOOLTEST_STAT SPT $arg + + ######################################################### # layout options (these files have no filters) ######################################################### diff --git a/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl new file mode 100644 index 0000000..d8ca992 --- /dev/null +++ b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-GS.h5repack_paged_nopersist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 512 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl new file mode 100644 index 0000000..87ccbc4 --- /dev/null +++ b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-S.h5repack_fsm_aggr_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl new file mode 100644 index 0000000..affa93b --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SP.h5repack_fsm_aggr_nopersist.h5 +Free-space persist: TRUE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 1000 - 9999: 2 + Total # of sections: 2 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1602 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 6307 bytes/38.5% + Unaccounted space: 8075 bytes +Total space: 16384 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl new file mode 100644 index 0000000..1084090 --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-SP.h5repack_paged_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_NONE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 0 bytes +Total space: 1672 bytes diff --git a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl new file mode 100644 index 0000000..1ce06b2 --- /dev/null +++ b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SPT.h5repack_aggr.h5 +Free-space persist: TRUE +Free-space section threshold: 5 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 100 - 999: 1 + Total # of sections: 1 +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1485 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 680 bytes/26.5% + Unaccounted space: 0 bytes +Total space: 2565 bytes diff --git a/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl new file mode 100644 index 0000000..51cd7e5 --- /dev/null +++ b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-STG.h5repack_none.h5 +Free-space persist: FALSE +Free-space section threshold: 10 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 2048 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 2424 bytes +Total space: 4096 bytes diff --git a/tools/test/h5repack/testfiles/h5repack_aggr.h5 b/tools/test/h5repack/testfiles/h5repack_aggr.h5 new file mode 100644 index 0000000..8da0751 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_aggr.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 new file mode 100644 index 0000000..1cd5f3f Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 new file mode 100644 index 0000000..469dbce Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_none.h5 b/tools/test/h5repack/testfiles/h5repack_none.h5 new file mode 100644 index 0000000..2e4d789 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_none.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 new file mode 100644 index 0000000..352e586 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 new file mode 100644 index 0000000..64c3b23 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 differ -- cgit v0.12 From bda86da40b5c4937fa02ff4aa5a8f0bf89ba2d0b Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Wed, 2 Aug 2017 23:36:35 -0500 Subject: Update release notes 1) Fixes for paged aggregation 2) Fix for HDFFV-10217 3) Fix for HDFFV-7853 --- release_docs/RELEASE.txt | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index b6f508b..08e2477 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -175,7 +175,41 @@ Bug Fixes since HDF5-1.10.0-patch1 release Library ------- - - + - Fixes for paged aggregation + + Skip test in test/fheap.c when: + (1) multi/split drivers and + (2) persisting free-space or using paged aggregation strategy + + (VC, 2017/07/10) + + - Fixes for paged aggregation + + Changes made based on RFC review comments: + (1) Add maximum value for file space page size + (2) Drop check for page end metadata threshold + (3) Remove "can_shrink" and "shrink" callbacks for small section class + + (VC, 2017/06/09) + + - Fix for infinite loop in H5VM_power2up(). + + The function H5VM_power2up() returns the next power of 2 + for n. When n exceeds 2^63, it overflows and becomes 0 causing + the infinite looping. + + The fix ensures that the function checks for n >= 2^63 + and returns 0. + + (HDFFV-10217, VC, 2017/07/10) + + - Fix for H5Ocopy doesn't work with open identifiers + + Changes made so that raw data for dataset objects are copied from + cached info when possible instead of flushing objects to file and + read them back in again. + + (HDFFV-7853, VC, 2017/07/05) Configuration ------------- @@ -225,6 +259,13 @@ Bug Fixes since HDF5-1.10.0-patch1 release Tools ----- + - h5repack + + Add tests to h5repack.sh.in to verify options added for paged + aggregation work as expected. + + (VC, 2017/08/03) + - h5diff h5diff segfaulted on compare of a NULL variable length string. -- cgit v0.12 From 98d4addcf7cbec71600663e7023b6b7e0b5b5f35 Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Wed, 2 Aug 2017 10:53:17 -0500 Subject: Add h5repack tests for paged aggregation --- tools/test/h5repack/CMakeTests.cmake | 97 ++++++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index 897209c..225f6a8 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -76,6 +76,12 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_shuffle.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_soffset.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_szip.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_aggr.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_nopersist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_persist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_none.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_nopersist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_persist.h5 # h5diff/testfile ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5 # tools/testfiles @@ -139,6 +145,12 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti-v.ddl ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5-plugin_zero.tst ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/crtorder.tordergr.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/STG.h5repack_none.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/GS.h5repack_paged_nopersist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_paged_persist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SPT.h5repack_aggr.h5.ddl ) foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) @@ -238,7 +250,8 @@ if (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_CMP-${testname} - COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) else () add_test ( NAME H5REPACK_CMP-${testname} @@ -272,7 +285,8 @@ if (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_MASK-${testname} - COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) else (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_MASK-${testname} @@ -305,7 +319,8 @@ # If using memchecker add tests without using scripts add_test ( NAME H5REPACK_DMP-${testname} - COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) if (NOT "${last_test}" STREQUAL "") set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES DEPENDS ${last_test}) endif () @@ -326,6 +341,40 @@ endif () endmacro () + macro (ADD_H5_STAT_TEST testname testtype resultcode statarg resultfile) + if ("${testtype}" STREQUAL "SKIP") + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_STAT-${testname}-SKIPPED + COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile}" + ) + endif () + else () + # If using memchecker add tests without using scripts + add_test ( + NAME H5REPACK_STAT-${testname} + COMMAND $ ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile} + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5REPACK_STAT-${testname} PROPERTIES DEPENDS ${last_test}) + endif () + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_STAT-h5stat-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=-S;-s;out-${statarg}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}-${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${statarg}.${resultfile}.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5REPACK_STAT-h5stat-${testname} PROPERTIES DEPENDS "H5REPACK_STAT-${testname}") + endif () + endif () + endmacro () + macro (ADD_H5_VERIFY_TEST testname testtype resultcode testfile testdset testfilter) if ("${testtype}" STREQUAL "SKIP") if (NOT HDF5_ENABLE_USING_MEMCHECKER) @@ -972,6 +1021,48 @@ set (TESTTYPE "TEST") ADD_H5_DMP_TEST (crtorder ${TESTTYPE} 0 ${arg}) +################################################################################################### +# Testing paged aggregation related options: +# -G pagesize +# -P 1 or 0 +# -S strategy +# -T threshold +# +# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": +# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ +# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ +# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ +# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ +# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ +# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ +# +##################################################################################################### +# + set (arg h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SP_PAGE ${TESTTYPE} 0 SP ${arg}) + + set (arg h5repack_fsm_aggr_persist.h5 -S AGGR) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (S_AGGR ${TESTTYPE} 0 S ${arg}) + + set (arg h5repack_none.h5 -S PAGE -T 10 -G 2048) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (STG_PAGE ${TESTTYPE} 0 STG ${arg}) + + set (arg h5repack_paged_nopersist.h5 -G 512 -S AGGR) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (GS_AGGR ${TESTTYPE} 0 GS ${arg}) + + set (arg h5repack_paged_persist.h5 -S NONE -P 1) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SP_NONE ${TESTTYPE} 0 SP ${arg}) + + set (arg h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SPT_FSM_AGGR ${TESTTYPE} 0 SPT ${arg}) + + ######################################################### # layout options (these files have no filters) ######################################################### -- cgit v0.12