diff options
author | Quincey Koziol <koziol@hdfgroup.org> | 2018-11-18 06:39:27 (GMT) |
---|---|---|
committer | Quincey Koziol <koziol@hdfgroup.org> | 2018-11-18 06:39:27 (GMT) |
commit | d0d360ff2d6afb275836645d01d5d6ddbed39caa (patch) | |
tree | 577fee05a1103303fcf1ce0208175cf4c241e59b | |
parent | 59c28780d35924b6512fe18a0eeb49d55e03f0fc (diff) | |
parent | e9125d6a9960a588244998e959598a27d3c8d352 (diff) | |
download | hdf5-d0d360ff2d6afb275836645d01d5d6ddbed39caa.zip hdf5-d0d360ff2d6afb275836645d01d5d6ddbed39caa.tar.gz hdf5-d0d360ff2d6afb275836645d01d5d6ddbed39caa.tar.bz2 |
Merge branch 'develop' of https://bitbucket.hdfgroup.org/scm/hdffv/hdf5 into stackable_vol
-rw-r--r-- | MANIFEST | 1 | ||||
-rw-r--r-- | fortran/src/H5Aff.F90 | 3 | ||||
-rw-r--r-- | fortran/src/H5Dff.F90 | 9 | ||||
-rw-r--r-- | fortran/src/H5Pff.F90 | 8 | ||||
-rw-r--r-- | fortran/src/H5Rff.F90 | 6 | ||||
-rw-r--r-- | fortran/src/H5Tff.F90 | 2 | ||||
-rw-r--r-- | fortran/src/H5_buildiface.F90 | 194 | ||||
-rw-r--r-- | java/test/TestH5Z.java | 34 | ||||
-rw-r--r-- | release_docs/RELEASE.txt | 14 | ||||
-rw-r--r-- | src/H5Dchunk.c | 22 | ||||
-rw-r--r-- | src/H5FDlog.c | 16 | ||||
-rw-r--r-- | src/H5FDsec2.c | 16 | ||||
-rw-r--r-- | src/H5Fprivate.h | 2 | ||||
-rw-r--r-- | src/H5Pfapl.c | 15 | ||||
-rw-r--r-- | tools/src/misc/h5repart.c | 45 | ||||
-rw-r--r-- | tools/test/misc/CMakeTestsRepart.cmake | 5 | ||||
-rw-r--r-- | tools/test/misc/repart_test.c | 21 | ||||
-rw-r--r-- | tools/test/misc/testh5repart.sh.in | 6 | ||||
-rw-r--r-- | tools/test/perform/CMakeLists.txt | 10 | ||||
-rw-r--r-- | tools/test/perform/Makefile.am | 4 | ||||
-rw-r--r-- | tools/test/perform/chunk_cache.c | 405 |
21 files changed, 716 insertions, 122 deletions
@@ -2616,6 +2616,7 @@ ./tools/test/perform/build_h5perf_alone.sh ./tools/test/perform/build_h5perf_serial_alone.sh ./tools/test/perform/chunk.c +./tools/test/perform/chunk_cache.c ./tools/test/perform/direct_write_perf.c ./tools/test/perform/gen_report.pl ./tools/test/perform/iopipe.c diff --git a/fortran/src/H5Aff.F90 b/fortran/src/H5Aff.F90 index 827b803..a728f2d 100644 --- a/fortran/src/H5Aff.F90 +++ b/fortran/src/H5Aff.F90 @@ -76,6 +76,9 @@ MODULE H5A USE H5GLOBAL + PRIVATE h5awrite_char_scalar, h5awrite_ptr + PRIVATE h5aread_char_scalar, h5aread_ptr + INTERFACE h5awrite_f MODULE PROCEDURE h5awrite_char_scalar ! This is the preferred way to call h5awrite diff --git a/fortran/src/H5Dff.F90 b/fortran/src/H5Dff.F90 index 3915f72..77f0a15 100644 --- a/fortran/src/H5Dff.F90 +++ b/fortran/src/H5Dff.F90 @@ -90,6 +90,15 @@ MODULE H5D USE, INTRINSIC :: ISO_C_BINDING USE H5GLOBAL + PRIVATE h5dread_vl_integer, h5dread_vl_real, h5dread_vl_string + PRIVATE h5dwrite_vl_integer, h5dwrite_vl_real, h5dwrite_vl_string + PRIVATE h5dwrite_reference_obj, h5dwrite_reference_dsetreg, h5dwrite_char_scalar, h5dwrite_ptr + PRIVATE h5dread_reference_obj, h5dread_reference_dsetreg, h5dread_char_scalar, h5dread_ptr + PRIVATE h5dfill_integer, h5dfill_c_float, h5dfill_c_double, h5dfill_char +#if H5_FORTRAN_C_LONG_DOUBLE_IS_UNIQUE!=0 + PRIVATE h5dfill_c_long_double +#endif + INTERFACE h5dextend_f MODULE PROCEDURE h5dset_extent_f END INTERFACE diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index afb9136..455c72e 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -42,6 +42,14 @@ MODULE H5P USE H5GLOBAL USE H5fortkit + PRIVATE h5pset_fapl_multi_l, h5pset_fapl_multi_s + PRIVATE h5pset_fill_value_integer, h5pset_fill_value_char, h5pset_fill_value_ptr + PRIVATE h5pget_fill_value_integer, h5pget_fill_value_char, h5pget_fill_value_ptr + PRIVATE h5pset_integer, h5pset_char, h5pset_ptr + PRIVATE h5pget_integer, h5pget_char, h5pget_ptr + PRIVATE h5pregister_integer, h5pregister_ptr + PRIVATE h5pinsert_integer, h5pinsert_char, h5pinsert_ptr + INTERFACE h5pset_fapl_multi_f MODULE PROCEDURE h5pset_fapl_multi_l MODULE PROCEDURE h5pset_fapl_multi_s diff --git a/fortran/src/H5Rff.F90 b/fortran/src/H5Rff.F90 index f5a9c6e..6c2ba28 100644 --- a/fortran/src/H5Rff.F90 +++ b/fortran/src/H5Rff.F90 @@ -56,6 +56,12 @@ MODULE H5R ! END TYPE ! + PRIVATE h5rget_object_type_obj_f + PRIVATE h5rget_region_region_f, h5rget_region_ptr_f + PRIVATE h5rcreate_object_f, h5rcreate_region_f, h5rcreate_ptr_f + PRIVATE h5rdereference_object_f, h5rdereference_region_f, h5rdereference_ptr_f + PRIVATE h5rget_name_object_f, h5rget_name_region_f, h5rget_name_ptr_f + INTERFACE h5rget_object_type_f MODULE PROCEDURE h5rget_object_type_obj_f diff --git a/fortran/src/H5Tff.F90 b/fortran/src/H5Tff.F90 index b63c61d..46c8f39 100644 --- a/fortran/src/H5Tff.F90 +++ b/fortran/src/H5Tff.F90 @@ -41,6 +41,8 @@ MODULE H5T USE H5GLOBAL IMPLICIT NONE + PRIVATE h5tenum_insert_f03, h5tenum_insert_f90 + !****t* H5T/hvl_t ! Fortran2003 Derived Type: TYPE hvl_t diff --git a/fortran/src/H5_buildiface.F90 b/fortran/src/H5_buildiface.F90 index d4ebdd3..f793b7f 100644 --- a/fortran/src/H5_buildiface.F90 +++ b/fortran/src/H5_buildiface.F90 @@ -135,162 +135,268 @@ PROGRAM H5_buildiface WRITE(11,'(A)') "MODULE H5_GEN" - WRITE(11,'(A)') ' USE, INTRINSIC :: ISO_C_BINDING' - WRITE(11,'(A)') ' USE H5GLOBAL' + WRITE(11,'(2X,A)') 'USE, INTRINSIC :: ISO_C_BINDING' + WRITE(11,'(2X,A)') 'USE H5GLOBAL' + + WRITE(11,'(2X,A)') 'USE H5A' + WRITE(11,'(2X,A)') 'USE H5D' + WRITE(11,'(2X,A)') 'USE H5P' + WRITE(11,'(2X,A)') 'IMPLICIT NONE' + +!****************************** +! DECLARE PRIVATE INTERFACES +!****************************** + + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5awrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5awrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5awrite_ckind_rank"//chr_rank(k) + ENDDO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5aread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5aread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5aread_ckind_rank"//chr_rank(k) + ENDDO + + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dwrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dwrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dwrite_ckind_rank"//chr_rank(k) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dread_ckind_rank"//chr_rank(k) + ENDDO + + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pset_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pget_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pset_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pget_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pregister_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pinsert_kind_"//TRIM(ADJUSTL(chr2)) + END DO - WRITE(11,'(A)') ' USE H5A' - WRITE(11,'(A)') ' USE H5D' - WRITE(11,'(A)') ' USE H5P' - WRITE(11,'(A)') ' IMPLICIT NONE' !*************** ! H5A INTERFACES !*************** ! ! H5Awrite_f ! - WRITE(11,'(A)') " INTERFACE h5awrite_f" + WRITE(11,'(2X,A)') "INTERFACE h5awrite_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5awrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5awrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5awrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5awrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5awrite_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5awrite_ckind_rank"//chr_rank(k) ENDDO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Aread_f - WRITE(11,'(A)') " INTERFACE h5aread_f" + WRITE(11,'(2X,A)') "INTERFACE h5aread_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5aread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5aread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5aread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5aread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5aread_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5aread_ckind_rank"//chr_rank(k) ENDDO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" !*************** ! H5D INTERFACES !*************** ! ! H5Dwrite_f - WRITE(11,'(A)') " INTERFACE h5dwrite_f" + WRITE(11,'(2X,A)') "INTERFACE h5dwrite_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dwrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dwrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dwrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dwrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dwrite_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dwrite_ckind_rank"//chr_rank(k) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Dread_f - WRITE(11,'(A)') " INTERFACE h5dread_f" + WRITE(11,'(2X,A)') "INTERFACE h5dread_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dread_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dread_ckind_rank"//chr_rank(k) ENDDO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" !*************** ! H5P INTERFACES !*************** ! ! H5Pset_fill_value_f - WRITE(11,'(A)') " INTERFACE h5pset_fill_value_f" + WRITE(11,'(2X,A)') "INTERFACE h5pset_fill_value_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pset_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pset_fill_value_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pget_fill_value_f - WRITE(11,'(A)') " INTERFACE h5pget_fill_value_f" + WRITE(11,'(2X,A)') "INTERFACE h5pget_fill_value_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pget_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pget_fill_value_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pset_f - WRITE(11,'(A)') " INTERFACE h5pset_f" + WRITE(11,'(2X,A)') "INTERFACE h5pset_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pset_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pset_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pget_f - WRITE(11,'(A)') " INTERFACE h5pget_f" + WRITE(11,'(2X,A)') "INTERFACE h5pget_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pget_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pget_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pregister_f - WRITE(11,'(A)') " INTERFACE h5pregister_f" + WRITE(11,'(2X,A)') "INTERFACE h5pregister_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pregister_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pregister_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pinsert_f - WRITE(11,'(A)') " INTERFACE h5pinsert_f" + WRITE(11,'(2X,A)') "INTERFACE h5pinsert_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pinsert_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pinsert_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" WRITE(11,'(A)') 'CONTAINS' diff --git a/java/test/TestH5Z.java b/java/test/TestH5Z.java index 31adf22..f3049ed 100644 --- a/java/test/TestH5Z.java +++ b/java/test/TestH5Z.java @@ -41,8 +41,8 @@ public class TestH5Z { @Test public void testH5Zfilter_avail() { try { - int filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_DEFLATE); - assertTrue("H5.H5Zfilter_avail_DEFLATE", filter_found > 0); + int filter_found; + filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_FLETCHER32); assertTrue("H5.H5Zfilter_avail_FLETCHER32", filter_found > 0); filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_NBIT); @@ -51,8 +51,12 @@ public class TestH5Z { assertTrue("H5.H5Zfilter_avail_SCALEOFFSET", filter_found > 0); filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SHUFFLE); assertTrue("H5.H5Zfilter_avail_SHUFFLE", filter_found > 0); -// filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SZIP); -// assertTrue("H5.H5Zfilter_avail_SZIP", filter_found > 0); + + // Just make sure H5Zfilter_avail() doesn't fail with szip/zlib + // since there is no way for us to determine if they should be present + // or not. + filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_DEFLATE); + filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SZIP); } catch (Throwable err) { err.printStackTrace(); @@ -63,9 +67,8 @@ public class TestH5Z { @Test public void testH5Zget_filter_info() { try { - int filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_DEFLATE); - assertTrue("H5.H5Zget_filter_info_DEFLATE_DECODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); - assertTrue("H5.H5Zget_filter_info_DEFLATE_ENCODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); + int filter_flag; + filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_FLETCHER32); assertTrue("H5.H5Zget_filter_info_FLETCHER32_DECODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); assertTrue("H5.H5Zget_filter_info_FLETCHER32_ENCODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); @@ -78,9 +81,19 @@ public class TestH5Z { filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_SHUFFLE); assertTrue("H5.H5Zget_filter_info_DECODE_SHUFFLE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); assertTrue("H5.H5Zget_filter_info_ENCODE_SHUFFLE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); -// filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_SZIP); -// assertTrue("H5.H5Zget_filter_info_DECODE_SZIP_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); -// assertTrue("H5.H5Zget_filter_info_ENCODE_SZIP_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); + + if(1 == H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_DEFLATE)) { + filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_DEFLATE); + assertTrue("H5.H5Zget_filter_info_DEFLATE_DECODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); + assertTrue("H5.H5Zget_filter_info_DEFLATE_ENCODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); + } + + if(1 == H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SZIP)) { + filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_SZIP); + // Decode should always be available, but we have no way of determining + // if encode is so don't assert on that. + assertTrue("H5.H5Zget_filter_info_DECODE_SZIP_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); + } } catch (Throwable err) { err.printStackTrace(); @@ -96,3 +109,4 @@ public class TestH5Z { H5.H5Zunregister(HDF5Constants.H5Z_FILTER_SHUFFLE); } } + diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index dcc9892..4661b59 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -170,7 +170,7 @@ New Features (DER - 2018/10/26, HDFFV-10614) - Parallel Library: + Parallel Library: ----------------- - @@ -233,7 +233,17 @@ New Features Tools: ------ - - + - The h5repart -family-to-sec2 argument was changed to -family-to-single + + In order to better support other single-file VFDs which could work with + h5repart, the -family-to-sec2 argument was renamed to -family-to-single. + This is just a name change and the functionality of the argument has not + changed. + + The -family-to-sec2 argument has been kept for backwards-compatibility. + This argument should be considered deprecated. + + (DER - 2018/11/14, HDFFV-10633) High-Level APIs: --------------- diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 22dc05a..cb6b925 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -949,7 +949,10 @@ H5D__chunk_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id) /* Initial scaled dimension sizes */ if(dset->shared->layout.u.chunk.dim[u] == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "chunk size must be > 0, dim = %u ", u) - rdcc->scaled_dims[u] = dset->shared->curr_dims[u] / dset->shared->layout.u.chunk.dim[u]; + + /* Round up to the next integer # of chunks, to accommodate partial chunks */ + rdcc->scaled_dims[u] = (dset->shared->curr_dims[u] + dset->shared->layout.u.chunk.dim[u] - 1) / + dset->shared->layout.u.chunk.dim[u]; if( !(scaled_power2up = H5VM_power2up(rdcc->scaled_dims[u])) ) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") @@ -2799,6 +2802,7 @@ H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled) hsize_t val; /* Intermediate value */ unsigned ndims = shared->ndims; /* Rank of dataset */ unsigned ret = 0; /* Value to return */ + unsigned u; /* Local index variable */ FUNC_ENTER_STATIC_NOERR @@ -2809,17 +2813,11 @@ H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled) /* If the fastest changing dimension doesn't have enough entropy, use * other dimensions too */ - if(ndims > 1 && shared->cache.chunk.scaled_dims[ndims - 1] <= shared->cache.chunk.nslots) { - unsigned u; /* Local index variable */ - - val = scaled[0]; - for(u = 1; u < ndims; u++) { - val <<= shared->cache.chunk.scaled_encode_bits[u]; - val ^= scaled[u]; - } /* end for */ - } /* end if */ - else - val = scaled[ndims - 1]; + val = scaled[0]; + for(u = 1; u < ndims; u++) { + val <<= shared->cache.chunk.scaled_encode_bits[u]; + val ^= scaled[u]; + } /* end for */ /* Modulo value against the number of array slots */ ret = (unsigned)(val % shared->cache.chunk.nslots); diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 7c2297a..3dcd7f5 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -117,7 +117,7 @@ typedef struct H5FD_log_t { * Whether to eliminate the family driver info and convert this file to * a single file */ - hbool_t fam_to_sec2; + hbool_t fam_to_single; /* Fields for tracking I/O operations */ unsigned char *nread; /* Number of reads from a file location */ @@ -635,13 +635,13 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from - * family to sec2 while using h5repart, this private property should be set so that - * in the later step, the library can ignore the family driver information saved - * in the superblock. + * family to one that uses single files (sec2, etc.) while using h5repart, this + * private property should be set so that in the later step, the library can ignore + * the family driver information saved in the superblock. */ - if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SEC2_NAME) > 0) - if(H5P_get(plist, H5F_ACS_FAMILY_TO_SEC2_NAME, &file->fam_to_sec2) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to sec2") + if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME) > 0) + if(H5P_get(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME, &file->fam_to_single) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to single") } /* end if */ /* Set return value */ @@ -900,7 +900,7 @@ H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) *flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default VFD */ /* Check for flags that are set by h5repart */ - if(file && file->fam_to_sec2) + if(file && file->fam_to_single) *flags |= H5FD_FEAT_IGNORE_DRVRINFO; /* Ignore the driver info when file is opened (which eliminates it) */ } /* end if */ diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 32aff0f..06c008d 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -94,7 +94,7 @@ typedef struct H5FD_sec2_t { * Whether to eliminate the family driver info and convert this file to * a single file. */ - hbool_t fam_to_sec2; + hbool_t fam_to_single; } H5FD_sec2_t; @@ -386,13 +386,13 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from - * family to sec2 while using h5repart, this private property should be set so that - * in the later step, the library can ignore the family driver information saved - * in the superblock. + * family to one that uses single files (sec2, etc.) while using h5repart, this + * private property should be set so that in the later step, the library can ignore + * the family driver information saved in the superblock. */ - if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SEC2_NAME) > 0) - if(H5P_get(plist, H5F_ACS_FAMILY_TO_SEC2_NAME, &file->fam_to_sec2) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to sec2") + if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME) > 0) + if(H5P_get(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME, &file->fam_to_single) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to single") } /* end if */ /* Set return value */ @@ -532,7 +532,7 @@ H5FD_sec2_query(const H5FD_t *_file, unsigned long *flags /* out */) *flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default VFD */ /* Check for flags that are set by h5repart */ - if(file && file->fam_to_sec2) + if(file && file->fam_to_single) *flags |= H5FD_FEAT_IGNORE_DRVRINFO; /* Ignore the driver info when file is opened (which eliminates it) */ } /* end if */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 00aefa0..83513a5 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -483,7 +483,7 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_CLOSE_DEGREE_NAME "close_degree" /* File close degree */ #define H5F_ACS_FAMILY_OFFSET_NAME "family_offset" /* Offset position in file for family file driver */ #define H5F_ACS_FAMILY_NEWSIZE_NAME "family_newsize" /* New member size of family driver. (private property only used by h5repart) */ -#define H5F_ACS_FAMILY_TO_SEC2_NAME "family_to_sec2" /* Whether to convert family to sec2 driver. (private property only used by h5repart) */ +#define H5F_ACS_FAMILY_TO_SINGLE_NAME "family_to_single" /* Whether to convert family to a single-file driver. (private property only used by h5repart) */ #define H5F_ACS_MULTI_TYPE_NAME "multi_type" /* Data type in multi file driver */ #define H5F_ACS_LIBVER_LOW_BOUND_NAME "libver_low_bound" /* 'low' bound of library format versions */ #define H5F_ACS_LIBVER_HIGH_BOUND_NAME "libver_high_bound" /* 'high' bound of library format versions */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index f4dbd0a..a792e57 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -135,10 +135,11 @@ * property only used by h5repart */ #define H5F_ACS_FAMILY_NEWSIZE_SIZE sizeof(hsize_t) #define H5F_ACS_FAMILY_NEWSIZE_DEF 0 -/* Definition for whether to convert family to sec2 driver. It's private - * property only used by h5repart */ -#define H5F_ACS_FAMILY_TO_SEC2_SIZE sizeof(hbool_t) -#define H5F_ACS_FAMILY_TO_SEC2_DEF FALSE +/* Definition for whether to convert family to a single-file driver. + * It's a private property only used by h5repart. + */ +#define H5F_ACS_FAMILY_TO_SINGLE_SIZE sizeof(hbool_t) +#define H5F_ACS_FAMILY_TO_SINGLE_DEF FALSE /* Definition for data type in multi file driver */ #define H5F_ACS_MULTI_TYPE_SIZE sizeof(H5FD_mem_t) #define H5F_ACS_MULTI_TYPE_DEF H5FD_MEM_DEFAULT @@ -391,7 +392,7 @@ static const unsigned H5F_def_gc_ref_g = H5F_ACS_GARBG_COLCT_REF_DEF; static const H5F_close_degree_t H5F_def_close_degree_g = H5F_CLOSE_DEGREE_DEF; /* Default file close degree */ static const hsize_t H5F_def_family_offset_g = H5F_ACS_FAMILY_OFFSET_DEF; /* Default offset for family VFD */ static const hsize_t H5F_def_family_newsize_g = H5F_ACS_FAMILY_NEWSIZE_DEF; /* Default size of new files for family VFD */ -static const hbool_t H5F_def_family_to_sec2_g = H5F_ACS_FAMILY_TO_SEC2_DEF; /* Default ?? for family VFD */ +static const hbool_t H5F_def_family_to_single_g = H5F_ACS_FAMILY_TO_SINGLE_DEF; /* Default ?? for family VFD */ static const H5FD_mem_t H5F_def_mem_type_g = H5F_ACS_MULTI_TYPE_DEF; /* Default file space type for multi VFD */ static const H5F_libver_t H5F_def_libver_low_bound_g = H5F_ACS_LIBVER_LOW_BOUND_DEF; /* Default setting for "low" bound of format version */ @@ -527,9 +528,9 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - /* Register the private property of whether convert family to sec2 driver. It's used by h5repart only. */ + /* Register the private property of whether convert family to a single-file driver. It's used by h5repart only. */ /* (Note: this property should not have an encode/decode callback -QAK) */ - if(H5P__register_real(pclass, H5F_ACS_FAMILY_TO_SEC2_NAME, H5F_ACS_FAMILY_TO_SEC2_SIZE, &H5F_def_family_to_sec2_g, + if(H5P__register_real(pclass, H5F_ACS_FAMILY_TO_SINGLE_NAME, H5F_ACS_FAMILY_TO_SINGLE_SIZE, &H5F_def_family_to_single_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") diff --git a/tools/src/misc/h5repart.c b/tools/src/misc/h5repart.c index cdc554f..2f4d93e 100644 --- a/tools/src/misc/h5repart.c +++ b/tools/src/misc/h5repart.c @@ -29,11 +29,12 @@ #define NAMELEN 4096 #define GB *1024*1024*1024 -/*Make these 2 private properties(defined in H5Fprivate.h) available to h5repart. - *The first one updates the member file size in the superblock. The second one - *change file driver from family to sec2. */ +/* Make these 2 private properties(defined in H5Fprivate.h) available to h5repart. + * The first one updates the member file size in the superblock. The second one + * change file driver from family to a single file driver. + */ #define H5F_ACS_FAMILY_NEWSIZE_NAME "family_newsize" -#define H5F_ACS_FAMILY_TO_SEC2_NAME "family_to_sec2" +#define H5F_ACS_FAMILY_TO_SINGLE_NAME "family_to_single" /*------------------------------------------------------------------------- @@ -53,13 +54,14 @@ static void usage (const char *progname) { - fprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] [-family_to_sec2] SRC DST\n", + fprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] [-family_to_sec2|-family_to_single] SRC DST\n", progname); fprintf(stderr, " -v Produce verbose output\n"); fprintf(stderr, " -V Print a version number and exit\n"); fprintf(stderr, " -b N The I/O block size, defaults to 1kB\n"); fprintf(stderr, " -m N The destination member size or 1GB\n"); - fprintf(stderr, " -family_to_sec2 Change file driver from family to sec2\n"); + fprintf(stderr, " -family_to_sec2 Deprecated version of -family_to_single (below)\n"); + fprintf(stderr, " -family_to_single Change file driver from family to the default single-file VFD (windows or sec2)\n"); fprintf(stderr, " SRC The name of the source file\n"); fprintf(stderr, " DST The name of the destination files\n"); fprintf(stderr, "Sizes may be suffixed with `g' for GB, `m' for MB or " @@ -186,7 +188,7 @@ main (int argc, char *argv[]) hid_t fapl; /*file access property list */ hid_t file; hsize_t hdsize; /*destination logical memb size */ - hbool_t family_to_sec2=FALSE; /*change family to sec2 driver? */ + hbool_t family_to_single = FALSE; /*change family to single file driver? */ /* * Get the program name from argv[0]. Use only the last component. @@ -206,7 +208,10 @@ main (int argc, char *argv[]) prog_name, H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE); exit(EXIT_SUCCESS); } else if (!strcmp (argv[argno], "-family_to_sec2")) { - family_to_sec2 = TRUE; + family_to_single = TRUE; + argno++; + } else if (!strcmp (argv[argno], "-family_to_single")) { + family_to_single = TRUE; argno++; } else if ('b'==argv[argno][1]) { blk_size = (size_t)get_size (prog_name, &argno, argc, argv); @@ -422,11 +427,12 @@ main (int argc, char *argv[]) exit (EXIT_FAILURE); } - if(family_to_sec2) { - /* The user wants to change file driver from family to sec2. Open the file - * with sec2 driver. This property signals the library to ignore the family - * driver information saved in the superblock. */ - if(H5Pset(fapl, H5F_ACS_FAMILY_TO_SEC2_NAME, &family_to_sec2) < 0) { + if(family_to_single) { + /* The user wants to change file driver from family to a single-file VFD. + * Open the file with the sec2, windows, etc. driver. This property signals + * the library to ignore the family driver information saved in the superblock. + */ + if(H5Pset(fapl, H5F_ACS_FAMILY_TO_SINGLE_NAME, &family_to_single) < 0) { perror ("H5Pset"); exit (EXIT_FAILURE); } @@ -449,13 +455,14 @@ main (int argc, char *argv[]) /* If the new file is a family file, try to open file for "read and write" to * flush metadata. Flushing metadata will update the superblock to the new - * member size. If the original file is a family file and the new file is a sec2 - * file, the property FAMILY_TO_SEC2 will signal the library to switch to sec2 - * driver when the new file is opened. If the original file is a sec2 file and the - * new file can only be a sec2 file, reopen the new file should fail. There's - * nothing to do in this case. */ + * member size. If the original file is a family file and the new file is a single + * file, the property FAMILY_TO_SINGLE will signal the library to switch to default + * single-file driver when the new file is opened. If the original file is a single + * file and the new file can only be a single file, reopen the new file should fail. + * There's nothing to do in this case. + */ H5E_BEGIN_TRY { - file=H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl); + file = H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl); } H5E_END_TRY; if(file>=0) { diff --git a/tools/test/misc/CMakeTestsRepart.cmake b/tools/test/misc/CMakeTestsRepart.cmake index 37da903..830de11 100644 --- a/tools/test/misc/CMakeTestsRepart.cmake +++ b/tools/test/misc/CMakeTestsRepart.cmake @@ -68,6 +68,7 @@ scd_family00001.h5 scd_family00002.h5 scd_family00003.h5 + family_to_single.h5 family_to_sec2.h5 ) if (NOT "${last_test}" STREQUAL "") @@ -84,6 +85,10 @@ set_tests_properties (H5REPART-h5repart_5K PROPERTIES DEPENDS H5REPART-clearall-objects) # convert family file to sec2 file of 20,000 bytes + add_test (NAME H5REPART-h5repart_single COMMAND $<TARGET_FILE:h5repart> -m 20000 -family_to_single family_file%05d.h5 family_to_single.h5) + set_tests_properties (H5REPART-h5repart_single PROPERTIES DEPENDS H5REPART-clearall-objects) + + # convert family file to sec2 file of 20,000 bytes (old argument) add_test (NAME H5REPART-h5repart_sec2 COMMAND $<TARGET_FILE:h5repart> -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5) set_tests_properties (H5REPART-h5repart_sec2 PROPERTIES DEPENDS H5REPART-clearall-objects) diff --git a/tools/test/misc/repart_test.c b/tools/test/misc/repart_test.c index 4016ee8..b8e0292 100644 --- a/tools/test/misc/repart_test.c +++ b/tools/test/misc/repart_test.c @@ -26,12 +26,13 @@ const char *FILENAME[] = { "fst_family%05d.h5", "scd_family%05d.h5", + "family_to_single.h5", "family_to_sec2.h5", NULL }; herr_t test_family_h5repart_opens(void); -herr_t test_sec2_h5repart_opens(void); +herr_t test_single_h5repart_opens(void); /*------------------------------------------------------------------------- @@ -90,23 +91,28 @@ error: /*------------------------------------------------------------------------- - * Function: test_sec2_h5repart_opens + * Function: test_single_h5repart_opens * - * Purpose: Tries to reopen a sec2 file. + * Purpose: Tries to reopen a single file. * * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ herr_t -test_sec2_h5repart_opens(void) +test_single_h5repart_opens(void) { hid_t fid = -1; - /* open the sec2 file */ + /* open the single file */ if ((fid = H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT)) < 0) goto error; + if (H5Fclose(fid) < 0) + goto error; + /* open the single file (created using the old argument) */ + if ((fid = H5Fopen(FILENAME[3], H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + goto error; if (H5Fclose(fid) < 0) goto error; @@ -119,7 +125,7 @@ error: return FAIL; -} /* end test_sec2_h5repart_opens() */ +} /* end test_single_h5repart_opens() */ /*------------------------------------------------------------------------- @@ -137,7 +143,7 @@ main(void) int nerrors = 0; nerrors += test_family_h5repart_opens() < 0 ? 1 : 0; - nerrors += test_sec2_h5repart_opens() < 0 ? 1 : 0; + nerrors += test_single_h5repart_opens() < 0 ? 1 : 0; if (nerrors) goto error; @@ -150,3 +156,4 @@ error: nerrors, 1 == nerrors ? "" : "S"); HDexit(EXIT_FAILURE); } /* end main() */ + diff --git a/tools/test/misc/testh5repart.sh.in b/tools/test/misc/testh5repart.sh.in index 594d37e..143f607 100644 --- a/tools/test/misc/testh5repart.sh.in +++ b/tools/test/misc/testh5repart.sh.in @@ -171,7 +171,9 @@ COPY_TESTFILES_TO_TESTDIR TOOLTEST -m 20000 family_file%05d.h5 fst_family%05d.h5 # repartition family member size to 5 KB. TOOLTEST -m 5k family_file%05d.h5 scd_family%05d.h5 -# convert family file to sec2 file of 20,000 bytes +# convert family file to single file of 20,000 bytes +TOOLTEST -m 20000 -family_to_single family_file%05d.h5 family_to_single.h5 +# convert family file to single file of 20,000 bytes (old argument) TOOLTEST -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5 # test the output files repartitioned above. @@ -183,7 +185,7 @@ CLEAN_TESTFILES_AND_TESTDIR if test -z "$HDF5_NOCLEANUP"; then cd $actual_dir - rm -f fst_family*.h5 scd_family*.h5 family_to_sec2.h5 + rm -f fst_family*.h5 scd_family*.h5 family_to_single.h5 family_to_sec2.h5 fi if test $nerrors -eq 0 ; then diff --git a/tools/test/perform/CMakeLists.txt b/tools/test/perform/CMakeLists.txt index fa41608..14abdec 100644 --- a/tools/test/perform/CMakeLists.txt +++ b/tools/test/perform/CMakeLists.txt @@ -51,6 +51,16 @@ TARGET_C_PROPERTIES (iopipe STATIC) target_link_libraries (iopipe PRIVATE ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) set_target_properties (iopipe PROPERTIES FOLDER perform) +#-- Adding test for chunk_cache +set (chunk_cache_SOURCES + ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/chunk_cache.c +) +add_executable (chunk_cache ${chunk_cache_SOURCES}) +target_include_directories(chunk_cache PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +TARGET_C_PROPERTIES (chunk_cache STATIC) +target_link_libraries (chunk_cache PRIVATE ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) +set_target_properties (chunk_cache PROPERTIES FOLDER perform) + #-- Adding test for overhead set (overhead_SOURCES ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/overhead.c diff --git a/tools/test/perform/Makefile.am b/tools/test/perform/Makefile.am index 5a89a66..39800d7 100644 --- a/tools/test/perform/Makefile.am +++ b/tools/test/perform/Makefile.am @@ -50,12 +50,12 @@ if BUILD_PARALLEL_CONDITIONAL TEST_PROG_PARA=h5perf perf endif # Serial test programs. -TEST_PROG = iopipe chunk overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) +TEST_PROG = iopipe chunk chunk_cache overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) # check_PROGRAMS will be built but not installed. Do not any executable # that is in bin_PROGRAMS already. Otherwise, it will be removed twice in # "make clean" and some systems, e.g., AIX, do not like it. -check_PROGRAMS= iopipe chunk overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf +check_PROGRAMS= iopipe chunk chunk_cache overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf h5perf_SOURCES=pio_perf.c pio_engine.c h5perf_serial_SOURCES=sio_perf.c sio_engine.c diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c new file mode 100644 index 0000000..429f70a --- /dev/null +++ b/tools/test/perform/chunk_cache.c @@ -0,0 +1,405 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: check the performance of chunk cache in these two cases (HDFFV-10601): + * 1. partial chunks exist along any dimension. + * 2. number of slots in chunk cache is smaller than the number of chunks + * in the fastest-growing dimension. + */ +#include "hdf5.h" +#include "H5private.h" + +#define FILENAME "chunk_cache_perf.h5" + +#define RANK 2 + +#define DSET1_NAME "partial_chunks" +#define DSET1_DIM1 9 * 1000 +#define DSET1_DIM2 9 +#define CHUNK1_DIM1 2 * 1000 +#define CHUNK1_DIM2 2 + +#define DSET2_NAME "hash_value" +#define DSET2_DIM1 300 +#define DSET2_DIM2 600 +#define CHUNK2_DIM1 100 +#define CHUNK2_DIM2 100 + +#define RDCC_NSLOTS 5 +#define RDCC_NBYTES 1024 * 1024 * 10 +#define RDCC_W0 0.75F + +#define FILTER_COUNTER 306 +static size_t nbytes_global; + +typedef struct test_time_t { + long tv_sec; + long tv_usec; +} test_time_t; + +/* Local function prototypes for the dummy filter */ +static size_t +counter (unsigned flags, size_t cd_nelmts, + const unsigned *cd_values, size_t nbytes, + size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_COUNTER[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + FILTER_COUNTER, /* Filter id number */ + 1, 1, /* Encoding and decoding enabled */ + "counter", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + counter, /* The actual filter function */ +}}; + +/*------------------------------------------------------------------------- + * Count number of bytes but don't do anything else. Keep + * track of the data of chunks being read from file into memory. + */ +static size_t +counter (unsigned flags, size_t cd_nelmts, + const unsigned *cd_values, size_t nbytes, + size_t *buf_size, void **buf) +{ + nbytes_global += nbytes; + return nbytes; +} + +/*---------------------------------------------------------------------------*/ +double retrieve_time(void) +{ +#ifdef H5_HAVE_GETTIMEOFDAY + struct timeval t; + HDgettimeofday(&t, NULL); + return ((double)t.tv_sec + (double)t.tv_usec / 1000000); +#else + return 0.0; +#endif +} + +/*---------------------------------------------------------------------------*/ +static void +cleanup (void) +{ + if (!getenv ("HDF5_NOCLEANUP")) { + remove (FILENAME); + } +} + +/*------------------------------------------------------------------------------- + * Create a chunked dataset with partial chunks along either dimensions: + * dataset dimension: 9000 x 9 + * chunk dimension: 2000 x 2 + */ +static int create_dset1(hid_t file) +{ + hid_t dataspace, dataset; + hid_t dcpl; + hsize_t dims[RANK] = {DSET1_DIM1, DSET1_DIM2}; + hsize_t chunk_dims[RANK] = {CHUNK1_DIM1, CHUNK1_DIM2}; + int data[DSET1_DIM1][DSET1_DIM2]; /* data for writing */ + int i, j; + + /* Create the data space. */ + if((dataspace = H5Screate_simple (RANK, dims, NULL)) < 0) + goto error; + + /* Modify dataset creation properties, i.e. enable chunking */ + if((dcpl = H5Pcreate (H5P_DATASET_CREATE)) < 0) + goto error; + if(H5Pset_chunk (dcpl, RANK, chunk_dims) < 0) + goto error; + + /* Set the dummy filter simply for counting the number of bytes being read into the memory */ + if(H5Zregister(H5Z_COUNTER) < 0) + goto error; + + if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0) + goto error; + + /* Create a new dataset within the file using chunk creation properties. */ + if((dataset = H5Dcreate2 (file, DSET1_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + for (i = 0; i < DSET1_DIM1; i++) + for (j = 0; j < DSET1_DIM2; j++) + data[i][j] = i+j; + + /* Write data to dataset */ + if(H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, data) < 0) + goto error; + + /* Close resources */ + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + } H5E_END_TRY; + + return 1; +} + +/*--------------------------------------------------------------------------- + * Create a chunked dataset for testing hash values: + * dataset dimensions: 300 x 600 + * chunk dimensions: 100 x 100 + */ +static int create_dset2(hid_t file) +{ + hid_t dataspace, dataset; + hid_t dcpl; + hsize_t dims[RANK] = {DSET2_DIM1, DSET2_DIM2}; + hsize_t chunk_dims[RANK] = {CHUNK2_DIM1, CHUNK2_DIM2}; + int data[DSET2_DIM1][DSET2_DIM2]; /* data for writing */ + int i, j; + + /* Create the data space. */ + if((dataspace = H5Screate_simple (RANK, dims, NULL)) < 0) + goto error; + + /* Modify dataset creation properties, i.e. enable chunking */ + if((dcpl = H5Pcreate (H5P_DATASET_CREATE)) < 0) + goto error; + if(H5Pset_chunk (dcpl, RANK, chunk_dims) < 0) + goto error; + + /* Set the dummy filter simply for counting the number of bytes being read into the memory */ + if(H5Zregister(H5Z_COUNTER) < 0) + goto error; + if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0) + goto error; + + /* Create a new dataset within the file using chunk creation properties. */ + if((dataset = H5Dcreate2 (file, DSET2_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + for (i = 0; i < DSET2_DIM1; i++) + for (j = 0; j < DSET2_DIM2; j++) + data[i][j] = i+j; + + /* Write data to dataset */ + if(H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, data) < 0) + goto error; + + /* Close resources */ + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + } H5E_END_TRY; + + return 1; +} +/*--------------------------------------------------------------------------- + * Check the performance of the chunk cache when partial chunks exist + * along the dataset dimensions. + */ +static int check_partial_chunks_perf(hid_t file) +{ + hid_t dataset; + hid_t filespace; + hid_t memspace; + hid_t dapl; + + int rdata[DSET1_DIM2]; /* data for reading */ + int i; + + hsize_t row_rank = 1; + hsize_t row_dim[1] = {DSET1_DIM2}; + hsize_t start[RANK] = {0, 0}; + hsize_t count[RANK] = {1, DSET1_DIM2}; + double start_t, end_t; + + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0) + goto error; + + dataset = H5Dopen2 (file, DSET1_NAME, dapl); + + memspace = H5Screate_simple(row_rank, row_dim, NULL); + filespace = H5Dget_space(dataset); + + nbytes_global = 0; + + start_t = retrieve_time(); + + /* Read the data row by row */ + for(i = 0; i < DSET1_DIM1; i++) { + start[0] = i; + if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + goto error; + + if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata) < 0) + goto error; + } + + end_t = retrieve_time(); + +#ifdef H5_HAVE_GETTIMEOFDAY + printf("1. Partial chunks: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global); +#else + printf("1. Partial chunks: no total read time because gettimeofday() is not available; number of bytes being read from file is %lu\n", nbytes_global); +#endif + + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + + return 0; +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + } H5E_END_TRY; + return 1; +} + +/*--------------------------------------------------------------------------- + * Check the performance of chunk cache when the number of cache slots + * is smaller than the number of chunks along the fastest-growing + * dimension of the dataset. + */ +static int check_hash_value_perf(hid_t file) +{ + hid_t dataset; + hid_t filespace; + hid_t memspace; + hid_t dapl; + + int rdata[DSET2_DIM1]; /* data for reading */ + int i; + + hsize_t column_rank = 1; + hsize_t column_dim[1] = {DSET2_DIM1}; + hsize_t start[RANK] = {0, 0}; + hsize_t count[RANK] = {DSET2_DIM1, 1}; + double start_t, end_t; + + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0) + goto error; + + if((dataset = H5Dopen2 (file, DSET2_NAME, dapl)) < 0) + goto error; + if((memspace = H5Screate_simple(column_rank, column_dim, NULL)) < 0) + goto error; + if((filespace = H5Dget_space(dataset)) < 0) + goto error; + + nbytes_global = 0; + + start_t = retrieve_time(); + + /* Read the data column by column */ + for(i = 0; i < DSET2_DIM2; i++) { + start[1] = i; + if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + goto error; + + if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata) < 0) + goto error; + } + + end_t = retrieve_time(); + +#ifdef H5_HAVE_GETTIMEOFDAY + printf("2. Hash value: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global); +#else + printf("2. Hash value: no total read time because gettimeofday() is not available; number of bytes being read from file is %lu\n", nbytes_global); +#endif + + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + } H5E_END_TRY; + return 1; +} + +/*------------------------------------------------------------------------------------- + * Purpose: check the performance of chunk cache in these two cases (HDFFV-10601): + * 1. partial chunks exist along any dimension. + * 2. number of slots in chunk cache is smaller than the number of chunks + * in the fastest-growing dimension. + *-------------------------------------------------------------------------------------*/ +int +main (void) +{ + hid_t file; /* handles */ + int nerrors = 0; + + /* Create a new file. If file exists its contents will be overwritten. */ + if((file = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + nerrors += create_dset1(file); + nerrors += create_dset2(file); + + if(H5Fclose (file) < 0) + goto error; + + /* Re-open the file for testing performance. */ + if((file = H5Fopen (FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + goto error; + + nerrors += check_partial_chunks_perf(file); + nerrors += check_hash_value_perf(file); + + if(H5Fclose (file) < 0) + goto error; + + if (nerrors>0) goto error; + cleanup(); + return 0; + +error: + fprintf(stderr, "*** ERRORS DETECTED ***\n"); + return 1; +} |