From 6fa26d7e4939ac1b61b74bed7bca70055715be24 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Fri, 2 Nov 2018 14:53:35 -0500 Subject: HDFFV-10601 Issues with chunk cache hash value calcuation: 1. H5D__chunk_hash_val: When the number of chunks in the fastest changing dimension is larger than the number of slots in the hash table, H5D__chunk_hash_val abandons the normal hash value calculation algorithm and simply uses the scaled dimension. This will cause chunks a selection that cuts across chunks in dimensions other than the fastest changing to all have the same hash value, and they will therefore always evict each other from the cache, having an obvious major performance impact. Eliminated the check for the number of slots in this function and always use the full algorithm. 2. H5D__chunk_init: When the scaled dimensions (number of chunks in each dimension) are calculated in H5D__chunk_init, a simple divide ("/") operator is used with the dataset size in elements and the chunk size in elements. While this is fine when the dataset size is an exact multiple of the chunk size, in other cases, since "/" rounds down, it results in a scaled dimension one less than it should (it ignores the partial edge chunk). This has trickle down effects on hash value calculation that can cause excess hash value collisions and therefore performance issues. Changed the calculation to (((dataset_size - 1) / chunk_size) + 1). Tested the build with Autotool and CMake. --- src/H5Dchunk.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 22dc05a..7eeb866 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -949,7 +949,12 @@ H5D__chunk_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id) /* Initial scaled dimension sizes */ if(dset->shared->layout.u.chunk.dim[u] == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "chunk size must be > 0, dim = %u ", u) - rdcc->scaled_dims[u] = dset->shared->curr_dims[u] / dset->shared->layout.u.chunk.dim[u]; + + /* Make a special case when the dimension is 0 because (0 - 1) is a big number for unsigned integer */ + if(dset->shared->curr_dims[u] == 0) + rdcc->scaled_dims[u] = 0; + else + rdcc->scaled_dims[u] = ((dset->shared->curr_dims[u] - 1) / dset->shared->layout.u.chunk.dim[u]) + 1; if( !(scaled_power2up = H5VM_power2up(rdcc->scaled_dims[u])) ) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") @@ -2799,6 +2804,7 @@ H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled) hsize_t val; /* Intermediate value */ unsigned ndims = shared->ndims; /* Rank of dataset */ unsigned ret = 0; /* Value to return */ + unsigned u; /* Local index variable */ FUNC_ENTER_STATIC_NOERR @@ -2809,17 +2815,11 @@ H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled) /* If the fastest changing dimension doesn't have enough entropy, use * other dimensions too */ - if(ndims > 1 && shared->cache.chunk.scaled_dims[ndims - 1] <= shared->cache.chunk.nslots) { - unsigned u; /* Local index variable */ - - val = scaled[0]; - for(u = 1; u < ndims; u++) { - val <<= shared->cache.chunk.scaled_encode_bits[u]; - val ^= scaled[u]; - } /* end for */ - } /* end if */ - else - val = scaled[ndims - 1]; + val = scaled[0]; + for(u = 1; u < ndims; u++) { + val <<= shared->cache.chunk.scaled_encode_bits[u]; + val ^= scaled[u]; + } /* end for */ /* Modulo value against the number of array slots */ ret = (unsigned)(val % shared->cache.chunk.nslots); -- cgit v0.12 From 3a13bb72e8d33808eb3bfd68c2a6b4ae5153f3a0 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Thu, 8 Nov 2018 15:41:49 -0600 Subject: HDFFV-10601: I changed to a better way to calculate the number of chunks in a dataset. --- src/H5Dchunk.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 7eeb866..cb6b925 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -950,11 +950,9 @@ H5D__chunk_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id) if(dset->shared->layout.u.chunk.dim[u] == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "chunk size must be > 0, dim = %u ", u) - /* Make a special case when the dimension is 0 because (0 - 1) is a big number for unsigned integer */ - if(dset->shared->curr_dims[u] == 0) - rdcc->scaled_dims[u] = 0; - else - rdcc->scaled_dims[u] = ((dset->shared->curr_dims[u] - 1) / dset->shared->layout.u.chunk.dim[u]) + 1; + /* Round up to the next integer # of chunks, to accommodate partial chunks */ + rdcc->scaled_dims[u] = (dset->shared->curr_dims[u] + dset->shared->layout.u.chunk.dim[u] - 1) / + dset->shared->layout.u.chunk.dim[u]; if( !(scaled_power2up = H5VM_power2up(rdcc->scaled_dims[u])) ) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") -- cgit v0.12 From 4fbd6bb8a5b4f9048574da65c09a110ff8b83f34 Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Fri, 9 Nov 2018 11:24:42 -0600 Subject: HDFFV-10511 -- Make fortran specific subroutines names PRIVATE Made non-public APIs private. --- fortran/src/H5Aff.F90 | 3 + fortran/src/H5Dff.F90 | 9 ++ fortran/src/H5Pff.F90 | 8 ++ fortran/src/H5Rff.F90 | 6 ++ fortran/src/H5Tff.F90 | 2 + fortran/src/H5_buildiface.F90 | 194 ++++++++++++++++++++++++++++++++---------- 6 files changed, 178 insertions(+), 44 deletions(-) diff --git a/fortran/src/H5Aff.F90 b/fortran/src/H5Aff.F90 index 827b803..a728f2d 100644 --- a/fortran/src/H5Aff.F90 +++ b/fortran/src/H5Aff.F90 @@ -76,6 +76,9 @@ MODULE H5A USE H5GLOBAL + PRIVATE h5awrite_char_scalar, h5awrite_ptr + PRIVATE h5aread_char_scalar, h5aread_ptr + INTERFACE h5awrite_f MODULE PROCEDURE h5awrite_char_scalar ! This is the preferred way to call h5awrite diff --git a/fortran/src/H5Dff.F90 b/fortran/src/H5Dff.F90 index 3915f72..77f0a15 100644 --- a/fortran/src/H5Dff.F90 +++ b/fortran/src/H5Dff.F90 @@ -90,6 +90,15 @@ MODULE H5D USE, INTRINSIC :: ISO_C_BINDING USE H5GLOBAL + PRIVATE h5dread_vl_integer, h5dread_vl_real, h5dread_vl_string + PRIVATE h5dwrite_vl_integer, h5dwrite_vl_real, h5dwrite_vl_string + PRIVATE h5dwrite_reference_obj, h5dwrite_reference_dsetreg, h5dwrite_char_scalar, h5dwrite_ptr + PRIVATE h5dread_reference_obj, h5dread_reference_dsetreg, h5dread_char_scalar, h5dread_ptr + PRIVATE h5dfill_integer, h5dfill_c_float, h5dfill_c_double, h5dfill_char +#if H5_FORTRAN_C_LONG_DOUBLE_IS_UNIQUE!=0 + PRIVATE h5dfill_c_long_double +#endif + INTERFACE h5dextend_f MODULE PROCEDURE h5dset_extent_f END INTERFACE diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index afb9136..455c72e 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -42,6 +42,14 @@ MODULE H5P USE H5GLOBAL USE H5fortkit + PRIVATE h5pset_fapl_multi_l, h5pset_fapl_multi_s + PRIVATE h5pset_fill_value_integer, h5pset_fill_value_char, h5pset_fill_value_ptr + PRIVATE h5pget_fill_value_integer, h5pget_fill_value_char, h5pget_fill_value_ptr + PRIVATE h5pset_integer, h5pset_char, h5pset_ptr + PRIVATE h5pget_integer, h5pget_char, h5pget_ptr + PRIVATE h5pregister_integer, h5pregister_ptr + PRIVATE h5pinsert_integer, h5pinsert_char, h5pinsert_ptr + INTERFACE h5pset_fapl_multi_f MODULE PROCEDURE h5pset_fapl_multi_l MODULE PROCEDURE h5pset_fapl_multi_s diff --git a/fortran/src/H5Rff.F90 b/fortran/src/H5Rff.F90 index f5a9c6e..6c2ba28 100644 --- a/fortran/src/H5Rff.F90 +++ b/fortran/src/H5Rff.F90 @@ -56,6 +56,12 @@ MODULE H5R ! END TYPE ! + PRIVATE h5rget_object_type_obj_f + PRIVATE h5rget_region_region_f, h5rget_region_ptr_f + PRIVATE h5rcreate_object_f, h5rcreate_region_f, h5rcreate_ptr_f + PRIVATE h5rdereference_object_f, h5rdereference_region_f, h5rdereference_ptr_f + PRIVATE h5rget_name_object_f, h5rget_name_region_f, h5rget_name_ptr_f + INTERFACE h5rget_object_type_f MODULE PROCEDURE h5rget_object_type_obj_f diff --git a/fortran/src/H5Tff.F90 b/fortran/src/H5Tff.F90 index b63c61d..46c8f39 100644 --- a/fortran/src/H5Tff.F90 +++ b/fortran/src/H5Tff.F90 @@ -41,6 +41,8 @@ MODULE H5T USE H5GLOBAL IMPLICIT NONE + PRIVATE h5tenum_insert_f03, h5tenum_insert_f90 + !****t* H5T/hvl_t ! Fortran2003 Derived Type: TYPE hvl_t diff --git a/fortran/src/H5_buildiface.F90 b/fortran/src/H5_buildiface.F90 index d4ebdd3..e5d103d 100644 --- a/fortran/src/H5_buildiface.F90 +++ b/fortran/src/H5_buildiface.F90 @@ -135,162 +135,268 @@ PROGRAM H5_buildiface WRITE(11,'(A)') "MODULE H5_GEN" - WRITE(11,'(A)') ' USE, INTRINSIC :: ISO_C_BINDING' - WRITE(11,'(A)') ' USE H5GLOBAL' + WRITE(11,'(2X,A)') 'USE, INTRINSIC :: ISO_C_BINDING' + WRITE(11,'(2X,A)') 'USE H5GLOBAL' + + WRITE(11,'(2X,A)') 'USE H5A' + WRITE(11,'(2X,A)') 'USE H5D' + WRITE(11,'(2X,A)') 'USE H5P' + WRITE(11,'(2X,A)') 'IMPLICIT NONE' + +!****************************** +! DECLARE PRIVETE INTERFACES +!****************************** + + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5awrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5awrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5awrite_ckind_rank"//chr_rank(k) + ENDDO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5aread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5aread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5aread_ckind_rank"//chr_rank(k) + ENDDO + + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dwrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dwrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dwrite_ckind_rank"//chr_rank(k) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO i = 1, num_ikinds + j = ikind(i) + WRITE(chr2,'(I2)') j + DO k = 1, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + ENDDO + END DO + DO k = 2, 8 + WRITE(11,'(2X,A)') "PRIVATE h5dread_ckind_rank"//chr_rank(k) + ENDDO + + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pset_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pget_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pset_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pget_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pregister_kind_"//TRIM(ADJUSTL(chr2)) + END DO + DO i = 1, num_rkinds + j = rkind(i) + WRITE(chr2,'(I2)') j + WRITE(11,'(2X,A)') "PRIVATE h5pinsert_kind_"//TRIM(ADJUSTL(chr2)) + END DO - WRITE(11,'(A)') ' USE H5A' - WRITE(11,'(A)') ' USE H5D' - WRITE(11,'(A)') ' USE H5P' - WRITE(11,'(A)') ' IMPLICIT NONE' !*************** ! H5A INTERFACES !*************** ! ! H5Awrite_f ! - WRITE(11,'(A)') " INTERFACE h5awrite_f" + WRITE(11,'(2X,A)') "INTERFACE h5awrite_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5awrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5awrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5awrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5awrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5awrite_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5awrite_ckind_rank"//chr_rank(k) ENDDO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Aread_f - WRITE(11,'(A)') " INTERFACE h5aread_f" + WRITE(11,'(2X,A)') "INTERFACE h5aread_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5aread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5aread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5aread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5aread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5aread_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5aread_ckind_rank"//chr_rank(k) ENDDO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" !*************** ! H5D INTERFACES !*************** ! ! H5Dwrite_f - WRITE(11,'(A)') " INTERFACE h5dwrite_f" + WRITE(11,'(2X,A)') "INTERFACE h5dwrite_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dwrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dwrite_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dwrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dwrite_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dwrite_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dwrite_ckind_rank"//chr_rank(k) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Dread_f - WRITE(11,'(A)') " INTERFACE h5dread_f" + WRITE(11,'(2X,A)') "INTERFACE h5dread_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dread_rkind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO i = 1, num_ikinds j = ikind(i) WRITE(chr2,'(I2)') j DO k = 1, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dread_ikind_"//TRIM(ADJUSTL(chr2))//'_rank'//chr_rank(k) ENDDO END DO DO k = 2, 8 - WRITE(11,'(A)') " MODULE PROCEDURE h5dread_ckind_rank"//chr_rank(k) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5dread_ckind_rank"//chr_rank(k) ENDDO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" !*************** ! H5P INTERFACES !*************** ! ! H5Pset_fill_value_f - WRITE(11,'(A)') " INTERFACE h5pset_fill_value_f" + WRITE(11,'(2X,A)') "INTERFACE h5pset_fill_value_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pset_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pset_fill_value_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pget_fill_value_f - WRITE(11,'(A)') " INTERFACE h5pget_fill_value_f" + WRITE(11,'(2X,A)') "INTERFACE h5pget_fill_value_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pget_fill_value_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pget_fill_value_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pset_f - WRITE(11,'(A)') " INTERFACE h5pset_f" + WRITE(11,'(2X,A)') "INTERFACE h5pset_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pset_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pset_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pget_f - WRITE(11,'(A)') " INTERFACE h5pget_f" + WRITE(11,'(2X,A)') "INTERFACE h5pget_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pget_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pget_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pregister_f - WRITE(11,'(A)') " INTERFACE h5pregister_f" + WRITE(11,'(2X,A)') "INTERFACE h5pregister_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pregister_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pregister_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" ! H5Pinsert_f - WRITE(11,'(A)') " INTERFACE h5pinsert_f" + WRITE(11,'(2X,A)') "INTERFACE h5pinsert_f" DO i = 1, num_rkinds j = rkind(i) WRITE(chr2,'(I2)') j - WRITE(11,'(A)') " MODULE PROCEDURE h5pinsert_kind_"//TRIM(ADJUSTL(chr2)) + WRITE(11,'(5X,A)') "MODULE PROCEDURE h5pinsert_kind_"//TRIM(ADJUSTL(chr2)) END DO - WRITE(11,'(A)') " END INTERFACE" + WRITE(11,'(2X,A)') "END INTERFACE" WRITE(11,'(A)') 'CONTAINS' -- cgit v0.12 From 9bbaca66ca1f987ede2f5c7e9c5938523a47262f Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Fri, 9 Nov 2018 11:29:24 -0600 Subject: fixed sp. --- fortran/src/H5_buildiface.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fortran/src/H5_buildiface.F90 b/fortran/src/H5_buildiface.F90 index e5d103d..f793b7f 100644 --- a/fortran/src/H5_buildiface.F90 +++ b/fortran/src/H5_buildiface.F90 @@ -144,7 +144,7 @@ PROGRAM H5_buildiface WRITE(11,'(2X,A)') 'IMPLICIT NONE' !****************************** -! DECLARE PRIVETE INTERFACES +! DECLARE PRIVATE INTERFACES !****************************** DO i = 1, num_rkinds -- cgit v0.12 From 110c1b671ea7f2f5e8154d66f6ca6fcfb00b2bb2 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Tue, 13 Nov 2018 07:45:56 -0800 Subject: The Java tests will no longer fail when the zlib is not present. --- java/test/TestH5Z.java | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/java/test/TestH5Z.java b/java/test/TestH5Z.java index 31adf22..f3049ed 100644 --- a/java/test/TestH5Z.java +++ b/java/test/TestH5Z.java @@ -41,8 +41,8 @@ public class TestH5Z { @Test public void testH5Zfilter_avail() { try { - int filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_DEFLATE); - assertTrue("H5.H5Zfilter_avail_DEFLATE", filter_found > 0); + int filter_found; + filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_FLETCHER32); assertTrue("H5.H5Zfilter_avail_FLETCHER32", filter_found > 0); filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_NBIT); @@ -51,8 +51,12 @@ public class TestH5Z { assertTrue("H5.H5Zfilter_avail_SCALEOFFSET", filter_found > 0); filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SHUFFLE); assertTrue("H5.H5Zfilter_avail_SHUFFLE", filter_found > 0); -// filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SZIP); -// assertTrue("H5.H5Zfilter_avail_SZIP", filter_found > 0); + + // Just make sure H5Zfilter_avail() doesn't fail with szip/zlib + // since there is no way for us to determine if they should be present + // or not. + filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_DEFLATE); + filter_found = H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SZIP); } catch (Throwable err) { err.printStackTrace(); @@ -63,9 +67,8 @@ public class TestH5Z { @Test public void testH5Zget_filter_info() { try { - int filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_DEFLATE); - assertTrue("H5.H5Zget_filter_info_DEFLATE_DECODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); - assertTrue("H5.H5Zget_filter_info_DEFLATE_ENCODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); + int filter_flag; + filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_FLETCHER32); assertTrue("H5.H5Zget_filter_info_FLETCHER32_DECODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); assertTrue("H5.H5Zget_filter_info_FLETCHER32_ENCODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); @@ -78,9 +81,19 @@ public class TestH5Z { filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_SHUFFLE); assertTrue("H5.H5Zget_filter_info_DECODE_SHUFFLE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); assertTrue("H5.H5Zget_filter_info_ENCODE_SHUFFLE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); -// filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_SZIP); -// assertTrue("H5.H5Zget_filter_info_DECODE_SZIP_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); -// assertTrue("H5.H5Zget_filter_info_ENCODE_SZIP_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); + + if(1 == H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_DEFLATE)) { + filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_DEFLATE); + assertTrue("H5.H5Zget_filter_info_DEFLATE_DECODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); + assertTrue("H5.H5Zget_filter_info_DEFLATE_ENCODE_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_ENCODE_ENABLED) > 0); + } + + if(1 == H5.H5Zfilter_avail(HDF5Constants.H5Z_FILTER_SZIP)) { + filter_flag = H5.H5Zget_filter_info(HDF5Constants.H5Z_FILTER_SZIP); + // Decode should always be available, but we have no way of determining + // if encode is so don't assert on that. + assertTrue("H5.H5Zget_filter_info_DECODE_SZIP_ENABLED", (filter_flag & HDF5Constants.H5Z_FILTER_CONFIG_DECODE_ENABLED) > 0); + } } catch (Throwable err) { err.printStackTrace(); @@ -96,3 +109,4 @@ public class TestH5Z { H5.H5Zunregister(HDF5Constants.H5Z_FILTER_SHUFFLE); } } + -- cgit v0.12 From 2fe10c647c003eaba2f84d52daaae23c3341916a Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Tue, 13 Nov 2018 11:37:29 -0600 Subject: HDFFV10601: Adding performance test to verify the improvement. --- tools/test/perform/CMakeLists.txt | 10 ++ tools/test/perform/Makefile.am | 4 +- tools/test/perform/chunk_cache.c | 361 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 tools/test/perform/chunk_cache.c diff --git a/tools/test/perform/CMakeLists.txt b/tools/test/perform/CMakeLists.txt index fa41608..14abdec 100644 --- a/tools/test/perform/CMakeLists.txt +++ b/tools/test/perform/CMakeLists.txt @@ -51,6 +51,16 @@ TARGET_C_PROPERTIES (iopipe STATIC) target_link_libraries (iopipe PRIVATE ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) set_target_properties (iopipe PROPERTIES FOLDER perform) +#-- Adding test for chunk_cache +set (chunk_cache_SOURCES + ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/chunk_cache.c +) +add_executable (chunk_cache ${chunk_cache_SOURCES}) +target_include_directories(chunk_cache PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") +TARGET_C_PROPERTIES (chunk_cache STATIC) +target_link_libraries (chunk_cache PRIVATE ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) +set_target_properties (chunk_cache PROPERTIES FOLDER perform) + #-- Adding test for overhead set (overhead_SOURCES ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/overhead.c diff --git a/tools/test/perform/Makefile.am b/tools/test/perform/Makefile.am index 5a89a66..39800d7 100644 --- a/tools/test/perform/Makefile.am +++ b/tools/test/perform/Makefile.am @@ -50,12 +50,12 @@ if BUILD_PARALLEL_CONDITIONAL TEST_PROG_PARA=h5perf perf endif # Serial test programs. -TEST_PROG = iopipe chunk overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) +TEST_PROG = iopipe chunk chunk_cache overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) # check_PROGRAMS will be built but not installed. Do not any executable # that is in bin_PROGRAMS already. Otherwise, it will be removed twice in # "make clean" and some systems, e.g., AIX, do not like it. -check_PROGRAMS= iopipe chunk overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf +check_PROGRAMS= iopipe chunk chunk_cache overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf h5perf_SOURCES=pio_perf.c pio_engine.c h5perf_serial_SOURCES=sio_perf.c sio_engine.c diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c new file mode 100644 index 0000000..9a6315a --- /dev/null +++ b/tools/test/perform/chunk_cache.c @@ -0,0 +1,361 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: check the performance of chunk cache in these two cases (HDFFV-10601): + * 1. partial chunks exist along any dimension. + * 2. number of slots in chunk cache is smaller than the number of chunks + * in the fastest-growing dimension. + */ +#include +#include "hdf5.h" + +#define FILENAME "chunk_cache_perf.h5" + +#define RANK 2 + +#define DSET1_NAME "partial_chunks" +#define DSET1_DIM1 9 * 1000 +#define DSET1_DIM2 9 +#define CHUNK1_DIM1 2 * 1000 +#define CHUNK1_DIM2 2 + +#define DSET2_NAME "hash_value" +#define DSET2_DIM1 300 +#define DSET2_DIM2 600 +#define CHUNK2_DIM1 100 +#define CHUNK2_DIM2 100 + +#define RDCC_NSLOTS 5 +#define RDCC_NBYTES 1024 * 1024 * 10 +#define RDCC_W0 0.75F + +#define FILTER_COUNTER 306 +static size_t nbytes_global; + +typedef struct test_time_t { + long tv_sec; + long tv_usec; +} test_time_t; + +/* Local function prototypes for the dummy filter */ +static size_t +counter (unsigned flags, size_t cd_nelmts, + const unsigned *cd_values, size_t nbytes, + size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_COUNTER[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + FILTER_COUNTER, /* Filter id number */ + 1, 1, /* Encoding and decoding enabled */ + "counter", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + counter, /* The actual filter function */ +}}; + +/*------------------------------------------------------------------------- + * Count number of bytes but don't do anything else. Keep + * track of the data of chunks being read from file into memory. + */ +static size_t +counter (unsigned flags, size_t cd_nelmts, + const unsigned *cd_values, size_t nbytes, + size_t *buf_size, void **buf) +{ + nbytes_global += nbytes; + return nbytes; +} + +/*---------------------------------------------------------------------------*/ +static int +test_time_get_current(test_time_t *tv) +{ + struct timespec tp; + + if (!tv) + return -1; + if (clock_gettime(CLOCK_MONOTONIC, &tp)) + return -1; + + tv->tv_sec = tp.tv_sec; + tv->tv_usec = tp.tv_nsec / 1000; + + return 0; +} + +/*---------------------------------------------------------------------------*/ +static double +test_time_to_double(test_time_t tv) +{ + return (double) tv.tv_sec + (double) (tv.tv_usec) * 0.000001; +} + +/*---------------------------------------------------------------------------*/ +static test_time_t +test_time_add(test_time_t in1, test_time_t in2) +{ + test_time_t out; + + out.tv_sec = in1.tv_sec + in2.tv_sec; + out.tv_usec = in1.tv_usec + in2.tv_usec; + if(out.tv_usec > 1000000) { + out.tv_usec -= 1000000; + out.tv_sec += 1; + } + + return out; +} + +/*---------------------------------------------------------------------------*/ +static test_time_t +test_time_subtract(test_time_t in1, test_time_t in2) +{ + test_time_t out; + + out.tv_sec = in1.tv_sec - in2.tv_sec; + out.tv_usec = in1.tv_usec - in2.tv_usec; + if(out.tv_usec < 0) { + out.tv_usec += 1000000; + out.tv_sec -= 1; + } + + return out; +} + +/*------------------------------------------------------------------------------- + * Create a chunked dataset with partial chunks along either dimensions: + * dataset dimension: 9000 x 9 + * chunk dimension: 2000 x 2 + */ +static void create_dset1(hid_t file) +{ + hid_t dataspace, dataset; + hid_t dcpl; + hsize_t dims[RANK] = {DSET1_DIM1, DSET1_DIM2}; + herr_t status; + hsize_t chunk_dims[RANK] = {CHUNK1_DIM1, CHUNK1_DIM2}; + int data[DSET1_DIM1][DSET1_DIM2]; /* data for writing */ + int i, j; + + /* Create the data space. */ + dataspace = H5Screate_simple (RANK, dims, NULL); + + /* Modify dataset creation properties, i.e. enable chunking */ + dcpl = H5Pcreate (H5P_DATASET_CREATE); + status = H5Pset_chunk (dcpl, RANK, chunk_dims); + + /* Set the dummy filter simply for counting the number of bytes being read into the memory */ + H5Zregister(H5Z_COUNTER); + H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL); + + /* Create a new dataset within the file using chunk creation properties. */ + dataset = H5Dcreate2 (file, DSET1_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT); + + for (i = 0; i < DSET1_DIM1; i++) + for (j = 0; j < DSET1_DIM2; j++) + data[i][j] = i+j; + + /* Write data to dataset */ + status = H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, data); + + /* Close resources */ + status = H5Dclose (dataset); + status = H5Pclose (dcpl); + status = H5Sclose (dataspace); +} + +/*--------------------------------------------------------------------------- + * Create a chunked dataset for testing hash values: + * dataset dimensions: 300 x 600 + * chunk dimensions: 100 x 100 + */ +static void create_dset2(hid_t file) +{ + hid_t dataspace, dataset; + hid_t dcpl; + hsize_t dims[RANK] = {DSET2_DIM1, DSET2_DIM2}; + herr_t status; + hsize_t chunk_dims[RANK] = {CHUNK2_DIM1, CHUNK2_DIM2}; + int data[DSET2_DIM1][DSET2_DIM2]; /* data for writing */ + int i, j; + + /* Create the data space. */ + dataspace = H5Screate_simple (RANK, dims, NULL); + + /* Modify dataset creation properties, i.e. enable chunking */ + dcpl = H5Pcreate (H5P_DATASET_CREATE); + status = H5Pset_chunk (dcpl, RANK, chunk_dims); + + /* Set the dummy filter simply for counting the number of bytes being read into the memory */ + H5Zregister(H5Z_COUNTER); + H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL); + + /* Create a new dataset within the file using chunk creation properties. */ + dataset = H5Dcreate2 (file, DSET2_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT); + + for (i = 0; i < DSET2_DIM1; i++) + for (j = 0; j < DSET2_DIM2; j++) + data[i][j] = i+j; + + /* Write data to dataset */ + status = H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, data); + + /* Close resources */ + status = H5Dclose (dataset); + status = H5Pclose (dcpl); + status = H5Sclose (dataspace); +} +/*--------------------------------------------------------------------------- + * Check the performance of the chunk cache when partial chunks exist + * along the dataset dimensions. + */ +static void check_partial_chunks_perf(hid_t file) +{ + hid_t dataset; + hid_t filespace; + hid_t memspace; + hid_t dapl; + + herr_t status; + int rdata[DSET1_DIM2]; /* data for reading */ + int i; + + hsize_t row_rank = 1; + hsize_t row_dim[1] = {DSET1_DIM2}; + hsize_t start[RANK] = {0, 0}; + hsize_t count[RANK] = {1, DSET1_DIM2}; + test_time_t t = {0, 0}, t1 = {0, 0}, t2 = {0, 0}; + + dapl = H5Pcreate(H5P_DATASET_ACCESS); + status = H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0); + + dataset = H5Dopen2 (file, DSET1_NAME, dapl); + + memspace = H5Screate_simple(row_rank, row_dim, NULL); + filespace = H5Dget_space(dataset); + + nbytes_global = 0; + + test_time_get_current(&t1); + + /* Read the data row by row */ + for(i = 0; i < DSET1_DIM1; i++) { + start[0] = i; + status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL); + + status = H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata); + } + + test_time_get_current(&t2); + t = test_time_add(t, test_time_subtract(t2, t1)); + + printf("1. Partial chunks: total read time is %lf; number of bytes being read from file is %lu\n", test_time_to_double(t), nbytes_global); + + status = H5Dclose (dataset); + status = H5Sclose (filespace); + status = H5Sclose (memspace); + status = H5Pclose (dapl); +} + +/*--------------------------------------------------------------------------- + * Check the performance of chunk cache when the number of cache slots + * is smaller than the number of chunks along the fastest-growing + * dimension of the dataset. + */ +static void check_hash_value_perf(hid_t file) +{ + hid_t dataset; + hid_t filespace; + hid_t memspace; + hid_t dapl; + + herr_t status; + int rdata[DSET2_DIM1]; /* data for reading */ + int i; + + hsize_t column_rank = 1; + hsize_t column_dim[1] = {DSET2_DIM1}; + hsize_t start[RANK] = {0, 0}; + hsize_t count[RANK] = {DSET2_DIM1, 1}; + test_time_t t = {0, 0}, t1 = {0, 0}, t2 = {0, 0}; + + dapl = H5Pcreate(H5P_DATASET_ACCESS); + status = H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0); + + dataset = H5Dopen2 (file, DSET2_NAME, dapl); + + memspace = H5Screate_simple(column_rank, column_dim, NULL); + filespace = H5Dget_space(dataset); + + nbytes_global = 0; + + test_time_get_current(&t1); + + /* Read the data column by column */ + for(i = 0; i < DSET2_DIM2; i++) { + start[1] = i; + status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL); + + status = H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata); + } + + test_time_get_current(&t2); + t = test_time_add(t, test_time_subtract(t2, t1)); + + printf("2. Hash value: total read time is %lf; number of bytes being read from file is %lu\n", test_time_to_double(t), nbytes_global); + + status = H5Dclose (dataset); + status = H5Sclose (filespace); + status = H5Sclose (memspace); + status = H5Pclose (dapl); +} + +/*------------------------------------------------------------------------------------- + * Purpose: check the performance of chunk cache in these two cases (HDFFV-10601): + * 1. partial chunks exist along any dimension. + * 2. number of slots in chunk cache is smaller than the number of chunks + * in the fastest-growing dimension. + *-------------------------------------------------------------------------------------*/ +int +main (void) +{ + hid_t file; /* handles */ + + /* Create a new file. If file exists its contents will be overwritten. */ + file = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + create_dset1(file); + create_dset2(file); + + H5Fclose (file); + + /* Re-open the file for testing performance. */ + file = H5Fopen (FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + + check_partial_chunks_perf(file); + check_hash_value_perf(file); + + H5Fclose (file); + + return 0; +} -- cgit v0.12 From 3edd97731e90b7fc3a378e2c6e9c043a23327d93 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Wed, 14 Nov 2018 13:37:31 -0800 Subject: Renamed h5repart's -family_to_sec2 to -family_to_single. --- release_docs/RELEASE.txt | 14 +++++++++-- src/H5FDlog.c | 16 ++++++------ src/H5FDsec2.c | 16 ++++++------ src/H5Fprivate.h | 2 +- src/H5Pfapl.c | 15 ++++++------ tools/src/misc/h5repart.c | 45 ++++++++++++++++++++-------------- tools/test/misc/CMakeTestsRepart.cmake | 5 ++++ tools/test/misc/repart_test.c | 21 ++++++++++------ tools/test/misc/testh5repart.sh.in | 6 +++-- 9 files changed, 86 insertions(+), 54 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index dcc9892..4661b59 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -170,7 +170,7 @@ New Features (DER - 2018/10/26, HDFFV-10614) - Parallel Library: + Parallel Library: ----------------- - @@ -233,7 +233,17 @@ New Features Tools: ------ - - + - The h5repart -family-to-sec2 argument was changed to -family-to-single + + In order to better support other single-file VFDs which could work with + h5repart, the -family-to-sec2 argument was renamed to -family-to-single. + This is just a name change and the functionality of the argument has not + changed. + + The -family-to-sec2 argument has been kept for backwards-compatibility. + This argument should be considered deprecated. + + (DER - 2018/11/14, HDFFV-10633) High-Level APIs: --------------- diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 7c2297a..3dcd7f5 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -117,7 +117,7 @@ typedef struct H5FD_log_t { * Whether to eliminate the family driver info and convert this file to * a single file */ - hbool_t fam_to_sec2; + hbool_t fam_to_single; /* Fields for tracking I/O operations */ unsigned char *nread; /* Number of reads from a file location */ @@ -635,13 +635,13 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from - * family to sec2 while using h5repart, this private property should be set so that - * in the later step, the library can ignore the family driver information saved - * in the superblock. + * family to one that uses single files (sec2, etc.) while using h5repart, this + * private property should be set so that in the later step, the library can ignore + * the family driver information saved in the superblock. */ - if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SEC2_NAME) > 0) - if(H5P_get(plist, H5F_ACS_FAMILY_TO_SEC2_NAME, &file->fam_to_sec2) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to sec2") + if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME) > 0) + if(H5P_get(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME, &file->fam_to_single) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to single") } /* end if */ /* Set return value */ @@ -900,7 +900,7 @@ H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) *flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default VFD */ /* Check for flags that are set by h5repart */ - if(file && file->fam_to_sec2) + if(file && file->fam_to_single) *flags |= H5FD_FEAT_IGNORE_DRVRINFO; /* Ignore the driver info when file is opened (which eliminates it) */ } /* end if */ diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 32aff0f..06c008d 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -94,7 +94,7 @@ typedef struct H5FD_sec2_t { * Whether to eliminate the family driver info and convert this file to * a single file. */ - hbool_t fam_to_sec2; + hbool_t fam_to_single; } H5FD_sec2_t; @@ -386,13 +386,13 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from - * family to sec2 while using h5repart, this private property should be set so that - * in the later step, the library can ignore the family driver information saved - * in the superblock. + * family to one that uses single files (sec2, etc.) while using h5repart, this + * private property should be set so that in the later step, the library can ignore + * the family driver information saved in the superblock. */ - if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SEC2_NAME) > 0) - if(H5P_get(plist, H5F_ACS_FAMILY_TO_SEC2_NAME, &file->fam_to_sec2) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to sec2") + if(H5P_exist_plist(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME) > 0) + if(H5P_get(plist, H5F_ACS_FAMILY_TO_SINGLE_NAME, &file->fam_to_single) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get property of changing family to single") } /* end if */ /* Set return value */ @@ -532,7 +532,7 @@ H5FD_sec2_query(const H5FD_t *_file, unsigned long *flags /* out */) *flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default VFD */ /* Check for flags that are set by h5repart */ - if(file && file->fam_to_sec2) + if(file && file->fam_to_single) *flags |= H5FD_FEAT_IGNORE_DRVRINFO; /* Ignore the driver info when file is opened (which eliminates it) */ } /* end if */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 6595677..a4b7880 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -483,7 +483,7 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_CLOSE_DEGREE_NAME "close_degree" /* File close degree */ #define H5F_ACS_FAMILY_OFFSET_NAME "family_offset" /* Offset position in file for family file driver */ #define H5F_ACS_FAMILY_NEWSIZE_NAME "family_newsize" /* New member size of family driver. (private property only used by h5repart) */ -#define H5F_ACS_FAMILY_TO_SEC2_NAME "family_to_sec2" /* Whether to convert family to sec2 driver. (private property only used by h5repart) */ +#define H5F_ACS_FAMILY_TO_SINGLE_NAME "family_to_single" /* Whether to convert family to a single-file driver. (private property only used by h5repart) */ #define H5F_ACS_MULTI_TYPE_NAME "multi_type" /* Data type in multi file driver */ #define H5F_ACS_LIBVER_LOW_BOUND_NAME "libver_low_bound" /* 'low' bound of library format versions */ #define H5F_ACS_LIBVER_HIGH_BOUND_NAME "libver_high_bound" /* 'high' bound of library format versions */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index a50c5ee..23eb924 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -135,10 +135,11 @@ * property only used by h5repart */ #define H5F_ACS_FAMILY_NEWSIZE_SIZE sizeof(hsize_t) #define H5F_ACS_FAMILY_NEWSIZE_DEF 0 -/* Definition for whether to convert family to sec2 driver. It's private - * property only used by h5repart */ -#define H5F_ACS_FAMILY_TO_SEC2_SIZE sizeof(hbool_t) -#define H5F_ACS_FAMILY_TO_SEC2_DEF FALSE +/* Definition for whether to convert family to a single-file driver. + * It's a private property only used by h5repart. + */ +#define H5F_ACS_FAMILY_TO_SINGLE_SIZE sizeof(hbool_t) +#define H5F_ACS_FAMILY_TO_SINGLE_DEF FALSE /* Definition for data type in multi file driver */ #define H5F_ACS_MULTI_TYPE_SIZE sizeof(H5FD_mem_t) #define H5F_ACS_MULTI_TYPE_DEF H5FD_MEM_DEFAULT @@ -391,7 +392,7 @@ static const unsigned H5F_def_gc_ref_g = H5F_ACS_GARBG_COLCT_REF_DEF; static const H5F_close_degree_t H5F_def_close_degree_g = H5F_CLOSE_DEGREE_DEF; /* Default file close degree */ static const hsize_t H5F_def_family_offset_g = H5F_ACS_FAMILY_OFFSET_DEF; /* Default offset for family VFD */ static const hsize_t H5F_def_family_newsize_g = H5F_ACS_FAMILY_NEWSIZE_DEF; /* Default size of new files for family VFD */ -static const hbool_t H5F_def_family_to_sec2_g = H5F_ACS_FAMILY_TO_SEC2_DEF; /* Default ?? for family VFD */ +static const hbool_t H5F_def_family_to_single_g = H5F_ACS_FAMILY_TO_SINGLE_DEF; /* Default ?? for family VFD */ static const H5FD_mem_t H5F_def_mem_type_g = H5F_ACS_MULTI_TYPE_DEF; /* Default file space type for multi VFD */ static const H5F_libver_t H5F_def_libver_low_bound_g = H5F_ACS_LIBVER_LOW_BOUND_DEF; /* Default setting for "low" bound of format version */ @@ -527,9 +528,9 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") - /* Register the private property of whether convert family to sec2 driver. It's used by h5repart only. */ + /* Register the private property of whether convert family to a single-file driver. It's used by h5repart only. */ /* (Note: this property should not have an encode/decode callback -QAK) */ - if(H5P__register_real(pclass, H5F_ACS_FAMILY_TO_SEC2_NAME, H5F_ACS_FAMILY_TO_SEC2_SIZE, &H5F_def_family_to_sec2_g, + if(H5P__register_real(pclass, H5F_ACS_FAMILY_TO_SINGLE_NAME, H5F_ACS_FAMILY_TO_SINGLE_SIZE, &H5F_def_family_to_single_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") diff --git a/tools/src/misc/h5repart.c b/tools/src/misc/h5repart.c index cdc554f..2f4d93e 100644 --- a/tools/src/misc/h5repart.c +++ b/tools/src/misc/h5repart.c @@ -29,11 +29,12 @@ #define NAMELEN 4096 #define GB *1024*1024*1024 -/*Make these 2 private properties(defined in H5Fprivate.h) available to h5repart. - *The first one updates the member file size in the superblock. The second one - *change file driver from family to sec2. */ +/* Make these 2 private properties(defined in H5Fprivate.h) available to h5repart. + * The first one updates the member file size in the superblock. The second one + * change file driver from family to a single file driver. + */ #define H5F_ACS_FAMILY_NEWSIZE_NAME "family_newsize" -#define H5F_ACS_FAMILY_TO_SEC2_NAME "family_to_sec2" +#define H5F_ACS_FAMILY_TO_SINGLE_NAME "family_to_single" /*------------------------------------------------------------------------- @@ -53,13 +54,14 @@ static void usage (const char *progname) { - fprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] [-family_to_sec2] SRC DST\n", + fprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] [-family_to_sec2|-family_to_single] SRC DST\n", progname); fprintf(stderr, " -v Produce verbose output\n"); fprintf(stderr, " -V Print a version number and exit\n"); fprintf(stderr, " -b N The I/O block size, defaults to 1kB\n"); fprintf(stderr, " -m N The destination member size or 1GB\n"); - fprintf(stderr, " -family_to_sec2 Change file driver from family to sec2\n"); + fprintf(stderr, " -family_to_sec2 Deprecated version of -family_to_single (below)\n"); + fprintf(stderr, " -family_to_single Change file driver from family to the default single-file VFD (windows or sec2)\n"); fprintf(stderr, " SRC The name of the source file\n"); fprintf(stderr, " DST The name of the destination files\n"); fprintf(stderr, "Sizes may be suffixed with `g' for GB, `m' for MB or " @@ -186,7 +188,7 @@ main (int argc, char *argv[]) hid_t fapl; /*file access property list */ hid_t file; hsize_t hdsize; /*destination logical memb size */ - hbool_t family_to_sec2=FALSE; /*change family to sec2 driver? */ + hbool_t family_to_single = FALSE; /*change family to single file driver? */ /* * Get the program name from argv[0]. Use only the last component. @@ -206,7 +208,10 @@ main (int argc, char *argv[]) prog_name, H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE); exit(EXIT_SUCCESS); } else if (!strcmp (argv[argno], "-family_to_sec2")) { - family_to_sec2 = TRUE; + family_to_single = TRUE; + argno++; + } else if (!strcmp (argv[argno], "-family_to_single")) { + family_to_single = TRUE; argno++; } else if ('b'==argv[argno][1]) { blk_size = (size_t)get_size (prog_name, &argno, argc, argv); @@ -422,11 +427,12 @@ main (int argc, char *argv[]) exit (EXIT_FAILURE); } - if(family_to_sec2) { - /* The user wants to change file driver from family to sec2. Open the file - * with sec2 driver. This property signals the library to ignore the family - * driver information saved in the superblock. */ - if(H5Pset(fapl, H5F_ACS_FAMILY_TO_SEC2_NAME, &family_to_sec2) < 0) { + if(family_to_single) { + /* The user wants to change file driver from family to a single-file VFD. + * Open the file with the sec2, windows, etc. driver. This property signals + * the library to ignore the family driver information saved in the superblock. + */ + if(H5Pset(fapl, H5F_ACS_FAMILY_TO_SINGLE_NAME, &family_to_single) < 0) { perror ("H5Pset"); exit (EXIT_FAILURE); } @@ -449,13 +455,14 @@ main (int argc, char *argv[]) /* If the new file is a family file, try to open file for "read and write" to * flush metadata. Flushing metadata will update the superblock to the new - * member size. If the original file is a family file and the new file is a sec2 - * file, the property FAMILY_TO_SEC2 will signal the library to switch to sec2 - * driver when the new file is opened. If the original file is a sec2 file and the - * new file can only be a sec2 file, reopen the new file should fail. There's - * nothing to do in this case. */ + * member size. If the original file is a family file and the new file is a single + * file, the property FAMILY_TO_SINGLE will signal the library to switch to default + * single-file driver when the new file is opened. If the original file is a single + * file and the new file can only be a single file, reopen the new file should fail. + * There's nothing to do in this case. + */ H5E_BEGIN_TRY { - file=H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl); + file = H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl); } H5E_END_TRY; if(file>=0) { diff --git a/tools/test/misc/CMakeTestsRepart.cmake b/tools/test/misc/CMakeTestsRepart.cmake index 37da903..830de11 100644 --- a/tools/test/misc/CMakeTestsRepart.cmake +++ b/tools/test/misc/CMakeTestsRepart.cmake @@ -68,6 +68,7 @@ scd_family00001.h5 scd_family00002.h5 scd_family00003.h5 + family_to_single.h5 family_to_sec2.h5 ) if (NOT "${last_test}" STREQUAL "") @@ -84,6 +85,10 @@ set_tests_properties (H5REPART-h5repart_5K PROPERTIES DEPENDS H5REPART-clearall-objects) # convert family file to sec2 file of 20,000 bytes + add_test (NAME H5REPART-h5repart_single COMMAND $ -m 20000 -family_to_single family_file%05d.h5 family_to_single.h5) + set_tests_properties (H5REPART-h5repart_single PROPERTIES DEPENDS H5REPART-clearall-objects) + + # convert family file to sec2 file of 20,000 bytes (old argument) add_test (NAME H5REPART-h5repart_sec2 COMMAND $ -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5) set_tests_properties (H5REPART-h5repart_sec2 PROPERTIES DEPENDS H5REPART-clearall-objects) diff --git a/tools/test/misc/repart_test.c b/tools/test/misc/repart_test.c index 4016ee8..b8e0292 100644 --- a/tools/test/misc/repart_test.c +++ b/tools/test/misc/repart_test.c @@ -26,12 +26,13 @@ const char *FILENAME[] = { "fst_family%05d.h5", "scd_family%05d.h5", + "family_to_single.h5", "family_to_sec2.h5", NULL }; herr_t test_family_h5repart_opens(void); -herr_t test_sec2_h5repart_opens(void); +herr_t test_single_h5repart_opens(void); /*------------------------------------------------------------------------- @@ -90,23 +91,28 @@ error: /*------------------------------------------------------------------------- - * Function: test_sec2_h5repart_opens + * Function: test_single_h5repart_opens * - * Purpose: Tries to reopen a sec2 file. + * Purpose: Tries to reopen a single file. * * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ herr_t -test_sec2_h5repart_opens(void) +test_single_h5repart_opens(void) { hid_t fid = -1; - /* open the sec2 file */ + /* open the single file */ if ((fid = H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT)) < 0) goto error; + if (H5Fclose(fid) < 0) + goto error; + /* open the single file (created using the old argument) */ + if ((fid = H5Fopen(FILENAME[3], H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + goto error; if (H5Fclose(fid) < 0) goto error; @@ -119,7 +125,7 @@ error: return FAIL; -} /* end test_sec2_h5repart_opens() */ +} /* end test_single_h5repart_opens() */ /*------------------------------------------------------------------------- @@ -137,7 +143,7 @@ main(void) int nerrors = 0; nerrors += test_family_h5repart_opens() < 0 ? 1 : 0; - nerrors += test_sec2_h5repart_opens() < 0 ? 1 : 0; + nerrors += test_single_h5repart_opens() < 0 ? 1 : 0; if (nerrors) goto error; @@ -150,3 +156,4 @@ error: nerrors, 1 == nerrors ? "" : "S"); HDexit(EXIT_FAILURE); } /* end main() */ + diff --git a/tools/test/misc/testh5repart.sh.in b/tools/test/misc/testh5repart.sh.in index 594d37e..143f607 100644 --- a/tools/test/misc/testh5repart.sh.in +++ b/tools/test/misc/testh5repart.sh.in @@ -171,7 +171,9 @@ COPY_TESTFILES_TO_TESTDIR TOOLTEST -m 20000 family_file%05d.h5 fst_family%05d.h5 # repartition family member size to 5 KB. TOOLTEST -m 5k family_file%05d.h5 scd_family%05d.h5 -# convert family file to sec2 file of 20,000 bytes +# convert family file to single file of 20,000 bytes +TOOLTEST -m 20000 -family_to_single family_file%05d.h5 family_to_single.h5 +# convert family file to single file of 20,000 bytes (old argument) TOOLTEST -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5 # test the output files repartitioned above. @@ -183,7 +185,7 @@ CLEAN_TESTFILES_AND_TESTDIR if test -z "$HDF5_NOCLEANUP"; then cd $actual_dir - rm -f fst_family*.h5 scd_family*.h5 family_to_sec2.h5 + rm -f fst_family*.h5 scd_family*.h5 family_to_single.h5 family_to_sec2.h5 fi if test $nerrors -eq 0 ; then -- cgit v0.12 From cd13d24e5140578a880aebe4e2d8b899179d0870 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Wed, 14 Nov 2018 17:30:23 -0600 Subject: HDFFV-10601: I added error checking to the HDF5 functions. --- tools/test/perform/chunk_cache.c | 226 ++++++++++++++++++++++++++++----------- 1 file changed, 161 insertions(+), 65 deletions(-) diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c index 9a6315a..01571e2 100644 --- a/tools/test/perform/chunk_cache.c +++ b/tools/test/perform/chunk_cache.c @@ -17,6 +17,8 @@ * 2. number of slots in chunk cache is smaller than the number of chunks * in the fastest-growing dimension. */ +#include +#include #include #include "hdf5.h" @@ -133,49 +135,87 @@ test_time_subtract(test_time_t in1, test_time_t in2) return out; } +/*------------------------------------------------------------------------- + * Function: cleanup + * + * Purpose: Removes test files + * + * Return: void + * + * Programmer: Robb Matzke + * Thursday, June 4, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void +cleanup (void) +{ + if (!getenv ("HDF5_NOCLEANUP")) { + remove (FILENAME); + } +} /*------------------------------------------------------------------------------- * Create a chunked dataset with partial chunks along either dimensions: * dataset dimension: 9000 x 9 * chunk dimension: 2000 x 2 */ -static void create_dset1(hid_t file) +static int create_dset1(hid_t file) { hid_t dataspace, dataset; hid_t dcpl; hsize_t dims[RANK] = {DSET1_DIM1, DSET1_DIM2}; - herr_t status; hsize_t chunk_dims[RANK] = {CHUNK1_DIM1, CHUNK1_DIM2}; int data[DSET1_DIM1][DSET1_DIM2]; /* data for writing */ int i, j; /* Create the data space. */ - dataspace = H5Screate_simple (RANK, dims, NULL); + if((dataspace = H5Screate_simple (RANK, dims, NULL)) < 0) + goto error; /* Modify dataset creation properties, i.e. enable chunking */ - dcpl = H5Pcreate (H5P_DATASET_CREATE); - status = H5Pset_chunk (dcpl, RANK, chunk_dims); + if((dcpl = H5Pcreate (H5P_DATASET_CREATE)) < 0) + goto error; + if(H5Pset_chunk (dcpl, RANK, chunk_dims) < 0) + goto error; /* Set the dummy filter simply for counting the number of bytes being read into the memory */ - H5Zregister(H5Z_COUNTER); - H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL); + if(H5Zregister(H5Z_COUNTER) < 0) + goto error; + + if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0) + goto error; /* Create a new dataset within the file using chunk creation properties. */ - dataset = H5Dcreate2 (file, DSET1_NAME, H5T_NATIVE_INT, dataspace, - H5P_DEFAULT, dcpl, H5P_DEFAULT); + if((dataset = H5Dcreate2 (file, DSET1_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < DSET1_DIM1; i++) for (j = 0; j < DSET1_DIM2; j++) data[i][j] = i+j; /* Write data to dataset */ - status = H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, data); + if(H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, data) < 0) + goto error; /* Close resources */ - status = H5Dclose (dataset); - status = H5Pclose (dcpl); - status = H5Sclose (dataspace); + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + } H5E_END_TRY; + + return 1; } /*--------------------------------------------------------------------------- @@ -183,56 +223,72 @@ static void create_dset1(hid_t file) * dataset dimensions: 300 x 600 * chunk dimensions: 100 x 100 */ -static void create_dset2(hid_t file) +static int create_dset2(hid_t file) { hid_t dataspace, dataset; hid_t dcpl; hsize_t dims[RANK] = {DSET2_DIM1, DSET2_DIM2}; - herr_t status; hsize_t chunk_dims[RANK] = {CHUNK2_DIM1, CHUNK2_DIM2}; int data[DSET2_DIM1][DSET2_DIM2]; /* data for writing */ int i, j; /* Create the data space. */ - dataspace = H5Screate_simple (RANK, dims, NULL); + if((dataspace = H5Screate_simple (RANK, dims, NULL)) < 0) + goto error; /* Modify dataset creation properties, i.e. enable chunking */ - dcpl = H5Pcreate (H5P_DATASET_CREATE); - status = H5Pset_chunk (dcpl, RANK, chunk_dims); + if((dcpl = H5Pcreate (H5P_DATASET_CREATE)) < 0) + goto error; + if(H5Pset_chunk (dcpl, RANK, chunk_dims) < 0) + goto error; /* Set the dummy filter simply for counting the number of bytes being read into the memory */ - H5Zregister(H5Z_COUNTER); - H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL); + if(H5Zregister(H5Z_COUNTER) < 0) + goto error; + if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0) + goto error; /* Create a new dataset within the file using chunk creation properties. */ - dataset = H5Dcreate2 (file, DSET2_NAME, H5T_NATIVE_INT, dataspace, - H5P_DEFAULT, dcpl, H5P_DEFAULT); + if((dataset = H5Dcreate2 (file, DSET2_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < DSET2_DIM1; i++) for (j = 0; j < DSET2_DIM2; j++) data[i][j] = i+j; /* Write data to dataset */ - status = H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, data); + if(H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, data) < 0) + goto error; /* Close resources */ - status = H5Dclose (dataset); - status = H5Pclose (dcpl); - status = H5Sclose (dataspace); + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + } H5E_END_TRY; + + return 1; } /*--------------------------------------------------------------------------- * Check the performance of the chunk cache when partial chunks exist * along the dataset dimensions. */ -static void check_partial_chunks_perf(hid_t file) +static int check_partial_chunks_perf(hid_t file) { hid_t dataset; hid_t filespace; hid_t memspace; hid_t dapl; - herr_t status; int rdata[DSET1_DIM2]; /* data for reading */ int i; @@ -242,8 +298,10 @@ static void check_partial_chunks_perf(hid_t file) hsize_t count[RANK] = {1, DSET1_DIM2}; test_time_t t = {0, 0}, t1 = {0, 0}, t2 = {0, 0}; - dapl = H5Pcreate(H5P_DATASET_ACCESS); - status = H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0); + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0) + goto error; dataset = H5Dopen2 (file, DSET1_NAME, dapl); @@ -257,11 +315,13 @@ static void check_partial_chunks_perf(hid_t file) /* Read the data row by row */ for(i = 0; i < DSET1_DIM1; i++) { start[0] = i; - status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, - start, NULL, count, NULL); + if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + goto error; - status = H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, - H5P_DEFAULT, rdata); + if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata) < 0) + goto error; } test_time_get_current(&t2); @@ -269,10 +329,20 @@ static void check_partial_chunks_perf(hid_t file) printf("1. Partial chunks: total read time is %lf; number of bytes being read from file is %lu\n", test_time_to_double(t), nbytes_global); - status = H5Dclose (dataset); - status = H5Sclose (filespace); - status = H5Sclose (memspace); - status = H5Pclose (dapl); + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + + return 0; +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + } H5E_END_TRY; + return 1; } /*--------------------------------------------------------------------------- @@ -280,14 +350,13 @@ static void check_partial_chunks_perf(hid_t file) * is smaller than the number of chunks along the fastest-growing * dimension of the dataset. */ -static void check_hash_value_perf(hid_t file) +static int check_hash_value_perf(hid_t file) { hid_t dataset; hid_t filespace; hid_t memspace; hid_t dapl; - herr_t status; int rdata[DSET2_DIM1]; /* data for reading */ int i; @@ -297,13 +366,17 @@ static void check_hash_value_perf(hid_t file) hsize_t count[RANK] = {DSET2_DIM1, 1}; test_time_t t = {0, 0}, t1 = {0, 0}, t2 = {0, 0}; - dapl = H5Pcreate(H5P_DATASET_ACCESS); - status = H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0); - - dataset = H5Dopen2 (file, DSET2_NAME, dapl); + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0) + goto error; - memspace = H5Screate_simple(column_rank, column_dim, NULL); - filespace = H5Dget_space(dataset); + if((dataset = H5Dopen2 (file, DSET2_NAME, dapl)) < 0) + goto error; + if((memspace = H5Screate_simple(column_rank, column_dim, NULL)) < 0) + goto error; + if((filespace = H5Dget_space(dataset)) < 0) + goto error; nbytes_global = 0; @@ -312,11 +385,13 @@ static void check_hash_value_perf(hid_t file) /* Read the data column by column */ for(i = 0; i < DSET2_DIM2; i++) { start[1] = i; - status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, - start, NULL, count, NULL); + if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + goto error; - status = H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, - H5P_DEFAULT, rdata); + if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata) < 0) + goto error; } test_time_get_current(&t2); @@ -324,10 +399,20 @@ static void check_hash_value_perf(hid_t file) printf("2. Hash value: total read time is %lf; number of bytes being read from file is %lu\n", test_time_to_double(t), nbytes_global); - status = H5Dclose (dataset); - status = H5Sclose (filespace); - status = H5Sclose (memspace); - status = H5Pclose (dapl); + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + } H5E_END_TRY; + return 1; } /*------------------------------------------------------------------------------------- @@ -340,22 +425,33 @@ int main (void) { hid_t file; /* handles */ - + int nerrors = 0; + /* Create a new file. If file exists its contents will be overwritten. */ - file = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if((file = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; - create_dset1(file); - create_dset2(file); + nerrors += create_dset1(file); + nerrors += create_dset2(file); - H5Fclose (file); + if(H5Fclose (file) < 0) + goto error; /* Re-open the file for testing performance. */ - file = H5Fopen (FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + if((file = H5Fopen (FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + goto error; - check_partial_chunks_perf(file); - check_hash_value_perf(file); + nerrors += check_partial_chunks_perf(file); + nerrors += check_hash_value_perf(file); - H5Fclose (file); + if(H5Fclose (file) < 0) + goto error; + if (nerrors>0) goto error; + cleanup(); return 0; + +error: + fprintf(stderr, "*** ERRORS DETECTED ***\n"); + return 1; } -- cgit v0.12 From d25afb294cbd921431ea3cb76e7856fc02676a7d Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Fri, 16 Nov 2018 12:06:30 -0600 Subject: HDFFV-10601: Some compiler doesn't support clock_gettime(). So I changed to use gettimeofday (HDgettimeofday in the HDF5 library). --- tools/test/perform/chunk_cache.c | 102 ++++++++++----------------------------- 1 file changed, 25 insertions(+), 77 deletions(-) diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c index 01571e2..429f70a 100644 --- a/tools/test/perform/chunk_cache.c +++ b/tools/test/perform/chunk_cache.c @@ -17,10 +17,8 @@ * 2. number of slots in chunk cache is smaller than the number of chunks * in the fastest-growing dimension. */ -#include -#include -#include #include "hdf5.h" +#include "H5private.h" #define FILENAME "chunk_cache_perf.h5" @@ -81,74 +79,18 @@ counter (unsigned flags, size_t cd_nelmts, } /*---------------------------------------------------------------------------*/ -static int -test_time_get_current(test_time_t *tv) +double retrieve_time(void) { - struct timespec tp; - - if (!tv) - return -1; - if (clock_gettime(CLOCK_MONOTONIC, &tp)) - return -1; - - tv->tv_sec = tp.tv_sec; - tv->tv_usec = tp.tv_nsec / 1000; - - return 0; -} - -/*---------------------------------------------------------------------------*/ -static double -test_time_to_double(test_time_t tv) -{ - return (double) tv.tv_sec + (double) (tv.tv_usec) * 0.000001; -} - -/*---------------------------------------------------------------------------*/ -static test_time_t -test_time_add(test_time_t in1, test_time_t in2) -{ - test_time_t out; - - out.tv_sec = in1.tv_sec + in2.tv_sec; - out.tv_usec = in1.tv_usec + in2.tv_usec; - if(out.tv_usec > 1000000) { - out.tv_usec -= 1000000; - out.tv_sec += 1; - } - - return out; +#ifdef H5_HAVE_GETTIMEOFDAY + struct timeval t; + HDgettimeofday(&t, NULL); + return ((double)t.tv_sec + (double)t.tv_usec / 1000000); +#else + return 0.0; +#endif } /*---------------------------------------------------------------------------*/ -static test_time_t -test_time_subtract(test_time_t in1, test_time_t in2) -{ - test_time_t out; - - out.tv_sec = in1.tv_sec - in2.tv_sec; - out.tv_usec = in1.tv_usec - in2.tv_usec; - if(out.tv_usec < 0) { - out.tv_usec += 1000000; - out.tv_sec -= 1; - } - - return out; -} -/*------------------------------------------------------------------------- - * Function: cleanup - * - * Purpose: Removes test files - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, June 4, 1998 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ static void cleanup (void) { @@ -296,7 +238,7 @@ static int check_partial_chunks_perf(hid_t file) hsize_t row_dim[1] = {DSET1_DIM2}; hsize_t start[RANK] = {0, 0}; hsize_t count[RANK] = {1, DSET1_DIM2}; - test_time_t t = {0, 0}, t1 = {0, 0}, t2 = {0, 0}; + double start_t, end_t; if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) goto error; @@ -310,7 +252,7 @@ static int check_partial_chunks_perf(hid_t file) nbytes_global = 0; - test_time_get_current(&t1); + start_t = retrieve_time(); /* Read the data row by row */ for(i = 0; i < DSET1_DIM1; i++) { @@ -324,10 +266,13 @@ static int check_partial_chunks_perf(hid_t file) goto error; } - test_time_get_current(&t2); - t = test_time_add(t, test_time_subtract(t2, t1)); + end_t = retrieve_time(); - printf("1. Partial chunks: total read time is %lf; number of bytes being read from file is %lu\n", test_time_to_double(t), nbytes_global); +#ifdef H5_HAVE_GETTIMEOFDAY + printf("1. Partial chunks: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global); +#else + printf("1. Partial chunks: no total read time because gettimeofday() is not available; number of bytes being read from file is %lu\n", nbytes_global); +#endif H5Dclose (dataset); H5Sclose (filespace); @@ -364,7 +309,7 @@ static int check_hash_value_perf(hid_t file) hsize_t column_dim[1] = {DSET2_DIM1}; hsize_t start[RANK] = {0, 0}; hsize_t count[RANK] = {DSET2_DIM1, 1}; - test_time_t t = {0, 0}, t1 = {0, 0}, t2 = {0, 0}; + double start_t, end_t; if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) goto error; @@ -380,7 +325,7 @@ static int check_hash_value_perf(hid_t file) nbytes_global = 0; - test_time_get_current(&t1); + start_t = retrieve_time(); /* Read the data column by column */ for(i = 0; i < DSET2_DIM2; i++) { @@ -394,10 +339,13 @@ static int check_hash_value_perf(hid_t file) goto error; } - test_time_get_current(&t2); - t = test_time_add(t, test_time_subtract(t2, t1)); + end_t = retrieve_time(); - printf("2. Hash value: total read time is %lf; number of bytes being read from file is %lu\n", test_time_to_double(t), nbytes_global); +#ifdef H5_HAVE_GETTIMEOFDAY + printf("2. Hash value: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global); +#else + printf("2. Hash value: no total read time because gettimeofday() is not available; number of bytes being read from file is %lu\n", nbytes_global); +#endif H5Dclose (dataset); H5Sclose (filespace); -- cgit v0.12 From 4f5a52e595194562644517b7600f3b96d4008c17 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Fri, 16 Nov 2018 16:25:50 -0600 Subject: HDFFV-10601: Adding the new hdf5/tools/test/perform/chunk_cache.c to MANIFEST. --- MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST b/MANIFEST index e586b3b..82b17a5 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2615,6 +2615,7 @@ ./tools/test/perform/build_h5perf_alone.sh ./tools/test/perform/build_h5perf_serial_alone.sh ./tools/test/perform/chunk.c +./tools/test/perform/chunk_cache.c ./tools/test/perform/direct_write_perf.c ./tools/test/perform/gen_report.pl ./tools/test/perform/iopipe.c -- cgit v0.12