From c0d6d9bb220bf995a268367cca88dc4febbacf99 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Mon, 8 Jan 2024 07:52:11 -0600 Subject: Sync 1.14 branch with develop (#3923) * Fix build error on freebsd (#3883) Fixes: checking for config freebsd12.1... no checking for config freebsd... found compiler '/home/svcpetsc/petsc-hash-pkgs/39f577/bin/mpicc' is GNU gcc-9.2.0 compiler '/home/svcpetsc/petsc-hash-pkgs/39f577/bin/mpif90' is GNU gfortran-9.2.0 stdout: .: cannot open ./config/classic-fflags: No such file or directory * Correct CMake command and example packaging (#3888) * Feat: Hashpin sensitive dependencies on GitHub Actions and enable Dependabot to update them monthly (#3892) * feat: hashpin sensitive dependencies on GHAs Signed-off-by: Diogo Teles Sant'Anna * feat: enable dependabot for monthly updates on GHA Signed-off-by: Diogo Teles Sant'Anna --------- Signed-off-by: Diogo Teles Sant'Anna * Some changes to portal links when they could be found on docs.hdfgroup.org, and changed the helpdesk link to help.hdfgroup.org (#3893) * Updated some portal links to go directly to docs.hdfgroup. * Fixed some portal and help desk links * Add variable option syncing for examples (#3885) * Add period(.) at the end of the sentence for consistency. (#3897) * Remove redundant backslash character from comment. (#3899) * Disable doxygen as errors for netcdf (#3900) * disable building doxygen for netcdf test * Doc versions (#3903) * Added missing \since tags to H5D. * Committing clang-format changes * Fixed H5T version info. * Committing clang-format changes * Added missing version info to H5E. * Committing clang-format changes * Added version info to H5F public APIs. * Committing clang-format changes * Added missing H5Z public API version info. * Added missing version info to H5G public APIs * Added missing version info to H5I public API. * Added missing version info to H5 public APIs * Committing clang-format changes * Added missing version info to H5P public APIs * Added missing version info to H5R public APIs * Fix comment error. * Committing clang-format changes --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> * Change Trouble Shooting to Troubleshooting (#3905) * Implement optimized support for vector I/O in Subfiling VFD (#3896) Vector I/O requests are now processed within a single set of I/O call batches, rather than each I/O vector entry (tuple constructed from the types, addrs, sizes and bufs arrays) being processed individually. This allows I/O to be more efficiently parallelized among the I/O concentrator processes during large I/O requests. * Fixed some calculations and add test cases for issues spotted from review * Removed a variable that was compensating for previous miscalculations * Add 'warning density' computation to the warnhist script (#3910) * Add 'warning density' computation to the warnhist script, along with several cleanups to it. Add "--enable-show-all-warnings" configure (and CMake) option to disable compiler diagnostic suppression (and therefore show all the otherwise suppressed compiler diagnostics), disabled by default. Clean up a buncn of misc. warnings. Signed-off-by: Quincey Koziol * Added H5Fdelete_f with test (#3912) * New Fortran Examples added (#3916) * added subfiling example * Added filtered writes with no selection example * Version and space corrections. * Restore H5_VERSION definition in configure.ac. * renamed defined H5_VERS* to avoid conflicts (#3926) --- .github/workflows/release-files.yml | 2 +- HDF5Examples/CMakeLists.txt | 2 +- HDF5Examples/FORTRAN/H5G/h5_version.h.in | 23 - HDF5Examples/FORTRAN/H5PAR/CMakeLists.txt | 1 + .../FORTRAN/H5PAR/Fortran_sourcefiles.cmake | 11 + HDF5Examples/FORTRAN/H5PAR/ph5_f90_dataset.F90 | 3 +- HDF5Examples/FORTRAN/H5PAR/ph5_f90_file_create.F90 | 2 +- .../H5PAR/ph5_f90_filtered_writes_no_sel.F90 | 354 +++ .../FORTRAN/H5PAR/ph5_f90_hyperslab_by_chunk.F90 | 3 +- .../FORTRAN/H5PAR/ph5_f90_hyperslab_by_col.F90 | 3 +- .../FORTRAN/H5PAR/ph5_f90_hyperslab_by_pattern.F90 | 2 +- .../FORTRAN/H5PAR/ph5_f90_hyperslab_by_row.F90 | 2 +- HDF5Examples/FORTRAN/H5PAR/ph5_f90_subfiling.F90 | 521 +++++ README.md | 10 +- bin/warnhist | 301 ++- config/cmake/ConfigureChecks.cmake | 11 + config/cmake/H5pubconf.h.in | 3 + config/freebsd | 2 +- configure.ac | 48 +- fortran/src/CMakeLists.txt | 2 + fortran/src/H5Fff.F90 | 43 +- fortran/src/H5_f.c | 2 + fortran/src/H5_ff.F90 | 6 +- fortran/src/H5config_f.inc.cmake | 25 +- fortran/src/H5config_f.inc.in | 18 + fortran/src/H5f90global.F90 | 6 +- fortran/src/Makefile.am | 2 + fortran/src/hdf5_fortrandll.def.in | 1 + fortran/test/tH5F.F90 | 51 +- hl/tools/h5watch/CMakeLists.txt | 4 +- release_docs/INSTALL_CMake.txt | 1 + release_docs/INSTALL_Cygwin.txt | 2 +- release_docs/RELEASE.txt | 23 + release_docs/USING_HDF5_VS.txt | 3 + src/H5Dpkg.h | 2 +- src/H5Dpublic.h | 17 +- src/H5EApkg.h | 2 +- src/H5ESpkg.h | 2 +- src/H5Epublic.h | 20 + src/H5F.c | 2 +- src/H5FDsubfiling/H5FDioc.c | 50 +- src/H5FDsubfiling/H5FDioc_int.c | 8 +- src/H5FDsubfiling/H5FDioc_threads.c | 12 +- src/H5FDsubfiling/H5FDsubfiling.c | 2395 +++++++++++--------- src/H5FDsubfiling/H5subfiling_common.c | 33 + src/H5FDsubfiling/H5subfiling_common.h | 3 + src/H5Fpublic.h | 40 + src/H5Gpublic.h | 39 +- src/H5Idbg.c | 15 +- src/H5Idevelop.h | 2 + src/H5Iint.c | 58 +- src/H5Ipkg.h | 11 +- src/H5Ipublic.h | 30 +- src/H5Pprivate.h | 2 +- src/H5Ppublic.h | 48 +- src/H5Rpublic.h | 45 +- src/H5Tpkg.h | 2 +- src/H5Tpublic.h | 106 +- src/H5Zdevelop.h | 5 + src/H5Zpublic.h | 2 +- src/H5private.h | 8 + src/H5public.h | 21 +- test/API/README.md | 4 +- test/event_set.c | 4 +- testpar/t_subfiling_vfd.c | 739 +++++- tools/libtest/CMakeLists.txt | 2 +- tools/src/h5copy/CMakeLists.txt | 4 +- tools/src/h5diff/CMakeLists.txt | 8 +- tools/src/h5dump/CMakeLists.txt | 4 +- tools/src/h5format_convert/CMakeLists.txt | 4 +- tools/src/h5import/CMakeLists.txt | 4 +- tools/src/h5import/h5import.c | 84 +- tools/src/h5jam/CMakeLists.txt | 8 +- tools/src/h5ls/CMakeLists.txt | 4 +- tools/src/h5repack/CMakeLists.txt | 4 +- tools/src/h5stat/CMakeLists.txt | 4 +- tools/src/misc/CMakeLists.txt | 20 +- tools/test/h5copy/CMakeTests.cmake | 4 +- tools/test/h5jam/CMakeLists.txt | 6 +- tools/test/h5repack/CMakeLists.txt | 4 +- utils/mirror_vfd/mirror_writer.c | 1 - utils/tools/h5dwalk/CMakeLists.txt | 6 +- utils/tools/test/h5dwalk/CMakeLists.txt | 4 +- 83 files changed, 4030 insertions(+), 1365 deletions(-) delete mode 100644 HDF5Examples/FORTRAN/H5G/h5_version.h.in create mode 100644 HDF5Examples/FORTRAN/H5PAR/ph5_f90_filtered_writes_no_sel.F90 create mode 100644 HDF5Examples/FORTRAN/H5PAR/ph5_f90_subfiling.F90 diff --git a/.github/workflows/release-files.yml b/.github/workflows/release-files.yml index ce58214..732fa1e 100644 --- a/.github/workflows/release-files.yml +++ b/.github/workflows/release-files.yml @@ -174,7 +174,7 @@ jobs: ls ${{ runner.workspace }} - name: branch-only-docs - uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3 + uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ${{ github.workspace }}/${{ steps.get-file-base.outputs.FILE_BASE }}.doxygen diff --git a/HDF5Examples/CMakeLists.txt b/HDF5Examples/CMakeLists.txt index 6f8b53c..63adad6 100644 --- a/HDF5Examples/CMakeLists.txt +++ b/HDF5Examples/CMakeLists.txt @@ -159,7 +159,7 @@ if (${H5_LIBVER_DIR} GREATER 16) endif () configure_file (${H5EX_F90_SRC_DIR}/H5D/h5_version.h.in ${PROJECT_BINARY_DIR}/FORTRAN/H5D/h5_version.h @ONLY) - configure_file (${H5EX_F90_SRC_DIR}/H5G/h5_version.h.in ${PROJECT_BINARY_DIR}/FORTRAN/H5G/h5_version.h @ONLY) + configure_file (${H5EX_F90_SRC_DIR}/H5D/h5_version.h.in ${PROJECT_BINARY_DIR}/FORTRAN/H5G/h5_version.h @ONLY) else () set (HDF_BUILD_FORTRAN OFF CACHE BOOL "Build examples FORTRAN support" FORCE) endif () diff --git a/HDF5Examples/FORTRAN/H5G/h5_version.h.in b/HDF5Examples/FORTRAN/H5G/h5_version.h.in deleted file mode 100644 index 6827675..0000000 --- a/HDF5Examples/FORTRAN/H5G/h5_version.h.in +++ /dev/null @@ -1,23 +0,0 @@ -! Version numbers -! -! For major interface/format changes -! -#define H5_VERS_MAJOR @H5_VERS_MAJOR@ -! -! For minor interface/format changes -! -#define H5_VERS_MINOR @H5_VERS_MINOR@ -! -! For tweaks, bug-fixes, or development -! -#define H5_VERS_RELEASE @H5_VERS_RELEASE@ - -! macros for comparing versions - -#define H5_VERSION_GE(Maj, Min, Rel) \ - (((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR == Min) && (H5_VERS_RELEASE >= Rel)) || \ - ((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR > Min)) || (H5_VERS_MAJOR > Maj)) - -#define H5_VERSION_LE(Maj, Min, Rel) \ - (((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR == Min) && (H5_VERS_RELEASE <= Rel)) || \ - ((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR < Min)) || (H5_VERS_MAJOR < Maj)) diff --git a/HDF5Examples/FORTRAN/H5PAR/CMakeLists.txt b/HDF5Examples/FORTRAN/H5PAR/CMakeLists.txt index d3124a1..866f3ef 100644 --- a/HDF5Examples/FORTRAN/H5PAR/CMakeLists.txt +++ b/HDF5Examples/FORTRAN/H5PAR/CMakeLists.txt @@ -12,6 +12,7 @@ project (HDF5Examples_FORTRAN_H5PAR Fortran) INCLUDE_DIRECTORIES ( ${CMAKE_Fortran_MODULE_DIRECTORY}${HDF_MOD_EXT} ${PROJECT_BINARY_DIR} + ${HDF5_F90_BINARY_DIR} ${CMAKE_LIBRARY_OUTPUT_DIRECTORY} ) diff --git a/HDF5Examples/FORTRAN/H5PAR/Fortran_sourcefiles.cmake b/HDF5Examples/FORTRAN/H5PAR/Fortran_sourcefiles.cmake index 39c8940..af2bb57 100644 --- a/HDF5Examples/FORTRAN/H5PAR/Fortran_sourcefiles.cmake +++ b/HDF5Examples/FORTRAN/H5PAR/Fortran_sourcefiles.cmake @@ -9,3 +9,14 @@ set (examples ph5_f90_hyperslab_by_pattern ph5_f90_hyperslab_by_chunk ) + +if (HDF5_ENABLE_SUBFILING_VFD) + set (examples ${examples} + ph5_f90_subfiling + ) +endif() +if (HDF5_VERSION_STRING VERSION_GREATER_EQUAL "1.14.4") + set (examples ${examples} + ph5_f90_filtered_writes_no_sel + ) +endif() diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_dataset.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_dataset.F90 index 9819ab3..f7e4185 100644 --- a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_dataset.F90 +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_dataset.F90 @@ -1,10 +1,9 @@ PROGRAM DATASET USE HDF5 ! This module contains all necessary modules + USE MPI IMPLICIT NONE - - INCLUDE 'mpif.h' CHARACTER(LEN=10), PARAMETER :: filename = "sds.h5" ! File name CHARACTER(LEN=8), PARAMETER :: dsetname = "IntArray" ! Dataset name diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_file_create.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_file_create.F90 index 7944b5a..b5aa090 100644 --- a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_file_create.F90 +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_file_create.F90 @@ -5,10 +5,10 @@ PROGRAM FILE_CREATE USE HDF5 ! This module contains all necessary modules + USE MPI IMPLICIT NONE - INCLUDE 'mpif.h' CHARACTER(LEN=10), PARAMETER :: filename = "sds.h5" ! File name INTEGER(HID_T) :: file_id ! File identifier diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_filtered_writes_no_sel.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_filtered_writes_no_sel.F90 new file mode 100644 index 0000000..ffec2fb --- /dev/null +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_filtered_writes_no_sel.F90 @@ -0,0 +1,354 @@ +! +! Example of using the parallel HDF5 library to collectively write to +! datasets with filters applied to them when one or MPI ranks do not +! have data to contribute to the dataset. +! +! If the HDF5_NOCLEANUP environment variable is set, the file that +! this example creates will not be removed as the example finishes. +! +! The need of requirement of parallel file prefix is that in general +! the current working directory in which compiling is done, is not suitable +! for parallel I/O and there is no standard pathname for parallel file +! systems. In some cases, the parallel file name may even need some +! parallel file type prefix such as: "pfs:/GF/...". Therefore, this +! example parses the HDF5_PARAPREFIX environment variable for a prefix, +! if one is needed. + +MODULE filter + USE HDF5 + USE MPI + + IMPLICIT NONE + + CHARACTER(LEN=29), PARAMETER :: EXAMPLE_FILE = "ph5_filtered_writes_no_sel.h5" + INTEGER , PARAMETER :: EXAMPLE_DSET_DIMS = 2 + CHARACTER(LEN=4) , PARAMETER :: EXAMPLE_DSET_NAME = "DSET" + INTEGER , PARAMETER :: EXAMPLE_DSET_CHUNK_DIM_SIZE = 10 + INTEGER , PARAMETER :: PATH_MAX = 512 + + ! Global variables + INTEGER :: mpi_rank, mpi_size + +CONTAINS + ! + ! Routine to set an HDF5 filter on the given DCPL + ! + SUBROUTINE set_filter(dcpl_id) + + IMPLICIT NONE + INTEGER(HID_T) :: dcpl_id + LOGICAL :: filter_avail + INTEGER :: status + + ! + ! Check if 'deflate' filter is available + ! + CALL H5Zfilter_avail_f(H5Z_FILTER_DEFLATE_F, filter_avail, status) + IF(status .LT. 0)THEN + RETURN + ELSE IF(filter_avail)THEN + ! + ! Set 'deflate' filter with reasonable + ! compression level on DCPL + + CALL H5Pset_deflate_f(dcpl_id, 6, status) + ELSE + ! + ! Set Fletcher32 checksum filter on DCPL + ! since it is always available in HDF5 + CALL H5Pset_fletcher32_f(dcpl_id, status) + ENDIF + END SUBROUTINE set_filter + ! + ! Routine to fill a data buffer with data. Assumes + ! dimension rank is 2 and data is stored contiguous. + + + SUBROUTINE fill_databuf(start, count, stride, wdata) + + IMPLICIT NONE + INTEGER(HSIZE_T), DIMENSION(*) :: start, count, stride + INTEGER, DIMENSION(*) :: wdata + INTEGER(HSIZE_T) :: i, j, icnt + + ! Use MPI rank value for data + icnt = 1 + DO i = 1, COUNT(1) + DO j = 1, COUNT(2) + wdata(icnt) = mpi_rank + icnt = icnt + 1 + ENDDO + ENDDO + + END SUBROUTINE fill_databuf + ! + ! Cleanup created files + ! + SUBROUTINE cleanup(filename) + + IMPLICIT NONE + CHARACTER(*) :: filename + + LOGICAL :: do_cleanup + INTEGER :: status + + CALL get_environment_variable("HDF5_NOCLEANUP", STATUS=status) + IF(status.EQ.0)THEN + CALL MPI_File_delete(filename, MPI_INFO_NULL, status) + ENDIF + + END SUBROUTINE cleanup + ! + ! Routine to write to a dataset in a fashion + ! where no chunks in the dataset are written + ! to by more than 1 MPI rank. This will + ! generally give the best performance as the + ! MPI ranks will need the least amount of + ! inter-process communication. + + SUBROUTINE write_dataset_some_no_sel(file_id, dxpl_id) + + IMPLICIT NONE + INTEGER(HID_T) :: file_id, dxpl_id + + INTEGER, DIMENSION(1:EXAMPLE_DSET_CHUNK_DIM_SIZE, 4*EXAMPLE_DSET_CHUNK_DIM_SIZE), TARGET :: wdata + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: dataset_dims + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: chunk_dims + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: start + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: stride + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: count + LOGICAL :: no_selection = .FALSE. + INTEGER(hid_t) :: dset_id + INTEGER(hid_t) :: dcpl_id + INTEGER(hid_t) :: file_dataspace + INTEGER(hid_t) :: sel_type + TYPE(C_PTR) :: f_ptr + INTEGER :: status + + ! + ! ------------------------------------ + ! Setup Dataset Creation Property List + ! ------------------------------------ + + CALL H5Pcreate_f(H5P_DATASET_CREATE_F, dcpl_id, status) + + ! + ! REQUIRED: Dataset chunking must be enabled to + ! apply a data filter to the dataset. + ! Chunks in the dataset are of size + ! EXAMPLE_DSET_CHUNK_DIM_SIZE x EXAMPLE_DSET_CHUNK_DIM_SIZE. + + chunk_dims(1) = EXAMPLE_DSET_CHUNK_DIM_SIZE + chunk_dims(2) = EXAMPLE_DSET_CHUNK_DIM_SIZE + CALL H5Pset_chunk_f(dcpl_id, EXAMPLE_DSET_DIMS, chunk_dims, status) + + ! Set filter to be applied to created datasets + CALL set_filter(dcpl_id) + + ! + ! ------------------------------------ + ! Define the dimensions of the dataset + ! and create it + ! ------------------------------------ + + ! Create a dataset composed of 4 chunks + ! per MPI rank. The first dataset dimension + ! scales according to the number of MPI ranks. + ! The second dataset dimension stays fixed + ! according to the chunk size. + + dataset_dims(1) = EXAMPLE_DSET_CHUNK_DIM_SIZE * mpi_size + dataset_dims(2) = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE + + CALL H5Screate_simple_f(EXAMPLE_DSET_DIMS, dataset_dims, file_dataspace, status) + + ! Create the dataset + CALL H5Dcreate_f(file_id, EXAMPLE_DSET_NAME, H5T_NATIVE_INTEGER, file_dataspace, dset_id, status, dcpl_id=dcpl_id) + + ! + ! ------------------------------------ + ! Setup selection in the dataset for + ! each MPI rank + ! ------------------------------------ + + ! + ! Odd rank value MPI ranks do not + ! contribute any data to the dataset. + + IF(MOD(mpi_rank, 2) .NE. 0) no_selection = .TRUE. + + IF(no_selection)THEN + ! + ! MPI ranks not contributing data to + ! the dataset should call H5Sselect_none + ! on the file dataspace that will be + ! passed to H5Dwrite. + + CALL H5Sselect_none_f(file_dataspace, status) + sel_type = H5S_BLOCK_F + ELSE + ! + ! Even MPI ranks contribute data to + ! the dataset. Each MPI rank's selection + ! covers a single chunk in the first dataset + ! dimension. Each MPI rank's selection + ! covers 4 chunks in the second dataset + ! dimension. This leads to each contributing + ! MPI rank writing to 4 chunks of the dataset. + + start(1) = mpi_rank * EXAMPLE_DSET_CHUNK_DIM_SIZE + start(2) = 0 + stride(1) = 1 + stride(2) = 1 + count(1) = EXAMPLE_DSET_CHUNK_DIM_SIZE + count(2) = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE + + CALL H5Sselect_hyperslab_f(file_dataspace, H5S_SELECT_SET_F, start, count, status, stride=stride) + + sel_type = H5S_ALL_F + ! + ! -------------------------------------- + ! Fill data buffer with MPI rank's rank + ! value to make it easy to see which + ! part of the dataset each rank wrote to + ! -------------------------------------- + + CALL fill_databuf(start, count, stride, wdata) + ENDIF + + ! + ! --------------------------------- + ! Write to the dataset collectively + ! --------------------------------- + f_ptr = C_LOC(wdata) + CALL H5Dwrite_f(dset_id, H5T_NATIVE_INTEGER, f_ptr, status, & + mem_space_id=sel_type, file_space_id=file_dataspace, xfer_prp=dxpl_id) + + ! + ! -------------- + ! Close HDF5 IDs + ! -------------- + + CALL H5Sclose_f(file_dataspace,status) + CALL H5Pclose_f(dcpl_id,status) + CALL H5Dclose_f(dset_id,status) + + END SUBROUTINE write_dataset_some_no_sel + END MODULE filter + + PROGRAM main + + USE filter + IMPLICIT NONE + + INTEGER :: comm = MPI_COMM_WORLD + INTEGER :: info = MPI_INFO_NULL + INTEGER(hid_t) :: file_id + INTEGER(hid_t) :: fapl_id + INTEGER(hid_t) :: dxpl_id + CHARACTER(LEN=PATH_MAX) :: par_prefix + CHARACTER(LEN=PATH_MAX) :: filename + INTEGER :: status + + CALL MPI_Init(status) + CALL MPI_Comm_size(comm, mpi_size, status) + CALL MPI_Comm_rank(comm, mpi_rank, status) + + ! + ! Initialize HDF5 library and Fortran interfaces. + ! + CALL h5open_f(status) + ! + ! ---------------------------------- + ! Start parallel access to HDF5 file + ! ---------------------------------- + + ! Setup File Access Property List with parallel I/O access + CALL H5Pcreate_f(H5P_FILE_ACCESS_F, fapl_id, status) + CALL H5Pset_fapl_mpio_f(fapl_id, comm, info, status) + + ! + ! OPTIONAL: Set collective metadata reads on FAPL to allow + ! parallel writes to filtered datasets to perform + ! better at scale. While not strictly necessary, + ! this is generally recommended. + + CALL H5Pset_all_coll_metadata_ops_f(fapl_id, .TRUE., status) + + ! + ! OPTIONAL: Set the latest file format version for HDF5 in + ! order to gain access to different dataset chunk + ! index types and better data encoding methods. + ! While not strictly necessary, this is generally + ! recommended. + + CALL H5Pset_libver_bounds_f(fapl_id, H5F_LIBVER_LATEST_F, H5F_LIBVER_LATEST_F, status) + + ! Parse any parallel prefix and create filename + par_prefix(:) = "" + CALL get_environment_variable("HDF5_PARAPREFIX", VALUE=par_prefix, STATUS=status) + filename = TRIM(par_prefix)//EXAMPLE_FILE + + ! Create HDF5 file + CALL H5Fcreate_f(filename, H5F_ACC_TRUNC_F, file_id, status, access_prp = fapl_id) + + ! + ! -------------------------------------- + ! Setup Dataset Transfer Property List + ! with collective I/O + ! -------------------------------------- + + + CALL H5Pcreate_f(H5P_DATASET_XFER_F, dxpl_id, status) + + ! + ! REQUIRED: Setup collective I/O for the dataset + ! write operations. Parallel writes to + ! filtered datasets MUST be collective, + ! even if some ranks have no data to + ! contribute to the write operation. + + CALL H5Pset_dxpl_mpio_f(dxpl_id, H5FD_MPIO_COLLECTIVE_F, status) + + ! + ! -------------------------------- + ! Create and write to the dataset + ! -------------------------------- + + ! + ! Write to a dataset in a fashion where no + ! chunks in the dataset are written to by + ! more than 1 MPI rank and some MPI ranks + ! have nothing to contribute to the dataset. + ! In this case, the MPI ranks that have no + ! data to contribute must still participate + ! in the collective H5Dwrite call, but should + ! call H5Sselect_none on the file dataspace + ! passed to the H5Dwrite call. + + CALL write_dataset_some_no_sel(file_id, dxpl_id) + + ! + ! ------------------ + ! Close all HDF5 IDs + ! ------------------ + + CALL H5Pclose_f(dxpl_id, status) + CALL H5Pclose_f(fapl_id, status) + CALL H5Fclose_f(file_id, status) + ! + ! Close FORTRAN interfaces and HDF5 library. + ! + CALL h5close_f(status) + + IF(mpi_rank .EQ. 0) WRITE(*,"(A)") "PHDF5 example finished with no errors" + + ! + ! ------------------------------------ + ! Cleanup created HDF5 file and finish + ! ------------------------------------ + CALL cleanup(filename) + + CALL MPI_Finalize(status) + +END PROGRAM main diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_chunk.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_chunk.F90 index c74e55d..7be9389 100644 --- a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_chunk.F90 +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_chunk.F90 @@ -4,11 +4,10 @@ PROGRAM DATASET_BY_CHUNK USE HDF5 ! This module contains all necessary modules -! USE MPI + USE MPI IMPLICIT NONE - include 'mpif.h' CHARACTER(LEN=11), PARAMETER :: filename = "sds_chnk.h5" ! File name CHARACTER(LEN=8), PARAMETER :: dsetname = "IntArray" ! Dataset name diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_col.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_col.F90 index dc92667..affb799 100644 --- a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_col.F90 +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_col.F90 @@ -1,7 +1,6 @@ ! -! Number of processes is assumed to be 1 or multiples of 2 (1,2,4,6,8) +! Number of processes is assumed to be 1 or powers of 2 (2,4,8) ! - PROGRAM DATASET_BY_COL USE HDF5 ! This module contains all necessary modules diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_pattern.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_pattern.F90 index dd02c63..c7e8da1 100644 --- a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_pattern.F90 +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_pattern.F90 @@ -5,10 +5,10 @@ PROGRAM DATASET_BY_PATTERN USE HDF5 ! This module contains all necessary modules + USE MPI IMPLICIT NONE - include 'mpif.h' CHARACTER(LEN=10), PARAMETER :: filename = "sds_pat.h5" ! File name CHARACTER(LEN=8), PARAMETER :: dsetname = "IntArray" ! Dataset name diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_row.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_row.F90 index f66da2a..66d5b25 100644 --- a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_row.F90 +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_hyperslab_by_row.F90 @@ -4,10 +4,10 @@ PROGRAM DATASET_BY_ROW USE HDF5 ! This module contains all necessary modules + USE MPI IMPLICIT NONE - include 'mpif.h' CHARACTER(LEN=10), PARAMETER :: filename = "sds_row.h5" ! File name CHARACTER(LEN=8), PARAMETER :: dsetname = "IntArray" ! Dataset name diff --git a/HDF5Examples/FORTRAN/H5PAR/ph5_f90_subfiling.F90 b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_subfiling.F90 new file mode 100644 index 0000000..fc30717 --- /dev/null +++ b/HDF5Examples/FORTRAN/H5PAR/ph5_f90_subfiling.F90 @@ -0,0 +1,521 @@ +! +! Example of using HDF5's Subfiling VFD to write to an +! HDF5 file that is striped across multiple subfiles +! +! If the HDF5_NOCLEANUP environment variable is set, the +! files that this example creates will not be removed as +! the example finishes. +! +! In general, the current working directory in which compiling +! is done, is not suitable for parallel I/O and there is no +! standard pathname for parallel file systems. In some cases, +! the parallel file name may even need some parallel file type +! prefix such as: "pfs:/GF/...". Therefore, this example parses +! the HDF5_PARAPREFIX environment variable for a prefix, if one +! is needed. +! + +MODULE subf + + USE HDF5 + USE MPI + + CHARACTER(LEN=31), PARAMETER :: EXAMPLE_FILE = "h5_subfiling_default_example.h5" + CHARACTER(LEN=30), PARAMETER :: EXAMPLE_FILE2 = "h5_subfiling_custom_example.h5" + CHARACTER(LEN=33), PARAMETER :: EXAMPLE_FILE3 = "h5_subfiling_precreate_example.h5" + + CHARACTER(LEN=4), PARAMETER :: EXAMPLE_DSET_NAME = "DSET" + INTEGER , PARAMETER :: EXAMPLE_DSET_DIMS = 2 + + ! Have each MPI rank write 16MiB of data + INTEGER, PARAMETER :: EXAMPLE_DSET_NY = 4194304 + +CONTAINS + + ! Cleanup created files + + SUBROUTINE cleanup(filename, fapl_id) + + IMPLICIT NONE + INTEGER(HID_T) :: fapl_id + CHARACTER(*) :: filename + + LOGICAL :: do_cleanup + INTEGER :: status + + CALL get_environment_variable("HDF5_NOCLEANUP", STATUS=status) + !IF(status.EQ.0) CALL H5Fdelete_f(filename, fapl_id, status) + IF(status.EQ.0)THEN + OPEN(UNIT=15, IOSTAT=status, FILE=filename, STATUS='old') + IF(status .EQ. 0) CLOSE(15, STATUS='DELETE') + ENDIF + + END SUBROUTINE cleanup + + ! An example of using the HDF5 Subfiling VFD with + ! its default settings of 1 subfile per node, with + ! a stripe size of 32MiB + + SUBROUTINE subfiling_write_default(fapl_id, mpi_size, mpi_rank) + + IMPLICIT NONE + INTEGER(HID_T) :: fapl_id + INTEGER :: mpi_size + INTEGER :: mpi_rank + + INTEGER, DIMENSION(:), ALLOCATABLE, TARGET :: wdata + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: dset_dims + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: start + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: count + INTEGER(hid_t) :: file_id + INTEGER(hid_t) :: subfiling_fapl + INTEGER(hid_t) :: dset_id + INTEGER(hid_t) :: filespace + CHARACTER(LEN=512) :: filename, par_prefix + INTEGER :: status + INTEGER(SIZE_T) :: i + TYPE(C_PTR) :: f_ptr + + ! + ! Make a copy of the FAPL so we don't disturb + ! it for the other examples + ! + CALL H5Pcopy_f(fapl_id, subfiling_fapl, status) + + ! + ! Set Subfiling VFD on FAPL using default settings + ! (use IOC VFD, 1 IOC per node, 32MiB stripe size) + ! + ! Note that all of Subfiling's configuration settings + ! can be adjusted with environment variables as well + ! in this case. + ! + + CALL H5Pset_fapl_subfiling_f(subfiling_fapl, status) + + ! + ! OPTIONAL: Set alignment of objects in HDF5 file to + ! be equal to the Subfiling stripe size. + ! Choosing a Subfiling stripe size and HDF5 + ! object alignment value that are some + ! multiple of the disk block size can + ! generally help performance by ensuring + ! that I/O is well-aligned and doesn't + ! excessively cross stripe boundaries. + ! + ! Note that this option can substantially + ! increase the size of the resulting HDF5 + ! files, so it is a good idea to keep an eye + ! on this. + ! + + CALL H5Pset_alignment_f(subfiling_fapl, 0_HSIZE_T, 33554432_HSIZE_T, status) ! ALIGN to default 32MiB stripe size + + ! Parse any parallel prefix and create filename + par_prefix(:) = "" + CALL get_environment_variable("HDF5_PARAPREFIX", VALUE=par_prefix, STATUS=status) + filename = TRIM(par_prefix)//EXAMPLE_FILE + + ! Create a new file collectively + CALL H5Fcreate_f(filename, H5F_ACC_TRUNC_F, file_id, status, access_prp = subfiling_fapl) + + ! Create the dataspace for the dataset. The second + ! dimension varies with the number of MPI ranks + ! while the first dimension is fixed. + + dset_dims(1) = EXAMPLE_DSET_NY + dset_dims(2) = mpi_size + CALL H5Screate_simple_f(EXAMPLE_DSET_DIMS, dset_dims, filespace, status) + + ! Create the dataset with default properties + + CALL H5Dcreate_f(file_id, EXAMPLE_DSET_NAME, H5T_NATIVE_INTEGER, filespace, dset_id, status) + ! Each MPI rank writes from a contiguous memory + ! region to the hyperslab in the file + + start(1) = 0 + start(2) = mpi_rank + count(1) = dset_dims(1) + count(2) = 1 + CALL H5Sselect_hyperslab_f(filespace, H5S_SELECT_SET_F, start, count, status) + + ! Initialize data buffer + ALLOCATE(wdata(COUNT(1)*COUNT(2))) + DO i = 1, COUNT(1)*COUNT(2) + wdata(i) = mpi_rank + ENDDO + + ! Write to dataset + f_ptr = C_LOC(wdata) + CALL H5Dwrite_f(dset_id, H5T_NATIVE_INTEGER, f_ptr, status, mem_space_id=H5S_BLOCK_F, file_space_id=filespace) + + ! Close/release resources. + DEALLOCATE(wdata) + CALL H5Dclose_f(dset_id, status) + CALL H5Sclose_f(filespace, status) + + CALL H5Fclose_f(file_id, status) + + CALL cleanup(EXAMPLE_FILE, subfiling_fapl) + + CALL H5Pclose_f(subfiling_fapl, status) + + END SUBROUTINE subfiling_write_default + + ! + ! An example of using the HDF5 Subfiling VFD with + ! custom settings + ! + + SUBROUTINE subfiling_write_custom(fapl_id, mpi_size, mpi_rank) + + IMPLICIT NONE + INTEGER(HID_T) :: fapl_id + INTEGER :: mpi_size + INTEGER :: mpi_rank + + INTEGER, DIMENSION(:), ALLOCATABLE, TARGET :: wdata + + TYPE(H5FD_subfiling_config_t) :: subf_config + TYPE(H5FD_ioc_config_t) :: ioc_config + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: dset_dims + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: start + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: count + INTEGER(hid_t) :: file_id + INTEGER(hid_t) :: subfiling_fapl + INTEGER(hid_t) :: dset_id + INTEGER(hid_t) :: filespace + CHARACTER(LEN=512) :: filename, par_prefix + INTEGER :: status + INTEGER(SIZE_T) :: i + TYPE(C_PTR) :: f_ptr + + ! Make a copy of the FAPL so we don't disturb + ! it for the other examples + + CALL H5Pcopy_f(fapl_id, subfiling_fapl, status) + + ! Get a default Subfiling and IOC configuration + CALL h5pget_fapl_subfiling_f(subfiling_fapl, subf_config, status) + CALL h5pget_fapl_ioc_f(subfiling_fapl,ioc_config, status) + + ! Set Subfiling configuration to use a 1MiB + ! stripe size and the SELECT_IOC_EVERY_NTH_RANK + ! selection method. By default, without a setting + ! in the H5FD_SUBFILING_IOC_SELECTION_CRITERIA + ! environment variable, this will use every MPI + ! rank as an I/O concentrator. + + subf_config%shared_cfg%stripe_size = 1048576 + subf_config%shared_cfg%ioc_selection = SELECT_IOC_EVERY_NTH_RANK_F + + ! Set IOC configuration to use 2 worker threads + ! per IOC instead of the default setting and + ! update IOC configuration with new subfiling + ! configuration. + + ioc_config%thread_pool_size = 2 + + ! Set our new configuration on the IOC + ! FAPL used for Subfiling + + CALL H5Pset_fapl_ioc_f(subf_config%ioc_fapl_id, status, ioc_config) + + ! Finally, set our new Subfiling configuration + ! on the original FAPL + + CALL H5Pset_fapl_subfiling_f(subfiling_fapl, status, subf_config) + ! + ! OPTIONAL: Set alignment of objects in HDF5 file to + ! be equal to the Subfiling stripe size. + ! Choosing a Subfiling stripe size and HDF5 + ! object alignment value that are some + ! multiple of the disk block size can + ! generally help performance by ensuring + ! that I/O is well-aligned and doesn't + ! excessively cross stripe boundaries. + ! + ! Note that this option can substantially + ! increase the size of the resulting HDF5 + ! files, so it is a good idea to keep an eye + ! on this. + ! + + CALL H5Pset_alignment_f(subfiling_fapl, 0_HSIZE_T, 33554432_HSIZE_T, status) ! ALIGN to default 32MiB stripe size + + ! Parse any parallel prefix and create filename + par_prefix(:) = "" + CALL get_environment_variable("HDF5_PARAPREFIX", VALUE=par_prefix, STATUS=status) + filename = TRIM(par_prefix)//EXAMPLE_FILE + + ! Create a new file collectively + CALL H5Fcreate_f(filename, H5F_ACC_TRUNC_F, file_id, status, access_prp = subfiling_fapl) + + ! Create the dataspace for the dataset. The second + ! dimension varies with the number of MPI ranks + ! while the first dimension is fixed. + + dset_dims(1) = EXAMPLE_DSET_NY + dset_dims(2) = mpi_size + CALL H5Screate_simple_f(EXAMPLE_DSET_DIMS, dset_dims, filespace, status) + + ! Create the dataset with default properties + + CALL H5Dcreate_f(file_id, EXAMPLE_DSET_NAME, H5T_NATIVE_INTEGER, filespace, dset_id, status) + ! Each MPI rank writes from a contiguous memory + ! region to the hyperslab in the file + + start(1) = 0 + start(2) = mpi_rank + count(1) = dset_dims(1) + count(2) = 1 + CALL H5Sselect_hyperslab_f(filespace, H5S_SELECT_SET_F, start, count, status) + + ! Initialize data buffer + ALLOCATE(wdata(COUNT(1)*COUNT(2))) + DO i = 1, COUNT(1)*COUNT(2) + wdata(i) = mpi_rank + ENDDO + + ! Write to dataset + f_ptr = C_LOC(wdata) + CALL H5Dwrite_f(dset_id, H5T_NATIVE_INTEGER, f_ptr, status, mem_space_id=H5S_BLOCK_F, file_space_id=filespace) + + ! Close/release resources. + DEALLOCATE(wdata) + CALL H5Dclose_f(dset_id, status) + CALL H5Sclose_f(filespace, status) + + CALL H5Fclose_f(file_id, status) + + CALL cleanup(EXAMPLE_FILE, subfiling_fapl) + + CALL H5Pclose_f(subfiling_fapl, status) + + END SUBROUTINE subfiling_write_custom + + ! + ! An example of pre-creating an HDF5 file on MPI rank + ! 0 when using the HDF5 Subfiling VFD. In this case, + ! the subfiling stripe count must be set so that rank + ! 0 knows how many subfiles to pre-create. + + SUBROUTINE subfiling_write_precreate(fapl_id, mpi_size, mpi_rank) + + IMPLICIT NONE + INTEGER(HID_T) :: fapl_id + INTEGER :: mpi_size + INTEGER :: mpi_rank + + INTEGER, DIMENSION(:), ALLOCATABLE, TARGET :: wdata + TYPE(H5FD_subfiling_config_t) :: subf_config + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: dset_dims + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: start + INTEGER(hsize_t), DIMENSION(1:EXAMPLE_DSET_DIMS) :: count + INTEGER(hid_t) :: file_id + INTEGER(hid_t) :: subfiling_fapl + INTEGER(hid_t) :: dset_id + INTEGER(hid_t) :: filespace + CHARACTER(LEN=512) :: filename, par_prefix + INTEGER :: status + INTEGER(SIZE_T) :: i + TYPE(C_PTR) :: f_ptr + + ! Make a copy of the FAPL so we don't disturb + ! it for the other examples + + CALL H5Pcopy_f(fapl_id, subfiling_fapl, status) + + ! Get a default Subfiling and IOC configuration + CALL h5pget_fapl_subfiling_f(subfiling_fapl, subf_config, status) + + ! + ! Set the Subfiling stripe count so that rank + ! 0 knows how many subfiles the logical HDF5 + ! file should consist of. In this case, use + ! 5 subfiles with a default stripe size of + ! 32MiB. + + subf_config%shared_cfg%stripe_count = 5 + ! + ! OPTIONAL: Set alignment of objects in HDF5 file to + ! be equal to the Subfiling stripe size. + ! Choosing a Subfiling stripe size and HDF5 + ! object alignment value that are some + ! multiple of the disk block size can + ! generally help performance by ensuring + ! that I/O is well-aligned and doesn't + ! excessively cross stripe boundaries. + ! + ! Note that this option can substantially + ! increase the size of the resulting HDF5 + ! files, so it is a good idea to keep an eye + ! on this. + ! + + CALL H5Pset_alignment_f(subfiling_fapl, 0_HSIZE_T, 1048576_HSIZE_T, status) ! Align to custom 1MiB stripe size + + ! Parse any parallel prefix and create filename + par_prefix(:) = "" + CALL get_environment_variable("HDF5_PARAPREFIX", VALUE=par_prefix, STATUS=status) + filename = TRIM(par_prefix)//EXAMPLE_FILE + + ! Set dataset dimensionality + dset_dims(1) = EXAMPLE_DSET_NY + dset_dims(2) = mpi_size + + IF (mpi_rank .EQ. 0) THEN + ! + ! Make sure only this rank opens the file + ! + CALL H5Pset_mpi_params_f(subfiling_fapl, MPI_COMM_SELF, MPI_INFO_NULL, status) + + ! + ! Set the Subfiling VFD on our FAPL using + ! our custom configuration + ! + CALL H5Pset_fapl_subfiling_f(subfiling_fapl, status, subf_config); + + ! + ! Create a new file on rank 0 + ! + CALL H5Fcreate_f(filename, H5F_ACC_TRUNC_F, file_id, status, access_prp = subfiling_fapl) + + ! Create the dataspace for the dataset. The second + ! dimension varies with the number of MPI ranks + ! while the first dimension is fixed. + ! + CALL H5Screate_simple_f(EXAMPLE_DSET_DIMS, dset_dims, filespace, status) + + ! Create the dataset with default properties + + CALL H5Dcreate_f(file_id, EXAMPLE_DSET_NAME, H5T_NATIVE_INTEGER, filespace, dset_id, status) + + ! Initialize data buffer + ALLOCATE(wdata(dset_dims(1)*dset_dims(2))) + DO i = 1, dset_dims(1)*dset_dims(2) + wdata(i) = i + ENDDO + + ! + ! Rank 0 writes to the whole dataset + ! + f_ptr = C_LOC(wdata) + CALL H5Dwrite_f(dset_id, H5T_NATIVE_INTEGER, f_ptr, status, mem_space_id=H5S_BLOCK_F, file_space_id=filespace) + + ! + ! Close/release resources. + ! + DEALLOCATE(wdata) + CALL H5Dclose_f(dset_id, status) + CALL H5Sclose_f(filespace, status) + + CALL H5Fclose_f(file_id, status) + ENDIF + + CALL MPI_Barrier(MPI_COMM_WORLD, status) + + ! + ! Use all MPI ranks to re-open the file and + ! read back the dataset that was created + ! + CALL H5Pset_mpi_params_f(subfiling_fapl, MPI_COMM_WORLD, MPI_INFO_NULL, status) + + ! + ! Use the same subfiling configuration as rank 0 + ! used to create the file + ! + CALL H5Pset_fapl_subfiling_f(subfiling_fapl, status, subf_config) + + ! + ! Re-open the file on all ranks + ! + + CALL H5Fopen_f(filename, H5F_ACC_RDONLY_F, file_id, status, access_prp=subfiling_fapl) + + ! + ! Open the dataset that was created + ! + CALL H5Dopen_f(file_id, EXAMPLE_DSET_NAME, dset_id, status) + + ! + ! Initialize data buffer + ! + + ALLOCATE(wdata(dset_dims(1)*dset_dims(2))) + ! + ! Read the dataset on all ranks + ! + f_ptr = C_LOC(wdata) + CALL H5Dread_f(dset_id, H5T_NATIVE_INTEGER, f_ptr, status, mem_space_id=H5S_BLOCK_F, file_space_id=H5S_ALL_F) + + DEALLOCATE(wdata) + + CALL H5Dclose_f(dset_id, status) + CALL H5Fclose_f(file_id, status) + + CALL cleanup(EXAMPLE_FILE, subfiling_fapl) + + CALL H5Pclose_f(subfiling_fapl, status) + + END SUBROUTINE subfiling_write_precreate + +END MODULE subf + +PROGRAM main + + USE SUBF + IMPLICIT NONE + + INTEGER :: comm = MPI_COMM_WORLD + INTEGER :: info = MPI_INFO_NULL + INTEGER(HID_T) :: fapl_id + INTEGER :: mpi_size + INTEGER :: mpi_rank + INTEGER :: required + INTEGER :: provided + INTEGER :: status + + ! HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE + required = MPI_THREAD_MULTIPLE + provided = 0 + CALL mpi_init_thread(required, provided, status) + IF (provided .NE. required) THEN + WRITE(*,*) "MPI doesn't support MPI_Init_thread with MPI_THREAD_MULTIPLE *FAILED*" + CALL MPI_Abort(comm, -1, status) + ENDIF + + CALL MPI_Comm_size(comm, mpi_size, status) + CALL MPI_Comm_rank(comm, mpi_rank, status) + + ! + ! Initialize HDF5 library and Fortran interfaces. + ! + CALL h5open_f(status) + + ! + ! Set up File Access Property List with MPI + ! parameters for the Subfiling VFD to use + CALL h5pcreate_f(H5P_FILE_ACCESS_F, fapl_id, status) + CALL H5Pset_mpi_params_f(fapl_id, comm, info, status) + + ! Use Subfiling VFD with default settings + CALL subfiling_write_default(fapl_id, mpi_size, mpi_rank) + + ! Use Subfiling VFD with custom settings + CALL subfiling_write_custom(fapl_id, mpi_size, mpi_rank) + + ! Use Subfiling VFD to precreate the HDF5 file on MPI rank + CALL subfiling_write_precreate(fapl_id, mpi_size, mpi_rank) + + CALL H5Pclose_f(fapl_id, status) + ! + ! Close FORTRAN interfaces and HDF5 library. + ! + CALL h5close_f(status) + + IF(mpi_rank .EQ. 0) WRITE(*,"(A)") "PHDF5 example finished with no errors" + + CALL MPI_Finalize(status) + +END PROGRAM main diff --git a/README.md b/README.md index 2e4c83c..944b325 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,13 @@ DOCUMENTATION ------------- This release is fully functional for the API described in the documentation. - https://portal.hdfgroup.org/display/HDF5/The+HDF5+API + https://docs.hdfgroup.org/hdf5/v1_14/_l_b_a_p_i.html Full Documentation and Programming Resources for this release can be found at - https://portal.hdfgroup.org/display/HDF5 + https://docs.hdfgroup.org/hdf5/v1_14/index.html -The latest doxygen documentation generated on changes to develop is available at: +The latest doxygen documentation generated on changes to HDF5 1.14.x is available at: https://hdfgroup.github.io/hdf5/ @@ -47,7 +47,7 @@ HELP AND SUPPORT ---------------- Information regarding Help Desk and Support services is available at - https://portal.hdfgroup.org/display/support/The+HDF+Help+Desk + https://help.hdfgroup.org @@ -77,7 +77,7 @@ Periodically development code snapshots are provided at the following URL: Source packages for current and previous releases are located at: - https://portal.hdfgroup.org/display/support/Downloads + https://portal.hdfgroup.org/Downloads Development code is available at our Github location: diff --git a/bin/warnhist b/bin/warnhist index 1e63a55..b77202f 100755 --- a/bin/warnhist +++ b/bin/warnhist @@ -22,8 +22,13 @@ use warnings; # Perl modules/settings use strict; use Getopt::Std; +use File::Find; +use File::Basename; +use Cwd; +#use Data::Dumper; # Global variables, for accumulating information +my %options=(); my $totalcount = 0; my $notecount = 0; my $dupcount = 0; @@ -54,11 +59,25 @@ my $last_fort_name; my $last_fort_line; my $last_fort_offset; +# Info about source files +my %c_files; +my %c_files_counted; +my %cpp_files; +my %cpp_files_counted; +my %fort_files; +my %fort_files_counted; +my $c_lines = 0; +my $cpp_lines = 0; +my $fort_lines = 0; + # Display usage sub do_help { - print "Usage: 'warnhist [-h, --help] [-t ] [-w ] [-W] [-f ] [-F] [-s ] [-S ] [file]'\n"; - print "\t-h, --help\tDisplay this usage\n"; - print "\t-t \tTrim pathname prefix from filenames, \n"; + print "Usage: 'warnhist [-h, --help] [-t ] [-w ] [-W] [-f ] [-F]\n"; + print "\t[-s ] [-S ] [-d] [-p Trim pathname prefix from filenames, \n"; print "\t-w \tDisplay files for a given warning index list, \n"; print "\t\t can be a single value, a range, or a comma separated list\n"; print "\t\tFor example: '0' or '0,4' or '8-10' or '0,2-4,8-10,13'\n"; @@ -67,10 +86,13 @@ sub do_help { print "\t\t can be a single value, a range, or a comma separated list\n"; print "\t\tFor example: '0' or '0,4' or '8-10' or '0,2-4,8-10,13'\n"; print "\t-F\tDisplay warnings for all files\n"; - print "\t-s \tDisplay files for warnings which contain a string, \n"; + print "\t-s Display files for warnings which contain a\n"; + print "\t\tstring, \n"; print "\t\t is a comma separated list, with no spaces\n"; - print "\t\tFor example: 'Wunused-dummy-argument' or 'Wunused-dummy-argument,Wunused-variable'\n"; - print "\t-S \tDisplay warnings for files which contain a string, \n"; + print "\t\tFor example: 'Wunused-dummy-argument' or\n"; + print "\t\t'Wunused-dummy-argument,Wunused-variable'\n"; + print "\t-S Display warnings for files which contain a\n"; + print "\t\tstring, \n"; print "\t\t is a comma separated list, with no spaces\n"; print "\t\tFor example: 'H5Fint' or 'H5Fint,H5Gnode'\n"; print "\t-l\tDisplay line numbers for file/warning\n"; @@ -78,18 +100,182 @@ sub do_help { print "\t-i \tIgnore named files, \n"; print "\t\t is a comma separated list, with no spaces\n"; print "\t\tFor example: 'H5LTparse' or 'H5LTparse,H5LTanalyze'\n"; + print "\t-d\tCompute warning density for compiled source files. Paths to the\n"; + print "\t\troot of a directory containing source may be provided with the\n"; + print "\t\t'-p ' option. If the path list is given, only those\n"; + print "\t\tdirectories are scanned for source files. If the path list\n"; + print "\t\toption is not given, the current working directory is scanned.\n"; + print "\t-p \tPaths to search for compiled files. Compiled files\n"; + print "\t\tare only used when computing warning density and are not\n"; + print "\t\tnecessary for just analyzing warnings in build output.\n"; + print "\t\t is a comma separated list, with no spaces\n"; + print "\t\tFor example: '/home/koziol/hdf5' or '.,~/dev/hdf5,~/dev/build'\n"; + print "\t-q\tSuppress warning output\n"; print "\tfile\tFilename containing build output\n"; print "\t\tIf no file is given, standard input is used.\n"; exit; } +# Count # of lines in a file +sub line_count { + my ($name) = @_; +#print "name = '$name'\n"; + my $tmp; + my $lines = 0; + + open (FILE, $name) or die "Can't open '$name': $!"; + $lines++ while ($tmp = ); + close FILE; +#print "$lines\n"; + + return $lines; +} + +# Recursively search a directory hierarchy for source files +# Adds results to the global %c_files, %cpp_files, and %fort_files hashes +sub parse_tree { + my ($root_path) = @_; +#print "root_path = $root_path\n"; + + my $path_checker = sub { + my $name = $File::Find::name; + if (-f $name) { + my $bn = basename($name); + + # Check for different kinds of source files + # Use lists here: https://gist.github.com/ppisarczyk/43962d06686722d26d176fad46879d41#file-programming_languages_extensions-json + + # FORTRAN source file + if($bn =~ /.*(\.f90)|(\.f)|(\.f03)|(\.f08)|(\.f77)|(\.f95)|(\.for)|(\.fpp)$/i) { + $bn =~ s/(\.f90)|(\.f)|(\.f03)|(\.f08)|(\.f77)|(\.f95)|(\.for)|(\.fpp)$//ig; + if(!exists $fort_files{$bn}) { + $fort_files{$bn} = [ $name ]; + } else { + push @{ $fort_files{$bn} }, $name; + } + # C++ source file + } elsif($bn =~ /.*(\.cpp)|(\.c\+\+)|(\.cc)|(\.cp)|(\.cxx)$/i) { + $bn =~ s/(\.cpp)|(\.c\+\+)|(\.cc)|(\.cp)|(\.cxx)$//ig; + if(!exists $cpp_files{$bn}) { + $cpp_files{$bn} = [ $name ]; + } else { + push @{ $cpp_files{$bn} }, $name; + } + # C source file + } elsif($bn =~ /.*(\.c)$/i) { + $bn =~ s/(\.c)$//g; + if(!exists $c_files{$bn}) { + $c_files{$bn} = [ $name ]; + } else { + push @{ $c_files{$bn} }, $name; + } + } + } + }; + find($path_checker, $root_path); + +#print Dumper \%c_files; +#print Dumper \%cpp_files; +#print Dumper \%fort_files; +} + +sub count_file_loc { + my ($filename, $typename, $file_paths, $files_counted, $count) = @_; + + # Attempt to detect and handle object file name mangling by Automake + if(!exists $file_paths->{$filename} && $filename =~ /\-/) { + my ($bn) = $filename =~ /\S+\-(\S+)$/x; + if(exists $file_paths->{$bn}) { + if(!exists $options{q}) { + warn "No path for $typename source file '$filename', but '$bn' has path, assuming automake generated object file name"; + } + $filename = $bn; + } + } + + if(exists $file_paths->{$filename}) { + my $filecount = 0; + + # Attempt to count LOC for files with same name + if(scalar(@{$file_paths->{$filename}}) > 1) { + $filecount = $files_counted->{$filename}++; + + # Issue warning about multiple source files with same name + if($filecount == 0 && !exists $options{q}) { + local $" = ', '; # '$"' is documented in https://perldoc.perl.org/perlvar + warn "Multiple paths for $typename source file named '$filename', assuming each is compiled once, paths: [@{$file_paths->{$filename}}]\n"; + } + + # Sanity check for too many compiles of a file + if($filecount >= scalar(@{$file_paths->{$filename}})) { + if(!exists $options{q}) { + local $" = ', '; # '$"' is documented in https://perldoc.perl.org/perlvar + warn "Too many compiles of $typename source file named '$filename' with paths: [@{$file_paths->{$filename}}], disabling warning density calculations\n"; + } + delete $options{d}; + } + } + + # Increment the # of lines of code (if not too many) + if($filecount < scalar(@{$file_paths->{$filename}})) { + ${$count} += line_count($file_paths->{$filename}[$filecount]); + } + } else { + if(!exists $options{q}) { + warn "No path for $typename source file '$filename', e '-p' option to specify, disabling warning density calculations\n"; + } + delete $options{d}; + } +} + +# Compute LOC for compiled source file +sub count_source_loc { + my ($compile_line) = @_; +#print "compile_line = $compile_line\n"; + my $filetype; + my $filename; + + ($filetype, $filename) = $compile_line =~ /^\s+(CC|FC|CXX|PPFC)\s+(\S*)\.l*o$/x; + if($filename =~ /\//) { + $filename = basename($filename); + } +#print "filetype = '$filetype'\n"; +#print "filename = '$filename'\n"; + + if($filetype =~ /FC|PPFC/) { # FORTRAN source file + count_file_loc($filename, "FORTRAN", \%fort_files, \%fort_files_counted, \$fort_lines); + } elsif($filetype =~ /CXX/) { # C++ source file + count_file_loc($filename, "C++", \%cpp_files, \%cpp_files_counted, \$cpp_lines); + } elsif($filetype =~ /CC/) { # C source file + count_file_loc($filename, "C", \%c_files, \%c_files_counted, \$c_lines); + } +} + +sub sanity_check_loc { + my ($typename, $file_paths, $files_counted) = @_; + + if(scalar keys %{$files_counted} > 0) { + for my $x (keys(%{$files_counted})) { +#print "x = $x, # of compiles = ${$files_counted}{$x}, # of paths = ", scalar(@{$file_paths->{$x}}), "\n"; + if($files_counted->{$x} != scalar(@{$file_paths->{$x}})) { + if(!exists $options{q}) { + warn "# of compiles of C source file '$x' ($files_counted->{$x}) != # of paths (", scalar(@{$file_paths->{$x}}), "), disabling warning density calculation\n"; + } + # Don't print warning density, it's not accurate + delete $options{d}; + last; + } + } + } +} + + sub main::HELP_MESSAGE { do_help(); } # declare the Perl command line flags/options we want to allow -my %options=(); -getopts("FWhut:w:f:s:S:i:l", \%options); +getopts("FWhut:w:f:s:S:i:ldp:q", \%options); # Display usage, if requested if($options{h}) { @@ -185,6 +371,20 @@ if($options{u}) { $genericize = 0; } +# Scan source files, if warning density requested +if(exists $options{d}) { + if(exists $options{p}) { + my @pathnames = split /,/, $options{p}; +#print STDERR @pathnames; + for my $path (@pathnames) { + parse_tree($path); + } + } else { + # Scan the current working directory + parse_tree(getcwd); + } +} + PARSE_LINES: while (<>) { my $name; @@ -197,7 +397,7 @@ while (<>) { my $extra2; # Retain last FORTRAN compile line, which comes a few lines before warning - if($_ =~ /.*\.[fF]90:.*/) { + if($_ =~ /.*((\.inc)|(\.f90)|(\.f)|(\.f03)|(\.f08)|(\.f77)|(\.f95)|(\.for)|(\.fpp))\:.*/i) { ($last_fort_name, $last_fort_line, $last_fort_offset) = split /\:/, $_; ($last_fort_line, $toss) = split /\./, $last_fort_line; } @@ -212,17 +412,30 @@ while (<>) { $last_c_name = $_; } + # Compute LOC for compiled source files, if warning density requested + if(exists $options{d}) { + # Check for compilation line + if($_ =~ /^\s+(CC|FC|CXX|PPFC)\s+/) { + count_source_loc($_); + } + } + # Skip lines that don't have the word "warning" next if $_ !~ /[Ww]arning/; # Skip warnings from linker next if $_ =~ /ld: warning:/; + # Skip warnings from make + next if $_ =~ /^Makefile:[\d]*: warning:/; + # Skip warnings from build_py and install_lib next if $_ =~ /warning: (build_py|install_lib)/; - # Skip variables with the word 'warning' in them - next if $_ =~ /_warning_/; + # Skip variables with the word 'warning' (case insensitively) in them + next if $_ =~ /_warning_/i; + next if $_ =~ /_warning/i; + next if $_ =~ /warning_/i; # Skip AMD Optimizing Compiler (aocc) lines "<#> warning(s) generated." next if $_ =~ / warnings? generated\./; @@ -235,13 +448,9 @@ while (<>) { # Check for weird formatting of warning message $line = "??"; $offset = "??"; - if($_ =~ /^cc1: warning:.*/) { + if($_ =~ /^(cc1|): warning:.*/) { $name = $last_c_name; ($toss, $toss, $warning, $extra, $extra2) = split /\:/, $_; - # Check for CMAKE build with warning on first line and no filename - } elsif($_ =~ /^\s*[Ww]arning:.*/) { - $name = $last_c_name; - ($toss, $warning, $extra, $extra2) = split /\:/, $_; # Check for file-scope gcc Fortran warning output } elsif($_ =~ /f\d\d\d: Warning:/) { # These are interspersed with the "compiling a file" output @@ -274,9 +483,9 @@ while (<>) { } elsif($_ =~ /^\".*, line [0-9]+: *[Ww]arning:.*/) { ($name, $toss, $warning, $extra, $extra2) = split /\:/, $_; ($name, $line) = split /\,/, $name; - $name =~ s/^\"//g; - $name =~ s/\"$//g; - $line =~ s/^\s*line\s*//g; + $name =~ s/^\"//g; + $name =~ s/\"$//g; + $line =~ s/^\s*line\s*//g; # Check for Intel icc warning } elsif($_ =~ /.*[A-Za-z0-9_]\.[chC]\(.*[0-9]\):.*#.*/) { ($last_c_name, $toss, $warning) = split /\:/, $last_c_name; @@ -294,19 +503,13 @@ while (<>) { # Check for extra ':' followed by more text in original warning string, # and append the ':' and text back onto the parsed warning - # (Use 'length $extra' idiom to avoid warning when $extra is undefined) - if(length $extra ) { + if(defined $extra) { $warning = join ':', $warning, $extra; } - if(length $extra2 ) { + if(defined $extra2) { $warning = join ':', $warning, $extra2; } - # Restore the C++ '::' symbol now that we've parsed out the parts of the line - while($warning =~ /@@@@/) { - $warning =~ s/@@@@/\:\:/g; - } - # Trim leading '..' paths from filename while($name =~ /^\.\.\//) { $name =~ s/^\.\.\///g; @@ -336,6 +539,11 @@ while (<>) { next } + # Restore the C++ '::' symbol now that we've parsed out the parts of the line + while($warning =~ /@@@@/) { + $warning =~ s/@@@@/\:\:/g; + } + # Get rid of leading & trailing whitespace $warning =~ s/^\s//g; $warning =~ s/\s$//g; @@ -383,8 +591,8 @@ while (<>) { if($warning =~ /'[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=]+[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=\ ]*'/) { $warning =~ s/'[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=]+[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=\ ]*'/'-'/g; } - if($warning =~ /'%[\#0\-\ \+]*[,;\:_]?[0-9\*]*\.?[0-9\*]*[hjltzL]*[aAcdeEfFgGinosuxX]'/) { - $warning =~ s/'%[\#0\-\ \+]*[,;\:_]?[0-9\*]*\.?[0-9\*]*[hjltzL]*[aAcdeEfFgGinosuxX]'/'-'/g; + if($warning =~ /'%[\#0\-\ \+]*[,;\:_]?[0-9\*]*\.?[0-9\*]*[hjltzL]*[aAcdeEfFgGinopsuxX]'/) { + $warning =~ s/'%[\#0\-\ \+]*[,;\:_]?[0-9\*]*\.?[0-9\*]*[hjltzL]*[aAcdeEfFgGinopsuxX]'/'-'/g; } # Genericize C/C++ "" warnings into "-" @@ -397,6 +605,11 @@ while (<>) { $warning =~ s/=[A-Za-z_0-9]*\]/=-\]/g; } + # Genericize C/C++ "No such file or directory" warnings into "-" + if($warning =~ /^[A-Za-z_0-9\/]*: No such file or directory/) { + $warning =~ s/^[A-Za-z_0-9\/]*:/'-':/g; + } + # Genericize FORTRAN "at ()" into "at (-)", "REAL()" into "REAL(-)", # and "INTEGER()" into "INTEGER(-)" if($warning =~ /.*at\s\([0-9]+\).*/) { @@ -449,7 +662,31 @@ while (<>) { # print STDERR "warning = \"$warning\"\n"; } -print "Total unique [non-ignored] warnings: $totalcount\n"; +# Sanity check compiled source files with multiple paths when computing +# warning density +# (Check $options{d} each time, because any of the sanity checks could disable +# displaying the warning density) +if(exists $options{d}) { + sanity_check_loc("C", \%c_files, \%c_files_counted); +} +if(exists $options{d}) { + sanity_check_loc("FORTRAN", \%fort_files, \%fort_files_counted); +} +if(exists $options{d}) { + sanity_check_loc("C++", \%cpp_files, \%cpp_files_counted); +} + + +# +# Display results +# + +print "\nTotal unique [non-ignored] warnings: $totalcount\n"; +# Display warning density, if requested +if(exists $options{d}) { + print "Lines of code compiled: (C/C++/FORTRAN): ", ($c_lines + $cpp_lines + $fort_lines), " ($c_lines/$cpp_lines/$fort_lines)\n"; + printf "Warning density (<# of warnings> / <# of LOC compiled>): %10.10f\n", $totalcount / ($c_lines + $cpp_lines + $fort_lines); +} print "Ignored notes / supplemental warning lines [not counted in unique warnings]: $notecount\n"; print "Duplicated warning lines [not counted in unique warnings]: $dupcount\n"; print "Total ignored warnings: $ignorecount\n"; @@ -484,7 +721,7 @@ for my $x (sort {$warn_count{$b} <=> $warn_count{$a}} keys(%warn_count)) { $match = 1; } - if($match) { + if($match || exists $options{W}) { for my $y (sort {$warn_file{$x}{$b} <=> $warn_file{$x}{$a}} keys(%{$warn_file{$x}})) { printf ("\t%4d - %s\n", $warn_file{$x}{$y}, $y); if(exists $options{l}) { @@ -522,7 +759,7 @@ for my $x (sort {$file_count{$b} <=> $file_count{$a}} keys(%file_count)) { $match = 1; } - if($match) { + if($match || exists $options{F}) { for my $y (sort {$file_warn{$x}{$b} <=> $file_warn{$x}{$a}} keys(%{$file_warn{$x}})) { printf ("\t%4d - %s\n", $file_warn{$x}{$y}, $y); if(exists $options{l}) { diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 8013363..df28f76 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -595,6 +595,17 @@ endif () MARK_AS_ADVANCED (HDF5_ENABLE_CODESTACK) # ---------------------------------------------------------------------- +# Check if they would like to show all warnings (not suppressed internally) +#----------------------------------------------------------------------------- +option (HDF5_SHOW_ALL_WARNINGS "Show all warnings (not suppressed internally)." OFF) +mark_as_advanced (HDF5_SHOW_ALL_WARNINGS) +if (HDF5_SHOW_ALL_WARNINGS) + message (STATUS "....All warnings will be displayed") + set (${HDF_PREFIX}_SHOW_ALL_WARNINGS 1) +endif () +MARK_AS_ADVANCED (HDF5_SHOW_ALL_WARNINGS) + +# ---------------------------------------------------------------------- # Check if they would like to use file locking by default #----------------------------------------------------------------------------- option (HDF5_USE_FILE_LOCKING "Use file locking by default (mainly for SWMR)" ON) diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index e9b2dd1..0ff22fe 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -628,6 +628,9 @@ /* Check exception handling functions during data conversions */ #cmakedefine H5_WANT_DCONV_EXCEPTION @H5_WANT_DCONV_EXCEPTION@ +/* Define if showing all compiler warnings are desired (i.e. don't suppress them internally) */ +#cmakedefine H5_SHOW_ALL_WARNINGS @H5_SHOW_ALL_WARNINGS@ + /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel). */ #if !defined(__APPLE__) diff --git a/config/freebsd b/config/freebsd index b0e825a..70cf44a 100644 --- a/config/freebsd +++ b/config/freebsd @@ -68,7 +68,7 @@ fi . $srcdir/config/oneapi-fflags # Figure out Intel classic FC compiler flags -. $srcdir/config/classic-fflags +. $srcdir/config/intel-fflags # The default C++ compiler diff --git a/configure.ac b/configure.ac index df98ead..76303d3 100644 --- a/configure.ac +++ b/configure.ac @@ -591,12 +591,19 @@ if test "X$HDF_FORTRAN" = "Xyes"; then AC_DEFINE_UNQUOTED([PAC_C_MAX_REAL_PRECISION], $PAC_C_MAX_REAL_PRECISION, [Determine the maximum decimal precision in C]) AC_MSG_RESULT([$PAC_C_MAX_REAL_PRECISION]) + VERS_MAJOR=`cat $srcdir/src/H5public.h | sed -n 's/^#define H5_VERS_MAJOR //p'` + VERS_MINOR=`cat $srcdir/src/H5public.h | sed -n 's/^#define H5_VERS_MINOR //p'` + VERS_RELEASE=`cat $srcdir/src/H5public.h | sed -n 's/^#define H5_VERS_RELEASE //p'` + AC_DEFINE_UNQUOTED([VERS_MAJOR_TMP], $VERS_MAJOR, [Define major library version]) + AC_DEFINE_UNQUOTED([VERS_MINOR_TMP], $VERS_MINOR, [Define minor library version]) + AC_DEFINE_UNQUOTED([VERS_RELEASE_TMP], $VERS_RELEASE, [Define release library version]) + ## We will output an include file for Fortran, H5config_f.inc which ## contains various configure definitions used by the Fortran Library. ## Prepend H5_ to all macro names. This avoids name conflict between HDF5 macro ## names and those generated by another software package that uses the HDF5 library. AC_CONFIG_HEADERS([fortran/src/H5config_f.inc], - [cat fortran/src/H5config_f.inc | sed '1d;s%^/\* \(.*\) \*/%\1%;s/#define /#define H5_/;s/#undef /#undef H5_/' >fortran/src/H5config_f.inc.tmp; mv -f fortran/src/H5config_f.inc.tmp fortran/src/H5config_f.inc]) + [cat fortran/src/H5config_f.inc | sed '1d;s%^/\* \(.*\) \*/%\1%;s/#define /#define H5_/;s/#undef /#undef H5_/' >fortran/src/H5config_f.inc.tmp; sed -i 's\_TMP\\g' fortran/src/H5config_f.inc.tmp; mv -f fortran/src/H5config_f.inc.tmp fortran/src/H5config_f.inc]) AC_SUBST([FC]) @@ -2312,6 +2319,45 @@ case "X-$DEV_WARNINGS" in esac ## ---------------------------------------------------------------------- +## Check if they would like suppessed compiler diagnostics displayed +## (i.e. not suppressed) +## +## NOTE: Compiler diagnostics (i.e. warnings) are suppressed for some +## "noisy" warnings that are harmless (in the opinion of the primary +## HDF5 development team), but this option is provided to allow +## developers to see those warnings. +## +AC_MSG_CHECKING([whether showing all compiler warnings is enabled]) +AC_ARG_ENABLE([show-all-warnings], + [AS_HELP_STRING([--enable-show-all-warnings], + [Enable showing all compiler warnings (for developer debugging). + [default=no] + ])], + [SHOW_ALL_WARNINGS=$enableval]) + +## Set the default level. +if test "X-$SHOW_ALL_WARNINGS" = X- ; then + SHOW_ALL_WARNINGS=no +fi + +## Allow this variable to be substituted in +## other files (src/libhdf5.settings.in, etc.) +AC_SUBST([SHOW_ALL_WARNINGS]) + +case "X-$SHOW_ALL_WARNINGS" in + X-yes) + AC_MSG_RESULT([yes]) + AC_DEFINE([SHOW_ALL_WARNINGS], [1], [Define if showing all warnings is desired (i.e. not suppressed internally with H5_DIAG_OFF)]) + ;; + X-no) + AC_MSG_RESULT([no]) + ;; + *) + AC_MSG_ERROR([Unrecognized value: $SHOW_ALL_WARNINGS]) + ;; +esac + +## ---------------------------------------------------------------------- ## Check if the compiler should use profiling flags/settings ## AC_MSG_CHECKING([profiling]) diff --git a/fortran/src/CMakeLists.txt b/fortran/src/CMakeLists.txt index 87557db..60c0c3a 100644 --- a/fortran/src/CMakeLists.txt +++ b/fortran/src/CMakeLists.txt @@ -431,6 +431,7 @@ if (BUILD_STATIC_LIBS) FILES ${HDF5_F90_BINARY_DIR}/static/H5f90i_gen.h ${HDF5_F90_BINARY_DIR}/static/H5fortran_types.F90 + ${HDF5_F90_BINARY_DIR}/H5config_f.inc DESTINATION ${HDF5_INSTALL_INCLUDE_DIR} COMPONENT @@ -441,6 +442,7 @@ else () FILES ${HDF5_F90_BINARY_DIR}/shared/H5f90i_gen.h ${HDF5_F90_BINARY_DIR}/shared/H5fortran_types.F90 + ${HDF5_F90_BINARY_DIR}/H5config_f.inc DESTINATION ${HDF5_INSTALL_INCLUDE_DIR} COMPONENT diff --git a/fortran/src/H5Fff.F90 b/fortran/src/H5Fff.F90 index d311177..79aa5a7 100644 --- a/fortran/src/H5Fff.F90 +++ b/fortran/src/H5Fff.F90 @@ -256,6 +256,43 @@ CONTAINS !> !! \ingroup FH5F !! +!! \brief Deletes an HDF5 file +!! +!! \param name Name of the file to delete +!! \param hdferr \fortran_error +!! \param access_prp File access property list identifier +!! +!! See C API: @ref H5Fdelete() +!! + SUBROUTINE h5fdelete_f(name, hdferr, access_prp) + IMPLICIT NONE + CHARACTER(LEN=*), INTENT(IN) :: name + INTEGER , INTENT(OUT) :: hdferr + INTEGER(HID_T) , INTENT(IN), OPTIONAL :: access_prp + + INTEGER(HID_T) :: access_prp_default + CHARACTER(LEN=LEN_TRIM(name)+1,KIND=C_CHAR) :: c_name + + INTERFACE + INTEGER(C_INT) FUNCTION H5Fdelete(name, access_prp_default) BIND(C,NAME='H5Fdelete') + IMPORT :: C_CHAR, C_INT + IMPORT :: HID_T + CHARACTER(KIND=C_CHAR), DIMENSION(*) :: name + INTEGER(HID_T), VALUE :: access_prp_default + END FUNCTION H5Fdelete + END INTERFACE + + c_name = TRIM(name)//C_NULL_CHAR + + access_prp_default = H5P_DEFAULT_F + IF (PRESENT(access_prp)) access_prp_default = access_prp + + hdferr = INT(H5Fdelete(c_name, access_prp_default)) + + END SUBROUTINE h5fdelete_f +!> +!! \ingroup FH5F +!! !! \brief Asynchronously flushes all buffers associated with a file to disk. !! !! \param object_id Identifier of object used to identify the file. @@ -285,7 +322,7 @@ CONTAINS INTEGER(KIND=C_INT) :: line_default = 0 INTERFACE - INTEGER FUNCTION H5Fflush_async(file, func, line, object_id, scope, es_id) & + INTEGER(C_INT) FUNCTION H5Fflush_async(file, func, line, object_id, scope, es_id) & BIND(C,NAME='H5Fflush_async') IMPORT :: C_CHAR, C_INT, C_PTR IMPORT :: HID_T @@ -303,8 +340,8 @@ CONTAINS IF(PRESENT(func)) func_default = func IF(PRESENT(line)) line_default = INT(line, C_INT) - hdferr = H5Fflush_async(file_default, func_default, line_default, & - object_id, INT(scope, C_INT), es_id) + hdferr = INT(H5Fflush_async(file_default, func_default, line_default, & + object_id, INT(scope, C_INT), es_id)) END SUBROUTINE h5fflush_async_f !> diff --git a/fortran/src/H5_f.c b/fortran/src/H5_f.c index 0392c2b..b1dc7db 100644 --- a/fortran/src/H5_f.c +++ b/fortran/src/H5_f.c @@ -771,6 +771,8 @@ h5init_flags_c(int_f *h5d_flags, size_t_f *h5d_size_flags, int_f *h5e_flags, hid * H5S flags */ h5s_hid_flags[0] = (hid_t_f)H5S_ALL; + h5s_hid_flags[1] = (hid_t_f)H5S_BLOCK; + h5s_hid_flags[2] = (hid_t_f)H5S_PLIST; h5s_hsize_flags[0] = (hsize_t_f)H5S_UNLIMITED; diff --git a/fortran/src/H5_ff.F90 b/fortran/src/H5_ff.F90 index 05a48ac..fe6337a 100644 --- a/fortran/src/H5_ff.F90 +++ b/fortran/src/H5_ff.F90 @@ -139,7 +139,7 @@ MODULE H5LIB INTEGER, DIMENSION(1:H5S_FLAGS_LEN) :: H5S_flags INTEGER, PARAMETER :: H5S_HSIZE_FLAGS_LEN = 1 INTEGER(HSIZE_T), DIMENSION(1:H5S_HSIZE_FLAGS_LEN) :: H5S_hsize_flags - INTEGER, PARAMETER :: H5S_HID_FLAGS_LEN = 1 + INTEGER, PARAMETER :: H5S_HID_FLAGS_LEN = 3 INTEGER(HSIZE_T), DIMENSION(1:H5S_HID_FLAGS_LEN) :: H5S_hid_flags ! ! H5T flags declaration @@ -632,7 +632,9 @@ CONTAINS ! ! H5S flags ! - H5S_ALL_F = H5S_hid_flags(1) + H5S_ALL_F = H5S_hid_flags(1) + H5S_BLOCK_F = H5S_hid_flags(2) + H5S_PLIST_F = H5S_hid_flags(3) H5S_UNLIMITED_F = H5S_hsize_flags(1) diff --git a/fortran/src/H5config_f.inc.cmake b/fortran/src/H5config_f.inc.cmake index 6652076..77ff707 100644 --- a/fortran/src/H5config_f.inc.cmake +++ b/fortran/src/H5config_f.inc.cmake @@ -9,7 +9,7 @@ ! If you do not have access to either file, you may request a copy from * ! help@hdfgroup.org. * ! * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -! fortran/src/H5config_f.inc. Generated from fortran/src/H5config_f.inc.in by configure +! fortran/H5config_f.inc. Generated from fortran/src/H5config_f.inc.cmake by CMake ! Define if there is parallel support #cmakedefine01 CMAKE_H5_HAVE_PARALLEL @@ -28,8 +28,8 @@ #endif ! Define if on APPLE -#cmakedefine01 H5_HAVE_DARWIN -#if H5_HAVE_DARWIN == 0 +#cmakedefine01 CMAKE_H5_HAVE_DARWIN +#if CMAKE_H5_HAVE_DARWIN == 0 #undef H5_HAVE_DARWIN #else #define H5_HAVE_DARWIN @@ -116,3 +116,22 @@ #else #define H5_NO_DEPRECATED_SYMBOLS #endif + +! For major interface/format changes +#define H5_VERS_MAJOR @H5_VERS_MAJOR@ + +! For minor interface/format changes +#define H5_VERS_MINOR @H5_VERS_MINOR@ + +! For tweaks, bug-fixes, or development +#define H5_VERS_RELEASE @H5_VERS_RELEASE@ + +! macros for comparing versions +#define H5_VERSION_GE(Maj, Min, Rel) \ + (((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR == Min) && (H5_VERS_RELEASE >= Rel)) || \ + ((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR > Min)) || (H5_VERS_MAJOR > Maj)) + +#define H5_VERSION_LE(Maj, Min, Rel) \ + (((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR == Min) && (H5_VERS_RELEASE <= Rel)) || \ + ((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR < Min)) || (H5_VERS_MAJOR < Maj)) + diff --git a/fortran/src/H5config_f.inc.in b/fortran/src/H5config_f.inc.in index 3aeded9..afcfa6e 100644 --- a/fortran/src/H5config_f.inc.in +++ b/fortran/src/H5config_f.inc.in @@ -80,3 +80,21 @@ ! Define if deprecated public API symbols are disabled #undef NO_DEPRECATED_SYMBOLS +! For major interface/format changes +#undef VERS_MAJOR_TMP + +! For minor interface/format changes +#undef VERS_MINOR_TMP + +! For tweaks, bug-fixes, or development +#undef VERS_RELEASE_TMP + +! macros for comparing versions +#define VERSION_GE(Maj, Min, Rel) \ + (((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR == Min) && (H5_VERS_RELEASE >= Rel)) || \ + ((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR > Min)) || (H5_VERS_MAJOR > Maj)) + +#define VERSION_LE(Maj, Min, Rel) \ + (((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR == Min) && (H5_VERS_RELEASE <= Rel)) || \ + ((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR < Min)) || (H5_VERS_MAJOR < Maj)) + diff --git a/fortran/src/H5f90global.F90 b/fortran/src/H5f90global.F90 index aa04623..fb25f7e 100644 --- a/fortran/src/H5f90global.F90 +++ b/fortran/src/H5f90global.F90 @@ -773,6 +773,8 @@ MODULE H5GLOBAL !DEC$if defined(BUILD_HDF5_DLL) !DEC$ATTRIBUTES DLLEXPORT :: H5S_UNLIMITED_F !DEC$ATTRIBUTES DLLEXPORT :: H5S_ALL_F + !DEC$ATTRIBUTES DLLEXPORT :: H5S_BLOCK_F + !DEC$ATTRIBUTES DLLEXPORT :: H5S_PLIST_F !DEC$ATTRIBUTES DLLEXPORT :: H5S_SCALAR_F !DEC$ATTRIBUTES DLLEXPORT :: H5S_SIMPLE_F !DEC$ATTRIBUTES DLLEXPORT :: H5S_NULL_F @@ -796,7 +798,9 @@ MODULE H5GLOBAL !> @{ INTEGER(HSIZE_T) :: H5S_UNLIMITED_F !< H5S_UNLIMITED - INTEGER(HID_T) :: H5S_ALL_F !< H5S_ALL + INTEGER(HID_T) :: H5S_ALL_F !< H5S_ALL + INTEGER(HID_T) :: H5S_BLOCK_F !< H5S_BLOCK + INTEGER(HID_T) :: H5S_PLIST_F !< H5S_PLIST INTEGER :: H5S_SCALAR_F !< H5S_SCALAR INTEGER :: H5S_SIMPLE_F !< H5S_SIMPLE diff --git a/fortran/src/Makefile.am b/fortran/src/Makefile.am index d42a41d..44561f6 100644 --- a/fortran/src/Makefile.am +++ b/fortran/src/Makefile.am @@ -75,6 +75,7 @@ install-data-local: fi $(CP) $(top_builddir)/$(subdir)/H5f90i_gen.h $(DESTDIR)$(includedir)/. $(CP) $(top_srcdir)/fortran/src/H5f90i.h $(DESTDIR)$(includedir)/. + $(CP) $(top_builddir)/$(subdir)/H5config_f.inc $(DESTDIR)$(includedir)/. uninstall-local: @if test -n "$(F9XMODEXT)" -a "X$(F9XMODEXT)" != "Xo"; then \ @@ -84,6 +85,7 @@ uninstall-local: fi; \ $(RM) $(DESTDIR)$(bindir)/$(H5FC_NAME) $(RM) $(DESTDIR)$(includedir)/H5f90i*.h + $(RM) $(DESTDIR)$(includedir)/H5config_f.inc # Also install and uninstall (uninstall-local above) h5fc script install-exec-local: diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 55f4f2b..119e140 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -121,6 +121,7 @@ H5ES_mp_H5ESGET_ERR_COUNT_F H5ES_mp_H5ESCLOSE_F ; H5F H5F_mp_H5FCREATE_F +H5F_mp_H5FDELETE_F H5F_mp_H5FCREATE_ASYNC_F H5F_mp_H5FFLUSH_F H5F_mp_H5FFLUSH_ASYNC_F diff --git a/fortran/test/tH5F.F90 b/fortran/test/tH5F.F90 index b4d973e..7f9490b 100644 --- a/fortran/test/tH5F.F90 +++ b/fortran/test/tH5F.F90 @@ -479,10 +479,11 @@ CONTAINS total_error = total_error + 1 ENDIF - if(cleanup) CALL h5_cleanup_f(filename1, H5P_DEFAULT_F, error) + IF(cleanup) CALL h5_cleanup_f(filename1, H5P_DEFAULT_F, error) CALL check("h5_cleanup_f", error, total_error) - if(cleanup) CALL h5_cleanup_f(filename2, H5P_DEFAULT_F, error) + IF(cleanup) CALL h5_cleanup_f(filename2, H5P_DEFAULT_F, error) CALL check("h5_cleanup_f", error, total_error) + RETURN END SUBROUTINE mountingtest @@ -853,7 +854,9 @@ CONTAINS INTEGER(HID_T) :: access_id ! File Access property list identifier !flag to check operation success - INTEGER :: error + INTEGER :: error + !file status + LOGICAL :: status ! !Create a file1 using default properties. @@ -920,10 +923,37 @@ CONTAINS CALL h5fclose_f(file2_id, error) CALL check("h5fclose_f",error,total_error) - if(cleanup) CALL h5_cleanup_f(filename1, H5P_DEFAULT_F, error) - CALL check("h5_cleanup_f", error, total_error) - if(cleanup) CALL h5_cleanup_f(filename2, H5P_DEFAULT_F, error) - CALL check("h5_cleanup_f", error, total_error) + ! Test file deletion + CALL h5fis_accessible_f(filename1, status, error) + CALL check("h5fis_accessible_f",error,total_error) + IF ( .NOT. status ) THEN + WRITE(*,*) "ERROR: File ", filename1, " is not accessible as hdf5" + END IF + + CALL h5fdelete_f(filename1, error, H5P_DEFAULT_F) + CALL check("h5fdelete_f", error, total_error) + + INQUIRE(FILE=filename1, EXIST=status) + IF ( status ) THEN + WRITE(*,*) "ERROR: File ", filename1, " was not removed by H5Fdelete_f" + END IF + + CALL h5fis_accessible_f(filename2, status, error) + CALL check("h5fis_accessible_f",error,total_error) + IF ( .NOT. status ) THEN + WRITE(*,*) "ERROR: File ", filename2, " is not accessible as hdf5" + total_error=total_error + 1 + END IF + + CALL h5fdelete_f(filename2, error) + CALL check("h5fdelete_f", error, total_error) + + INQUIRE(FILE=filename2, EXIST=status) + IF ( status ) THEN + WRITE(*,*) "ERROR: File ", filename2, " was not removed by H5Fdelete_f" + total_error=total_error + 1 + END IF + RETURN END SUBROUTINE plisttest @@ -1320,6 +1350,7 @@ CONTAINS TYPE(C_PTR) :: f_ptr ! Pointer INTEGER(hid_t) :: fapl ! File access property INTEGER :: error ! Error flag + CHARACTER(LEN=18), PARAMETER :: filename="tget_file_image.h5" ! Create new properties for file access CALL h5pcreate_f(H5P_FILE_ACCESS_F, fapl, error) @@ -1330,7 +1361,7 @@ CONTAINS CALL check("h5pset_fapl_stdio_f", error, total_error) ! Create the file - CALL h5fcreate_f("tget_file_image.h5", H5F_ACC_TRUNC_F, file_id, error, H5P_DEFAULT_F, fapl) + CALL h5fcreate_f(filename, H5F_ACC_TRUNC_F, file_id, error, H5P_DEFAULT_F, fapl) CALL check("h5fcreate_f", error, total_error) ! Set up data space for new data set @@ -1357,7 +1388,7 @@ CONTAINS CALL check("h5fflush_f",error, total_error) ! Open the test file using standard I/O calls - OPEN(UNIT=10,FILE='tget_file_image.h5', ACCESS='STREAM') + OPEN(UNIT=10,FILE=filename, ACCESS='STREAM') ! Get the size of the test file ! ! Since we use the eoa to calculate the image size, the file size @@ -1406,7 +1437,7 @@ CONTAINS ALLOCATE(file_image_ptr(1:image_size)) ! Open the test file using standard I/O calls - OPEN(UNIT=10,FILE='tget_file_image.h5', FORM='UNFORMATTED', ACCESS='STREAM') + OPEN(UNIT=10,FILE=filename, FORM='UNFORMATTED', ACCESS='STREAM') ! Read the test file from disk into the buffer DO i = 1, image_size diff --git a/hl/tools/h5watch/CMakeLists.txt b/hl/tools/h5watch/CMakeLists.txt index 1eb3618..890ac37 100644 --- a/hl/tools/h5watch/CMakeLists.txt +++ b/hl/tools/h5watch/CMakeLists.txt @@ -12,7 +12,7 @@ set (H5WATCH_SOURCES if (BUILD_STATIC_LIBS) add_executable (h5watch ${H5WATCH_SOURCES}) target_compile_options(h5watch PRIVATE "${HDF5_CMAKE_C_FLAGS}") - target_include_directories (h5watch PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5watch PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5watch STATIC) target_link_libraries (h5watch PRIVATE ${HDF5_HL_LIB_TARGET} ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) set_target_properties (h5watch PROPERTIES FOLDER tools/hl) @@ -21,7 +21,7 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5watch-shared ${H5WATCH_SOURCES}) target_compile_options(h5watch-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") - target_include_directories (h5watch-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5watch-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5watch-shared SHARED) target_link_libraries (h5watch-shared PRIVATE ${HDF5_HL_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${HDF5_TOOLS_LIBSH_TARGET}) set_target_properties (h5watch-shared PROPERTIES FOLDER tools/hl) diff --git a/release_docs/INSTALL_CMake.txt b/release_docs/INSTALL_CMake.txt index e27f74b..7839f42 100644 --- a/release_docs/INSTALL_CMake.txt +++ b/release_docs/INSTALL_CMake.txt @@ -826,6 +826,7 @@ ALLOW_UNSUPPORTED "Allow unsupported combinations of configure opti HDF5_EXTERNAL_LIB_PREFIX "Use prefix for custom library naming." "" HDF5_DISABLE_COMPILER_WARNINGS "Disable compiler warnings" OFF HDF5_ENABLE_ALL_WARNINGS "Enable all warnings" OFF +HDF5_SHOW_ALL_WARNINGS "Show all warnings (i.e. not suppress "noisy" ones internally)" OFF HDF5_ENABLE_CODESTACK "Enable the function stack tracing (for developer debugging)." OFF HDF5_ENABLE_COVERAGE "Enable code coverage for Libraries and Programs" OFF HDF5_ENABLE_DEBUG_APIS "Turn on extra debug output in all packages" OFF diff --git a/release_docs/INSTALL_Cygwin.txt b/release_docs/INSTALL_Cygwin.txt index c78f61f..34a57ee 100644 --- a/release_docs/INSTALL_Cygwin.txt +++ b/release_docs/INSTALL_Cygwin.txt @@ -267,4 +267,4 @@ Build, Test and Install HDF5 on Cygwin ----------------------------------------------------------------------- HDF Forum: https://forum.hdfgroup.org/ - HDF Helpdesk: https://portal.hdfgroup.org/display/support/The+HDF+Help+Desk + HDF Helpdesk: https://help.hdfgroup.org/ diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 04ce910..c1188ae 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -47,6 +47,20 @@ New Features Configuration: ------------- + - Incorporated HDF5 examples repository into HDF5 library. + + The HDF5Examples folder is equivalent to the repository hdf5-examples. + As such it can build and test the examples during library build or after + the library is installed. Previously, the hdf5-repository archives were + downloaded for packaging with the library. Now the examples can be built + and tested without a packaged install of the library. + + However to maintain the ability to use the HDF5Examples with an installed + library, it is necessary to translate or synch the option names from those + used by the library to those used by the examples. The typical pattern is: + = + HDF_BUILD_FORTRAN = ${HDF5_BUILD_FORTRAN} + - Added new option for CMake to mark tests as SKIPPED. HDF5_DISABLE_TESTS_REGEX is a REGEX string that will be checked with @@ -88,6 +102,15 @@ New Features Fortran Library: ---------------- + - Added Fortran Parameters: + H5S_BLOCK_F and H5S_PLIST_F + + - The configuration definitions file, H5config_f.inc, is now installed + and the HDF5 version number has been added to it. + + - Added Fortran APIs: + h5fdelete_f + - Added Fortran APIs: h5vlnative_addr_to_token_f and h5vlnative_token_to_address_f diff --git a/release_docs/USING_HDF5_VS.txt b/release_docs/USING_HDF5_VS.txt index 3375a77..79d9c3c 100644 --- a/release_docs/USING_HDF5_VS.txt +++ b/release_docs/USING_HDF5_VS.txt @@ -11,6 +11,9 @@ be found in the USING_HDF5_CMake.txt file found in this folder. NOTE: Building applications with the dynamic/shared hdf5 libraries requires that the "H5_BUILT_AS_DYNAMIC_LIB" compile definition be used. + Go to "Project" and select "Properties", find "Configuration Properties", + and then "C/C+±"and then "Preprocessor". + Add H5_BUILT_AS_DYNAMIC_LIB to Preprocessor definitions. The following two sections are helpful if you do not use CMake to build your applications. diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index a3695ae..dc842e8 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -727,7 +727,7 @@ H5_DLL herr_t H5D__chunk_allocated(const H5D_t *dset, hsize_t *nbytes); H5_DLL herr_t H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_dim[]); H5_DLL herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk, H5F_block_t *new_chunk, bool *need_insert, const hsize_t *scaled); -H5_DLL void *H5D__chunk_mem_alloc(size_t size, void *pline); +H5_DLL void *H5D__chunk_mem_alloc(size_t size, void *pline) H5_ATTR_MALLOC; H5_DLL void H5D__chunk_mem_free(void *chk, void *pline); H5_DLL void *H5D__chunk_mem_xfree(void *chk, const void *pline); H5_DLL void *H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline); diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index a4c60cd..35d0edf 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -424,6 +424,8 @@ H5_DLL hid_t H5Dopen_async(hid_t loc_id, const char *name, hid_t dapl_id, hid_t * be released with H5Sclose() when the identifier is no longer * needed so that resource leaks will not occur. * + * \since 1.0.0 + * * \par Example * \snippet H5D_examples.c update * @@ -494,6 +496,8 @@ H5_DLL herr_t H5Dget_space_status(hid_t dset_id, H5D_space_status_t *allocation) * opened datatype is returned. Otherwise, the returned datatype * is read-only. * + * \since 1.0.0 + * */ H5_DLL hid_t H5Dget_type(hid_t dset_id); @@ -515,6 +519,8 @@ H5_DLL hid_t H5Dget_type(hid_t dset_id); * The creation property list identifier should be released with * H5Pclose() to prevent resource leaks. * + * \since 1.0.0 + * */ H5_DLL hid_t H5Dget_create_plist(hid_t dset_id); @@ -590,6 +596,7 @@ H5_DLL hid_t H5Dget_access_plist(hid_t dset_id); * with no stored values, and 0 (zero), the value returned to * indicate an error. * + * \since 1.2.0 * */ H5_DLL hsize_t H5Dget_storage_size(hid_t dset_id); @@ -872,6 +879,8 @@ H5_DLL haddr_t H5Dget_offset(hid_t dset_id); * \par Example * \snippet H5D_examples.c read * + * \since 1.0.0 + * */ H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t dxpl_id, void *buf /*out*/); @@ -1060,6 +1069,8 @@ H5_DLL herr_t H5Dread_multi_async(size_t count, hid_t dset_id[], hid_t mem_type_ * \par Example * \snippet H5D_examples.c update * + * \since 1.0.0 + * * \see H5Pset_fill_time(), H5Pset_alloc_time() * */ @@ -1355,6 +1366,7 @@ H5_DLL herr_t H5Dvlen_get_buf_size(hid_t dset_id, hid_t type_id, hid_t space_id, * * \see H5Pset_fill_value(), H5Pget_fill_value(), H5Pfill_value_defined(), * H5Pset_fill_time(), H5Pget_fill_time(), H5Pcreate(), H5Dcreate_anon() + * \since 1.6.0 * */ H5_DLL herr_t H5Dfill(const void *fill, hid_t fill_type_id, void *buf, hid_t buf_type_id, hid_t space_id); @@ -1598,7 +1610,7 @@ H5_DLL herr_t H5Dgather(hid_t src_space_id, const void *src_buf, hid_t type_id, * \par Example * \snippet H5D_examples.c read * - * \since 1.8.0 + * \since 1.0.0 * * \see H5Dcreate2(), H5Dopen2() * @@ -1811,6 +1823,7 @@ H5_DLL hid_t H5Dopen1(hid_t loc_id, const char *name); * * \version 1.8.0 Function deprecated in this release. Parameter size * syntax changed to \Code{const hsize_t size[]} in this release. + * \since 1.0.0 * */ H5_DLL herr_t H5Dextend(hid_t dset_id, const hsize_t size[]); @@ -1847,6 +1860,8 @@ H5_DLL herr_t H5Dextend(hid_t dset_id, const hsize_t size[]); * * \version 1.12.0 Function was deprecated * + * \since 1.2.0 + * */ H5_DLL herr_t H5Dvlen_reclaim(hid_t type_id, hid_t space_id, hid_t dxpl_id, void *buf); diff --git a/src/H5EApkg.h b/src/H5EApkg.h index 90ba02a..e8b5a13 100644 --- a/src/H5EApkg.h +++ b/src/H5EApkg.h @@ -389,7 +389,7 @@ H5_DLL herr_t H5EA__destroy_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t H5_DLL H5EA_hdr_t *H5EA__hdr_alloc(H5F_t *f); H5_DLL herr_t H5EA__hdr_init(H5EA_hdr_t *hdr, void *ctx_udata); H5_DLL haddr_t H5EA__hdr_create(H5F_t *f, const H5EA_create_t *cparam, void *ctx_udata); -H5_DLL void *H5EA__hdr_alloc_elmts(H5EA_hdr_t *hdr, size_t nelmts); +H5_DLL void *H5EA__hdr_alloc_elmts(H5EA_hdr_t *hdr, size_t nelmts) H5_ATTR_MALLOC; H5_DLL herr_t H5EA__hdr_free_elmts(H5EA_hdr_t *hdr, size_t nelmts, void *elmts); H5_DLL herr_t H5EA__hdr_incr(H5EA_hdr_t *hdr); H5_DLL herr_t H5EA__hdr_decr(H5EA_hdr_t *hdr); diff --git a/src/H5ESpkg.h b/src/H5ESpkg.h index f7c70e8..1da58a6 100644 --- a/src/H5ESpkg.h +++ b/src/H5ESpkg.h @@ -75,7 +75,7 @@ typedef int (*H5ES_list_iter_func_t)(H5ES_event_t *ev, void *ctx); /******************************/ /* Package Private Prototypes */ /******************************/ -H5_DLL H5ES_t *H5ES__create(void); +H5_DLL H5ES_t *H5ES__create(void) H5_ATTR_MALLOC; H5_DLL herr_t H5ES__insert_request(H5ES_t *es, H5VL_t *connector, void *token); H5_DLL herr_t H5ES__wait(H5ES_t *es, uint64_t timeout, size_t *num_in_progress, bool *op_failed); H5_DLL herr_t H5ES__get_requests(H5ES_t *es, H5_iter_order_t order, hid_t *connector_ids, void **requests, diff --git a/src/H5Epublic.h b/src/H5Epublic.h index b6cc1cb..a22c9c6 100644 --- a/src/H5Epublic.h +++ b/src/H5Epublic.h @@ -697,6 +697,9 @@ typedef struct H5E_error1_t { * \param[in] client_data Pointer to client data in the format expected by the * user-defined function * \return \herr_t + * + * \since 1.0.0 + * */ typedef herr_t (*H5E_walk1_t)(int n, H5E_error1_t *err_desc, void *client_data); //! @@ -708,6 +711,9 @@ typedef herr_t (*H5E_walk1_t)(int n, H5E_error1_t *err_desc, void *client_data); * \param[in] client_data Pointer to client data in the format expected by the * user-defined function * \return \herr_t + * + * \since 1.0.0 + * */ typedef herr_t (*H5E_auto1_t)(void *client_data); //! @@ -728,6 +734,8 @@ typedef herr_t (*H5E_auto1_t)(void *client_data); * The stack is also cleared whenever an API function is called, with * certain exceptions (for instance, H5Eprint1()). * + * \since 1.0.0 + * */ H5_DLL herr_t H5Eclear1(void); /** @@ -772,6 +780,8 @@ H5_DLL herr_t H5Eclear1(void); * H5Eprint2(), mixing H5Eset_auto1() and H5Eget_auto2() or mixing * H5Eset_auto2() and H5Eget_auto1() does not fail. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Eget_auto1(H5E_auto1_t *func, void **client_data); /** @@ -826,6 +836,8 @@ H5_DLL herr_t H5Epush1(const char *file, const char *func, unsigned line, H5E_ma * that prints error messages. Users are encouraged to write their own * more specific error handlers. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Eprint1(FILE *stream); /** @@ -857,6 +869,8 @@ H5_DLL herr_t H5Eprint1(FILE *stream); * Automatic stack traversal is always in the #H5E_WALK_DOWNWARD * direction. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Eset_auto1(H5E_auto1_t func, void *client_data); /** @@ -890,6 +904,8 @@ H5_DLL herr_t H5Eset_auto1(H5E_auto1_t func, void *client_data); * is as follows: * \snippet this H5E_walk1_t_snip * + * \since 1.0.0 + * */ H5_DLL herr_t H5Ewalk1(H5E_direction_t direction, H5E_walk1_t func, void *client_data); /** @@ -911,6 +927,8 @@ H5_DLL herr_t H5Ewalk1(H5E_direction_t direction, H5E_walk1_t func, void *client * array). An application calling this function must free the memory * associated with the return value to prevent a memory leak. * + * \since 1.0.0 + * */ H5_DLL char *H5Eget_major(H5E_major_t maj); /** @@ -934,6 +952,8 @@ H5_DLL char *H5Eget_major(H5E_major_t maj); * the memory associated with the return value to prevent a memory * leak. This is a change from the 1.6.x release series. * + * \since 1.0.0 + * */ H5_DLL char *H5Eget_minor(H5E_minor_t min); #endif /* H5_NO_DEPRECATED_SYMBOLS */ diff --git a/src/H5F.c b/src/H5F.c index e64782f..d814503 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -2211,7 +2211,7 @@ done: * * Purpose: Releases the external file cache associated with the * provided file, potentially closing any cached files - * unless they are held open from somewhere\ else. + * unless they are held open from somewhere else. * * Return: Success: Non-negative * Failure: Negative diff --git a/src/H5FDsubfiling/H5FDioc.c b/src/H5FDsubfiling/H5FDioc.c index 80771c0..b019add 100644 --- a/src/H5FDsubfiling/H5FDioc.c +++ b/src/H5FDsubfiling/H5FDioc.c @@ -1610,12 +1610,14 @@ H5FD__ioc_write_vector_internal(H5FD_t *_file, uint32_t count, H5FD_mem_t H5_ATT H5FD_ioc_t *file_ptr = (H5FD_ioc_t *)_file; io_req_t **sf_io_reqs = NULL; int64_t sf_context_id = -1; + size_t io_size = 0; + bool extend_sizes = false; herr_t ret_value = SUCCEED; assert(_file); - assert(addrs); - assert(sizes); - assert(bufs); + assert((addrs) || (count == 0)); + assert((sizes) || (count == 0)); + assert((bufs) || (count == 0)); if (count == 0) H5_SUBFILING_GOTO_DONE(SUCCEED); @@ -1648,12 +1650,22 @@ H5FD__ioc_write_vector_internal(H5FD_t *_file, uint32_t count, H5FD_mem_t H5_ATT for (size_t i = 0; i < (size_t)count; i++) { herr_t write_status; - if (sizes[i] == 0) + if (!extend_sizes) { + if ((i > 0) && (sizes[i] == 0)) { + extend_sizes = true; + io_size = sizes[i - 1]; + } + else { + io_size = sizes[i]; + } + } + + if (io_size == 0) H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "invalid size argument of 0"); H5_CHECK_OVERFLOW(addrs[i], haddr_t, int64_t); - H5_CHECK_OVERFLOW(sizes[i], size_t, int64_t); - write_status = ioc__write_independent_async(sf_context_id, (int64_t)addrs[i], (int64_t)sizes[i], + H5_CHECK_OVERFLOW(io_size, size_t, int64_t); + write_status = ioc__write_independent_async(sf_context_id, (int64_t)addrs[i], (int64_t)io_size, bufs[i], &sf_io_reqs[i]); if (write_status < 0) @@ -1691,12 +1703,14 @@ H5FD__ioc_read_vector_internal(H5FD_t *_file, uint32_t count, haddr_t addrs[], s H5FD_ioc_t *file_ptr = (H5FD_ioc_t *)_file; io_req_t **sf_io_reqs = NULL; int64_t sf_context_id = -1; + size_t io_size = 0; + bool extend_sizes = false; herr_t ret_value = SUCCEED; assert(_file); - assert(addrs); - assert(sizes); - assert(bufs); + assert((addrs) || (count == 0)); + assert((sizes) || (count == 0)); + assert((bufs) || (count == 0)); if (count == 0) H5_SUBFILING_GOTO_DONE(SUCCEED); @@ -1720,12 +1734,22 @@ H5FD__ioc_read_vector_internal(H5FD_t *_file, uint32_t count, haddr_t addrs[], s H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate MPI request array"); for (size_t i = 0; i < (size_t)count; i++) { - int read_status; + herr_t read_status; + + if (!extend_sizes) { + if ((i > 0) && (sizes[i] == 0)) { + extend_sizes = true; + io_size = sizes[i - 1]; + } + else { + io_size = sizes[i]; + } + } H5_CHECK_OVERFLOW(addrs[i], haddr_t, int64_t); - H5_CHECK_OVERFLOW(sizes[i], size_t, int64_t); - read_status = ioc__read_independent_async(sf_context_id, (int64_t)addrs[i], (int64_t)sizes[i], - bufs[i], &sf_io_reqs[i]); + H5_CHECK_OVERFLOW(io_size, size_t, int64_t); + read_status = ioc__read_independent_async(sf_context_id, (int64_t)addrs[i], (int64_t)io_size, bufs[i], + &sf_io_reqs[i]); if (read_status < 0) H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't queue read operation"); diff --git a/src/H5FDsubfiling/H5FDioc_int.c b/src/H5FDsubfiling/H5FDioc_int.c index 5528fc8..75a36d0 100644 --- a/src/H5FDsubfiling/H5FDioc_int.c +++ b/src/H5FDsubfiling/H5FDioc_int.c @@ -297,9 +297,13 @@ ioc__read_independent_async(int64_t context_id, int64_t offset, int64_t elements * unpredictable order. However, if some IOCs own more than * 1 subfile, we need to associate each read with a unique * message tag to make sure the data is received in the - * correct order. + * correct order. We also need a unique message tag in the + * case where only 1 subfile is used in total. In this case, + * vector I/O calls are passed directly down to this VFD without + * being split up into multiple I/O requests, so we need the + * tag to distinguish each I/O request. */ - need_data_tag = num_subfiles != num_io_concentrators; + need_data_tag = (num_subfiles == 1) || (num_subfiles != num_io_concentrators); if (!need_data_tag) data_tag = READ_INDEP_DATA; diff --git a/src/H5FDsubfiling/H5FDioc_threads.c b/src/H5FDsubfiling/H5FDioc_threads.c index c86157b..85c2561 100644 --- a/src/H5FDsubfiling/H5FDioc_threads.c +++ b/src/H5FDsubfiling/H5FDioc_threads.c @@ -456,8 +456,9 @@ translate_opcode(io_op_t op) case LOGGING_OP: return "LOGGING_OP"; break; + default: + return "unknown"; } - return "unknown"; } #endif @@ -873,9 +874,14 @@ ioc_file_queue_read_indep(sf_work_request_t *msg, int ioc_idx, int source, MPI_C * unpredictable order. However, if some IOCs own more than * 1 subfile, we need to associate each read with a unique * message tag to make sure the data is received in the - * correct order. + * correct order. We also need a unique message tag in the + * case where only 1 subfile is used in total. In this case, + * vector I/O calls are passed directly down to this VFD without + * being split up into multiple I/O requests, so we need the + * tag to distinguish each I/O request. */ - need_data_tag = sf_context->sf_num_subfiles != sf_context->topology->n_io_concentrators; + need_data_tag = (sf_context->sf_num_subfiles == 1) || + (sf_context->sf_num_subfiles != sf_context->topology->n_io_concentrators); if (!need_data_tag) send_tag = READ_INDEP_DATA; diff --git a/src/H5FDsubfiling/H5FDsubfiling.c b/src/H5FDsubfiling/H5FDsubfiling.c index 461fa16..bf175e6 100644 --- a/src/H5FDsubfiling/H5FDsubfiling.c +++ b/src/H5FDsubfiling/H5FDsubfiling.c @@ -121,6 +121,11 @@ typedef struct H5FD_subfiling_t { char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ } H5FD_subfiling_t; +typedef enum H5FD_subfiling_io_type_t { + IO_TYPE_WRITE, + IO_TYPE_READ, +} H5FD_subfiling_io_type_t; + /* * These macros check for overflow of various quantities. These macros * assume that HDoff_t is signed and haddr_t and size_t are unsigned. @@ -187,27 +192,52 @@ static int H5FD__copy_plist(hid_t fapl_id, hid_t *id_out_ptr); static herr_t H5FD__subfiling_close_int(H5FD_subfiling_t *file_ptr); -static herr_t init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_nelemts, - size_t dtype_extent, size_t max_iovec_len, int64_t *mem_buf_offset, - int64_t *target_file_offset, int64_t *io_block_len, int *first_subfile_index, - int *n_subfiles_used, int64_t *max_io_req_per_subfile); -static herr_t iovec_fill_first(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t target_datasize, - int64_t start_mem_offset, int64_t start_file_offset, int64_t first_io_len, - int64_t *mem_offset_out, int64_t *target_file_offset_out, - int64_t *io_block_len_out); -static herr_t iovec_fill_last(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t target_datasize, - int64_t start_mem_offset, int64_t start_file_offset, int64_t last_io_len, - int64_t *mem_offset_out, int64_t *target_file_offset_out, - int64_t *io_block_len_out); -static herr_t iovec_fill_first_last(subfiling_context_t *sf_context, int64_t iovec_depth, - int64_t target_datasize, int64_t start_mem_offset, - int64_t start_file_offset, int64_t first_io_len, int64_t last_io_len, - int64_t *mem_offset_out, int64_t *target_file_offset_out, - int64_t *io_block_len_out); -static herr_t iovec_fill_uniform(subfiling_context_t *sf_context, int64_t iovec_depth, +static herr_t H5FD__subfiling_io_helper(H5FD_subfiling_t *file_ptr, size_t io_count, H5FD_mem_t types[], + haddr_t addrs[], size_t sizes[], H5_flexible_const_ptr_t bufs[], + H5FD_subfiling_io_type_t io_type); +static herr_t H5FD__subfiling_mirror_writes_to_stub(H5FD_subfiling_t *file_ptr, uint32_t count, + H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], + const void *bufs[]); +static herr_t generate_io_vectors(subfiling_context_t *sf_context, size_t in_count, H5FD_mem_t types[], + haddr_t file_offsets[], size_t nelemts[], H5_flexible_const_ptr_t bufs[], + size_t dtype_extent, H5FD_subfiling_io_type_t io_type, size_t *ioreq_count, + uint32_t *iovec_len, H5FD_mem_t **io_types, haddr_t **io_addrs, + size_t **io_sizes, H5_flexible_const_ptr_t **io_bufs); +static void get_iovec_sizes(subfiling_context_t *sf_context, size_t in_count, haddr_t file_offsets[], + size_t nelemts[], size_t dtype_extent, size_t *max_iovec_depth, + size_t *max_num_subfiles); +static herr_t translate_io_req_to_iovec(subfiling_context_t *sf_context, size_t iovec_idx, size_t iovec_len, + size_t iovec_count, H5FD_mem_t type, haddr_t addr, size_t io_size, + H5_flexible_const_ptr_t io_buf, H5FD_subfiling_io_type_t io_type, + H5FD_mem_t *io_types, haddr_t *io_addrs, size_t *io_sizes, + H5_flexible_const_ptr_t *io_bufs); +static herr_t iovec_fill_first(subfiling_context_t *sf_context, size_t iovec_len, int64_t cur_iovec_depth, + int64_t target_datasize, int64_t start_mem_offset, int64_t start_file_offset, + int64_t first_io_len, H5_flexible_const_ptr_t buf, + H5FD_subfiling_io_type_t io_type, haddr_t *io_addrs_ptr, size_t *io_sizes_ptr, + H5_flexible_const_ptr_t *io_bufs_ptr); +static herr_t iovec_fill_last(subfiling_context_t *sf_context, size_t iovec_len, int64_t cur_iovec_depth, + int64_t target_datasize, int64_t start_mem_offset, int64_t start_file_offset, + int64_t last_io_len, H5_flexible_const_ptr_t buf, + H5FD_subfiling_io_type_t io_type, haddr_t *io_addrs_ptr, size_t *io_sizes_ptr, + H5_flexible_const_ptr_t *io_bufs_ptr); +static herr_t iovec_fill_first_last(subfiling_context_t *sf_context, size_t iovec_len, + int64_t cur_iovec_depth, int64_t target_datasize, + int64_t start_mem_offset, int64_t start_file_offset, int64_t first_io_len, + int64_t last_io_len, H5_flexible_const_ptr_t buf, + H5FD_subfiling_io_type_t io_type, haddr_t *io_addrs_ptr, + size_t *io_sizes_ptr, H5_flexible_const_ptr_t *io_bufs_ptr); +static herr_t iovec_fill_uniform(subfiling_context_t *sf_context, size_t iovec_len, int64_t cur_iovec_depth, int64_t target_datasize, int64_t start_mem_offset, int64_t start_file_offset, - int64_t *mem_offset_out, int64_t *target_file_offset_out, - int64_t *io_block_len_out); + H5_flexible_const_ptr_t buf, H5FD_subfiling_io_type_t io_type, + haddr_t *io_addrs_ptr, size_t *io_sizes_ptr, + H5_flexible_const_ptr_t *io_bufs_ptr); + +#ifdef H5_SUBFILING_DEBUG +void H5_subfiling_dump_iovecs(subfiling_context_t *sf_context, size_t ioreq_count, size_t iovec_len, + H5FD_subfiling_io_type_t io_type, H5FD_mem_t *io_types, haddr_t *io_addrs, + size_t *io_sizes, H5_flexible_const_ptr_t *io_bufs); +#endif void H5FD__subfiling_mpi_finalize(void); @@ -384,7 +414,7 @@ H5FD__subfiling_term(void) if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&H5_subfiling_rpc_msg_type))) H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code); } -#ifdef H5FD_SUBFILING_DEBUG +#ifdef H5_SUBFILING_DEBUG else printf("** WARNING **: HDF5 is terminating the Subfiling VFD after MPI_Finalize() was " "called - an HDF5 ID was probably left unclosed\n"); @@ -1535,199 +1565,16 @@ static herr_t H5FD__subfiling_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, void *buf /*out*/) { - subfiling_context_t *sf_context = NULL; - H5FD_subfiling_t *file_ptr = (H5FD_subfiling_t *)_file; - H5FD_mem_t *io_types = NULL; - haddr_t *io_addrs = NULL; - size_t *io_sizes = NULL; - void **io_bufs = NULL; - int64_t *source_data_offset = NULL; - int64_t *sf_data_size = NULL; - int64_t *sf_offset = NULL; - bool rank0_bcast = false; - int num_subfiles; - herr_t ret_value = SUCCEED; + H5FD_subfiling_t *file_ptr = (H5FD_subfiling_t *)_file; + herr_t ret_value = SUCCEED; - assert(file_ptr && file_ptr->pub.cls); + assert(file_ptr); + assert(file_ptr->pub.driver_id == H5FD_SUBFILING); assert(buf); - /* Check for overflow conditions */ - if (!H5_addr_defined(addr)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr undefined, addr = %" PRIuHADDR, addr); - if (REGION_OVERFLOW(addr, size)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, - "addr overflow, addr = %" PRIuHADDR ", size = %zu", addr, size); - - /* Temporarily reject collective I/O until support is implemented (unless types are simple MPI_BYTE) */ - { - H5FD_mpio_xfer_t xfer_mode; - - if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, - "can't determine I/O collectivity setting"); - - if (xfer_mode == H5FD_MPIO_COLLECTIVE) { - MPI_Datatype btype, ftype; - - if (H5CX_get_mpi_coll_datatypes(&btype, &ftype) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O datatypes"); - if (MPI_BYTE != btype || MPI_BYTE != ftype) - H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, - "collective I/O is currently unsupported"); - } - - /* Determine whether a rank 0 bcast approach has been requested */ - rank0_bcast = H5CX_get_mpio_rank0_bcast(); - - /* - * If we reached here, we're still doing independent I/O regardless - * of collectivity setting, so set that. - */ - H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT); - } - - /* - * Retrieve the subfiling context object and the number - * of subfiles. - * - * Given the current I/O and the I/O concentrator info, - * we can determine some I/O transaction parameters. - * In particular, for large I/O operations, each IOC - * may require multiple I/Os to fulfill the user I/O - * request. The block size and number of IOCs are used - * to size the vectors that will be used to invoke the - * underlying I/O operations. - */ - sf_context = (subfiling_context_t *)H5_get_subfiling_object(file_ptr->context_id); - assert(sf_context); - assert(sf_context->topology); - - num_subfiles = sf_context->sf_num_subfiles; - - if (num_subfiles <= 0) { - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid number of subfiles (%d)", - num_subfiles); - } - else if (num_subfiles == 1) { - /*************************************** - * No striping - just a single subfile * - ***************************************/ - - /* Make vector read call to subfile */ - if (H5FD_read_vector(file_ptr->sf_file, 1, &type, &addr, &size, &buf) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "read from subfile failed"); - } - else { - int64_t max_io_req_per_subfile; - int64_t file_offset; - int64_t block_size; - size_t max_depth; - herr_t status; - int num_subfiles_used = 0; - int first_subfile_idx = -1; - - /************************************* - * Striping across multiple subfiles * - *************************************/ - - block_size = sf_context->sf_blocksize_per_stripe; - max_depth = (size / (size_t)block_size) + 2; - - /* - * Given the number of subfiles, allocate vectors (one per subfile) - * to contain the translation of the I/O request into a collection of - * I/O requests. - */ - if (NULL == - (source_data_offset = calloc(1, (size_t)num_subfiles * max_depth * sizeof(*source_data_offset)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate source data offset I/O vector"); - if (NULL == (sf_data_size = calloc(1, (size_t)num_subfiles * max_depth * sizeof(*sf_data_size)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile data size I/O vector"); - if (NULL == (sf_offset = calloc(1, (size_t)num_subfiles * max_depth * sizeof(*sf_offset)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile offset I/O vector"); - - H5_CHECKED_ASSIGN(file_offset, int64_t, addr, haddr_t); - - /* - * Get the potential set of IOC transactions; e.g., data sizes, - * offsets and datatypes. - */ - status = init_indep_io(sf_context, /* IN: Context used to look up config info */ - file_offset, /* IN: Starting file offset */ - size, /* IN: I/O size */ - 1, /* IN: Data extent of the 'type' assumes byte */ - max_depth, /* IN: Maximum stripe depth */ - source_data_offset, /* OUT: Memory offset */ - sf_offset, /* OUT: File offset */ - sf_data_size, /* OUT: Length of this contiguous block */ - &first_subfile_idx, /* OUT: Subfile index corresponding to starting offset */ - &num_subfiles_used, /* OUT: Number of actual subfiles used */ - &max_io_req_per_subfile); /* OUT: Maximum number of requests to any subfile */ - - if (status < 0) - H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't initialize IOC transactions"); - - if (max_io_req_per_subfile > 0) { - uint32_t vector_len; - - H5_CHECKED_ASSIGN(vector_len, uint32_t, num_subfiles_used, int); - - /* Allocate I/O vectors */ - if (NULL == (io_types = malloc(vector_len * sizeof(*io_types)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O types vector"); - if (NULL == (io_addrs = malloc(vector_len * sizeof(*io_addrs)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O addresses vector"); - if (NULL == (io_sizes = malloc(vector_len * sizeof(*io_sizes)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O sizes vector"); - if (NULL == (io_bufs = malloc(vector_len * sizeof(*io_bufs)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O buffers vector"); - - for (int64_t i = 0; i < max_io_req_per_subfile; i++) { - uint32_t final_vec_len = vector_len; - int next_subfile_idx = first_subfile_idx; - - /* Fill in I/O types, offsets, sizes and buffers vectors */ - for (uint32_t k = 0, vec_idx = 0; k < vector_len; k++) { - size_t idx = (size_t)next_subfile_idx * max_depth + (size_t)i; - - io_types[vec_idx] = type; - H5_CHECKED_ASSIGN(io_addrs[vec_idx], haddr_t, sf_offset[idx], int64_t); - H5_CHECKED_ASSIGN(io_sizes[vec_idx], size_t, sf_data_size[idx], int64_t); - io_bufs[vec_idx] = ((char *)buf + source_data_offset[idx]); - - next_subfile_idx = (next_subfile_idx + 1) % num_subfiles; - - /* Skip 0-sized I/Os */ - if (io_sizes[vec_idx] == 0) { - final_vec_len--; - continue; - } - - vec_idx++; - } - - if (!rank0_bcast || (file_ptr->mpi_rank == 0)) { - /* Make vector read call to subfile */ - if (H5FD_read_vector(file_ptr->sf_file, final_vec_len, io_types, io_addrs, io_sizes, - io_bufs) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "read from subfile failed"); - } - } - - if (rank0_bcast && (file_ptr->mpi_size > 1)) { - H5_CHECK_OVERFLOW(size, size_t, int); - if (MPI_SUCCESS != MPI_Bcast(buf, (int)size, MPI_BYTE, 0, file_ptr->comm)) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't broadcast data from rank 0"); - } - } - } + if (H5FD__subfiling_io_helper(file_ptr, 1, &type, &addr, &size, (H5_flexible_const_ptr_t *)&buf, + IO_TYPE_READ) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read from subfiles failed"); /* Point to the end of the current I/O */ addr += (haddr_t)size; @@ -1737,14 +1584,6 @@ H5FD__subfiling_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_i file_ptr->op = OP_READ; done: - free(io_bufs); - free(io_sizes); - free(io_addrs); - free(io_types); - free(sf_offset); - free(sf_data_size); - free(source_data_offset); - if (ret_value < 0) { /* Reset last file I/O information */ file_ptr->pos = HADDR_UNDEF; @@ -1769,214 +1608,16 @@ static herr_t H5FD__subfiling_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, const void *buf /*in*/) { - subfiling_context_t *sf_context = NULL; - H5FD_subfiling_t *file_ptr = (H5FD_subfiling_t *)_file; - const void **io_bufs = NULL; - H5FD_mem_t *io_types = NULL; - haddr_t *io_addrs = NULL; - size_t *io_sizes = NULL; - int64_t *source_data_offset = NULL; - int64_t *sf_data_size = NULL; - int64_t *sf_offset = NULL; - int num_subfiles; - herr_t ret_value = SUCCEED; + H5FD_subfiling_t *file_ptr = (H5FD_subfiling_t *)_file; + herr_t ret_value = SUCCEED; - assert(file_ptr && file_ptr->pub.cls); + assert(file_ptr); + assert(file_ptr->pub.driver_id == H5FD_SUBFILING); assert(buf); - /* Check for overflow conditions */ - if (!H5_addr_defined(addr)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr undefined, addr = %" PRIuHADDR, addr); - if (REGION_OVERFLOW(addr, size)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, - "addr overflow, addr = %" PRIuHADDR ", size = %zu", addr, size); - - /* Temporarily reject collective I/O until support is implemented (unless types are simple MPI_BYTE) */ - { - H5FD_mpio_xfer_t xfer_mode; - - if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, - "can't determine I/O collectivity setting"); - - if (xfer_mode == H5FD_MPIO_COLLECTIVE) { - MPI_Datatype btype, ftype; - - if (H5CX_get_mpi_coll_datatypes(&btype, &ftype) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI-I/O datatypes"); - if (MPI_BYTE != btype || MPI_BYTE != ftype) - H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, - "collective I/O is currently unsupported"); - } - - /* - * If we reached here, we're still doing independent I/O regardless - * of collectivity setting, so set that. - */ - H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT); - } - - /* - * Retrieve the subfiling context object and the number - * of subfiles. - * - * Given the current I/O and the I/O concentrator info, - * we can determine some I/O transaction parameters. - * In particular, for large I/O operations, each IOC - * may require multiple I/Os to fulfill the user I/O - * request. The block size and number of IOCs are used - * to size the vectors that will be used to invoke the - * underlying I/O operations. - */ - sf_context = (subfiling_context_t *)H5_get_subfiling_object(file_ptr->context_id); - assert(sf_context); - assert(sf_context->topology); - - num_subfiles = sf_context->sf_num_subfiles; - - if (num_subfiles <= 0) { - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid number of subfiles (%d)", - num_subfiles); - } - else if (num_subfiles == 1) { - /*************************************** - * No striping - just a single subfile * - ***************************************/ - - /* Make vector write call to subfile */ - if (H5FD_write_vector(file_ptr->sf_file, 1, &type, &addr, &size, &buf) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write to subfile failed"); - - /* - * Mirror superblock writes to the stub file so that - * legacy HDF5 applications can check what type of - * file they are reading - */ - if ((type == H5FD_MEM_SUPER) && (file_ptr->mpi_rank == 0)) { - if (H5FD_write_vector(file_ptr->stub_file, 1, &type, &addr, &size, &buf) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "couldn't write superblock information to stub file"); - } - } - else { - int64_t max_io_req_per_subfile; - int64_t file_offset; - int64_t block_size; - size_t max_depth; - herr_t status; - int num_subfiles_used = 0; - int first_subfile_idx = -1; - - /************************************* - * Striping across multiple subfiles * - *************************************/ - - block_size = sf_context->sf_blocksize_per_stripe; - max_depth = (size / (size_t)block_size) + 2; - - /* - * Given the number of subfiles, allocate vectors (one per subfile) - * to contain the translation of the I/O request into a collection of - * I/O requests. - */ - if (NULL == - (source_data_offset = calloc(1, (size_t)num_subfiles * max_depth * sizeof(*source_data_offset)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate source data offset I/O vector"); - if (NULL == (sf_data_size = calloc(1, (size_t)num_subfiles * max_depth * sizeof(*sf_data_size)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile data size I/O vector"); - if (NULL == (sf_offset = calloc(1, (size_t)num_subfiles * max_depth * sizeof(*sf_offset)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile offset I/O vector"); - - H5_CHECKED_ASSIGN(file_offset, int64_t, addr, haddr_t); - - /* - * Get the potential set of IOC transactions; e.g., data sizes, - * offsets and datatypes. - */ - status = init_indep_io(sf_context, /* IN: Context used to look up config info */ - file_offset, /* IN: Starting file offset */ - size, /* IN: I/O size */ - 1, /* IN: Data extent of the 'type' assumes byte */ - max_depth, /* IN: Maximum stripe depth */ - source_data_offset, /* OUT: Memory offset */ - sf_offset, /* OUT: File offset */ - sf_data_size, /* OUT: Length of this contiguous block */ - &first_subfile_idx, /* OUT: Subfile index corresponding to starting offset */ - &num_subfiles_used, /* OUT: Number of actual subfiles used */ - &max_io_req_per_subfile); /* OUT: Maximum number of requests to any subfile */ - - if (status < 0) - H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't initialize IOC transactions"); - - if (max_io_req_per_subfile > 0) { - uint32_t vector_len; - - H5_CHECKED_ASSIGN(vector_len, uint32_t, num_subfiles_used, int); - - /* Allocate I/O vectors */ - if (NULL == (io_types = malloc(vector_len * sizeof(*io_types)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O types vector"); - if (NULL == (io_addrs = malloc(vector_len * sizeof(*io_addrs)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O addresses vector"); - if (NULL == (io_sizes = malloc(vector_len * sizeof(*io_sizes)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O sizes vector"); - if (NULL == (io_bufs = malloc(vector_len * sizeof(*io_bufs)))) - H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "can't allocate subfile I/O buffers vector"); - - for (int64_t i = 0; i < max_io_req_per_subfile; i++) { - uint32_t final_vec_len = vector_len; - int next_subfile_idx = first_subfile_idx; - - /* Fill in I/O types, offsets, sizes and buffers vectors */ - for (uint32_t k = 0, vec_idx = 0; k < vector_len; k++) { - size_t idx = (size_t)next_subfile_idx * max_depth + (size_t)i; - - io_types[vec_idx] = type; - H5_CHECKED_ASSIGN(io_addrs[vec_idx], haddr_t, sf_offset[idx], int64_t); - H5_CHECKED_ASSIGN(io_sizes[vec_idx], size_t, sf_data_size[idx], int64_t); - io_bufs[vec_idx] = ((const char *)buf + source_data_offset[idx]); - - next_subfile_idx = (next_subfile_idx + 1) % num_subfiles; - - /* Skip 0-sized I/Os */ - if (io_sizes[vec_idx] == 0) { - final_vec_len--; - continue; - } - - vec_idx++; - } - - /* Make vector write call to subfile */ - if (H5FD_write_vector(file_ptr->sf_file, final_vec_len, io_types, io_addrs, io_sizes, - io_bufs) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "write to subfile failed"); - - /* - * Mirror superblock writes to the stub file so that - * legacy HDF5 applications can check what type of - * file they are reading - */ - if (file_ptr->mpi_rank == 0) { - for (size_t count_idx = 0; count_idx < (size_t)final_vec_len; count_idx++) { - if (io_types[count_idx] == H5FD_MEM_SUPER) { - if (H5FD_write(file_ptr->stub_file, H5FD_MEM_SUPER, io_addrs[count_idx], - io_sizes[count_idx], io_bufs[count_idx]) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "couldn't write superblock information to stub file"); - } - } - } - } - } - } + if (H5FD__subfiling_io_helper(file_ptr, 1, &type, &addr, &size, (H5_flexible_const_ptr_t *)&buf, + IO_TYPE_WRITE) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to subfiles failed"); /* Point to the end of the current I/O */ addr += (haddr_t)size; @@ -1992,14 +1633,6 @@ H5FD__subfiling_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_ file_ptr->local_eof = file_ptr->pos; done: - free(io_bufs); - free(io_sizes); - free(io_addrs); - free(io_types); - free(sf_offset); - free(sf_data_size); - free(source_data_offset); - if (ret_value < 0) { /* Reset last file I/O information */ file_ptr->pos = HADDR_UNDEF; @@ -2048,31 +1681,21 @@ H5FD__subfiling_read_vector(H5FD_t *_file, hid_t dxpl_id, uint32_t count, H5FD_m size_t sizes[], void *bufs[] /* out */) { H5FD_subfiling_t *file_ptr = (H5FD_subfiling_t *)_file; - H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_INDEPENDENT; - herr_t ret_value = SUCCEED; /* Return value */ - - /* Check arguments - * RAW - Do we really need to check arguments once again? - * These have already been checked in H5FD__subfiling_read_vector (see below)! - */ - if (!file_ptr) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file pointer cannot be NULL"); - - if ((!types) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "types parameter can't be NULL if count is positive"); - - if ((!addrs) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "addrs parameter can't be NULL if count is positive"); + herr_t ret_value = SUCCEED; - if ((!sizes) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "sizes parameter can't be NULL if count is positive"); + assert(file_ptr); + assert(file_ptr->pub.driver_id == H5FD_SUBFILING); + assert((types) || (count == 0)); + assert((addrs) || (count == 0)); + assert((sizes) || (count == 0)); + assert((bufs) || (count == 0)); - if ((!bufs) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "bufs parameter can't be NULL if count is positive"); + /* + * Verify that the first elements of the sizes and + * types arrays are valid. + */ + assert((count == 0) || (sizes[0] != 0)); + assert((count == 0) || (types[0] != H5FD_MEM_NOLIST)); /* Get the default dataset transfer property list if the user didn't provide one */ if (H5P_DEFAULT == dxpl_id) { @@ -2086,98 +1709,27 @@ H5FD__subfiling_read_vector(H5FD_t *_file, hid_t dxpl_id, uint32_t count, H5FD_m /* Set DXPL for operation */ H5CX_set_dxpl(dxpl_id); - /* TODO: setup real support for vector I/O */ if (file_ptr->fa.require_ioc) { - - bool extend_sizes = false; - bool extend_types = false; - int k; - size_t size; - H5FD_mem_t type; - haddr_t eoa; - - assert((count == 0) || (sizes[0] != 0)); - assert((count == 0) || (types[0] != H5FD_MEM_NOLIST)); - - if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, - "can't determine I/O collectivity setting"); - - /* Currently, treat collective calls as independent */ - if (xfer_mode != H5FD_MPIO_INDEPENDENT) - if (H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTSET, FAIL, "can't set I/O collectivity setting"); - - /* Note that the following code does not let the sub-filing VFD participate - * in collective calls when there is no data to write. This is not an issue - * now, as we don't do anything special with collective operations. However - * this needs to be fixed. - */ - for (k = 0; k < (int)count; k++) { - - if (!extend_sizes) { - - if (sizes[k] == 0) { - - extend_sizes = true; - size = sizes[k - 1]; - } - else { - - size = sizes[k]; - } - } - - if (!extend_types) { - - if (types[k] == H5FD_MEM_NOLIST) { - - extend_types = true; - type = types[k - 1]; - } - else { - - type = types[k]; - } - } - - if (HADDR_UNDEF == (eoa = H5FD__subfiling_get_eoa(_file, type))) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed"); - - if ((addrs[k] + size) > eoa) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, - "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, eoa = %llu", - (int)k, (unsigned long long)(addrs[k]), (int)k, - (unsigned long long)size, (unsigned long long)eoa); - - if (H5FD__subfiling_read(_file, type, dxpl_id, addrs[k], size, bufs[k]) != SUCCEED) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file vector read request failed"); - } + if (H5FD__subfiling_io_helper(file_ptr, (size_t)count, types, addrs, sizes, + (H5_flexible_const_ptr_t *)bufs, IO_TYPE_READ) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't read data"); } else { - /* sec2 driver.. - * Call the subfiling 'direct write' version - * of subfiling. - */ - if (H5FD_read_vector(_file, count, types, addrs, sizes, bufs) != SUCCEED) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file vector read request failed"); + if (H5FD_read_vector(_file, count, types, addrs, sizes, bufs) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't read data"); } done: - if (xfer_mode != H5FD_MPIO_INDEPENDENT) - if (H5CX_set_io_xfer_mode(xfer_mode) < 0) - H5_SUBFILING_DONE_ERROR(H5E_CONTEXT, H5E_CANTSET, FAIL, "can't set I/O collectivity setting"); - H5_SUBFILING_FUNC_LEAVE_API; } /* end H5FD__subfiling_read_vector() */ /*------------------------------------------------------------------------- - * Function: H5FD__subfile_write_vector (internal function) + * Function: H5FD__subfiling_write_vector * * Purpose: Perform count writes to the specified file at the offsets - * provided in the addrs array. Lengths and memory - * types provided in the sizes and types arrays. Data to be - * written is referenced by the bufs array. + * provided in the addrs array. Lengths and memory types + * types are provided in the sizes and types arrays. Data to + * be written is referenced by the bufs array. * * All writes are done according to the data transfer property * list dxpl_id (which may be the constant H5P_DEFAULT). @@ -2190,17 +1742,6 @@ done: * input arguments are not valid, or the actual * subfiling writes have failed for some reason. * - * Notes: Thus function doesn't actually implement vector write. - * Instead, it converts the vector write call into a series - * of scalar read calls. Fix this when time permits. - * - * Also, it didn't support the sizes and types optimization. - * I implemented a version of this which is more generous - * than that currently defined in the RFC. This is good - * enough for now, but the final version should follow - * the RFC. - * JRM -- 10/5/21 - * *------------------------------------------------------------------------- */ static herr_t @@ -2208,33 +1749,21 @@ H5FD__subfiling_write_vector(H5FD_t *_file, hid_t dxpl_id, uint32_t count, H5FD_ haddr_t addrs[], size_t sizes[], const void *bufs[] /* in */) { H5FD_subfiling_t *file_ptr = (H5FD_subfiling_t *)_file; - H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_INDEPENDENT; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; - assert(file_ptr != NULL); /* sanity check */ + assert(file_ptr); + assert(file_ptr->pub.driver_id == H5FD_SUBFILING); + assert((types) || (count == 0)); + assert((addrs) || (count == 0)); + assert((sizes) || (count == 0)); + assert((bufs) || (count == 0)); - /* Check arguments - * RAW - Do we really need to check arguments once again? - * These have already been checked in H5FD__subfiling_write_vector (see below)! + /* + * Verify that the first elements of the sizes and + * types arrays are valid. */ - if (!file_ptr) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file pointer cannot be NULL"); - - if ((!types) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "types parameter can't be NULL if count is positive"); - - if ((!addrs) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "addrs parameter can't be NULL if count is positive"); - - if ((!sizes) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "sizes parameter can't be NULL if count is positive"); - - if ((!bufs) && (count > 0)) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "bufs parameter can't be NULL if count is positive"); + assert((count == 0) || (sizes[0] != 0)); + assert((count == 0) || (types[0] != H5FD_MEM_NOLIST)); /* Get the default dataset transfer property list if the user didn't provide one */ if (H5P_DEFAULT == dxpl_id) { @@ -2244,88 +1773,21 @@ H5FD__subfiling_write_vector(H5FD_t *_file, hid_t dxpl_id, uint32_t count, H5FD_ if (true != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list"); } - /* Call the subfiling IOC write*/ - if (file_ptr->fa.require_ioc) { - - bool extend_sizes = false; - bool extend_types = false; - int k; - size_t size; - H5FD_mem_t type; - haddr_t eoa; - - assert((count == 0) || (sizes[0] != 0)); - assert((count == 0) || (types[0] != H5FD_MEM_NOLIST)); - - if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, - "can't determine I/O collectivity setting"); - - /* Currently, treat collective calls as independent */ - if (xfer_mode != H5FD_MPIO_INDEPENDENT) - if (H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTSET, FAIL, "can't set I/O collectivity setting"); - - /* Note that the following code does not let the sub-filing VFD participate - * in collective calls when there is no data to write. This is not an issue - * now, as we don't do anything special with collective operations. However - * this needs to be fixed. - */ - for (k = 0; k < (int)count; k++) { - if (!extend_sizes) { + /* Set DXPL for operation */ + H5CX_set_dxpl(dxpl_id); - if (sizes[k] == 0) { - - extend_sizes = true; - size = sizes[k - 1]; - } - else { - - size = sizes[k]; - } - } - - if (!extend_types) { - - if (types[k] == H5FD_MEM_NOLIST) { - - extend_types = true; - type = types[k - 1]; - } - else { - - type = types[k]; - } - } - - if (HADDR_UNDEF == (eoa = H5FD__subfiling_get_eoa(_file, type))) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed"); - - if ((addrs[k] + size) > eoa) - H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, - "addr overflow, addrs[%d] = %llu, sizes[%d] = %llu, eoa = %llu", - (int)k, (unsigned long long)(addrs[k]), (int)k, - (unsigned long long)size, (unsigned long long)eoa); - - if (H5FD__subfiling_write(_file, type, dxpl_id, addrs[k], size, bufs[k]) != SUCCEED) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file vector write request failed"); - } + if (file_ptr->fa.require_ioc) { + if (H5FD__subfiling_io_helper(file_ptr, (size_t)count, types, addrs, sizes, + (H5_flexible_const_ptr_t *)bufs, IO_TYPE_WRITE) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't write data"); } else { - /* sec2 driver.. - * Call the subfiling 'direct write' version - * of subfiling. - */ - if (H5FD_write_vector(_file, count, types, addrs, sizes, bufs) != SUCCEED) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file vector write request failed"); + if (H5FD_write_vector(_file, count, types, addrs, sizes, bufs) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't write data"); } done: - if (xfer_mode != H5FD_MPIO_INDEPENDENT) - if (H5CX_set_io_xfer_mode(xfer_mode) < 0) - H5_SUBFILING_DONE_ERROR(H5E_CONTEXT, H5E_CANTSET, FAIL, "can't set I/O collectivity setting"); - H5_SUBFILING_FUNC_LEAVE_API; } /* end H5FDsubfile__write_vector() */ @@ -2581,138 +2043,918 @@ done: } /* end H5FD__subfiling_ctl() */ /*------------------------------------------------------------------------- - * Function: init_indep_io - * - * Purpose: Utility function to initialize the set of I/O transactions - * used to communicate with I/O concentrators for read and - * write I/O operations. - * - * Fills the I/O vectors contained in the output arrays - * `mem_buf_offset`, `target_file_offset` and `io_block_len`. - * As a consequence of not allowing use of MPI derived - * datatypes in the VFD layer, we need to accommodate the - * possibility that large I/O transactions will be required to - * use multiple I/Os per subfile. - * - * Example: Using 4 subfiles, each with 1M stripe-depth; when - * presented an I/O request for 8MB then at a minimum each - * subfile will require 2 I/Os of 1MB each. Depending on the - * starting file offset, the 2 I/Os can instead be 3... - * - * To fully describe the I/O transactions for reads and writes - * the output arrays are therefore arrays of I/O vectors, - * where each vector has a length of which corresponds to the - * max number of I/O transactions per subfile. In the example - * above, these vector lengths can be 2 or 3. The actual - * length is determined by the 'container_depth' variable. - * - * For I/O operations which involve a subset of subfiles, the - * vector entries for the unused subfiles will have lengths of - * zero and be empty. The 'container_depth' in this case will - * always be 1. + * Function: H5FD__subfiling_io_helper + * + * Purpose: Helper routine to manage the common portions of I/O between + * normal and vector I/O calls. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__subfiling_io_helper(H5FD_subfiling_t *file_ptr, size_t io_count, H5FD_mem_t types[], haddr_t addrs[], + size_t sizes[], H5_flexible_const_ptr_t bufs[], H5FD_subfiling_io_type_t io_type) +{ + H5_flexible_const_ptr_t *io_bufs = NULL; + subfiling_context_t *sf_context = NULL; + H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_INDEPENDENT; + H5FD_mem_t *io_types = NULL; + haddr_t *io_addrs = NULL; + size_t *io_sizes = NULL; + haddr_t file_eoa = HADDR_UNDEF; + size_t io_size = 0; + bool rank0_bcast = false; + bool extend_sizes = false; + int num_subfiles; + herr_t ret_value = SUCCEED; + + assert(file_ptr); + + if (HADDR_UNDEF == (file_eoa = H5FD__subfiling_get_eoa((const H5FD_t *)file_ptr, H5FD_MEM_DEFAULT))) + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get file EOA"); + + /* Perform some sanity checking on the given (address, size) pairs */ + extend_sizes = false; + for (size_t i = 0; i < io_count; i++) { + if (!extend_sizes) { + if ((i > 0) && (sizes[i] == 0)) { + extend_sizes = true; + io_size = sizes[i - 1]; + } + else { + io_size = sizes[i]; + } + } + + if (!H5_addr_defined(addrs[i])) + H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr[%zu] undefined, addr = %" PRIuHADDR, + i, addrs[i]); + if (REGION_OVERFLOW(addrs[i], io_size)) + H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, + "addr[%zu] overflow, addr = %" PRIuHADDR ", size = %zu", i, addrs[i], + io_size); + if ((addrs[i] + io_size) > file_eoa) + H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, + "addr overflow, addrs[%zu] = %" PRIuHADDR + ", sizes[%zu] = %zu, eoa = %" PRIuHADDR, + i, addrs[i], i, io_size, file_eoa); + } + + /* + * Temporarily reject collective I/O until support is + * implemented (unless types are simple MPI_BYTE), which + * can be properly handled here. + */ + if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "can't determine I/O collectivity setting"); + + if (xfer_mode == H5FD_MPIO_COLLECTIVE) { + MPI_Datatype btype, ftype; + + if (H5CX_get_mpi_coll_datatypes(&btype, &ftype) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "can't get MPI-I/O datatypes"); + if (MPI_BYTE != btype || MPI_BYTE != ftype) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL, "collective I/O is currently unsupported"); + } + + /* + * If we reached here, we're still doing independent I/O regardless + * of collectivity setting, so set that. + */ + H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT); + + /* Determine whether a rank 0 bcast approach has been requested */ + if (io_type == IO_TYPE_READ) + rank0_bcast = H5CX_get_mpio_rank0_bcast(); + + /* + * Retrieve the subfiling context object and the number + * of subfiles. + * + * Given the current I/O and the I/O concentrator info, + * we can determine some I/O transaction parameters. + * In particular, for large I/O operations, each IOC + * may require multiple I/Os to fulfill the user I/O + * request. The block size and number of IOCs are used + * to size the vectors that will be used to invoke the + * underlying I/O operations. + */ + if (NULL == (sf_context = (subfiling_context_t *)H5_get_subfiling_object(file_ptr->context_id))) + H5_SUBFILING_GOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, + "invalid or missing subfiling context object"); + assert(sf_context->topology); + + if ((num_subfiles = sf_context->sf_num_subfiles) <= 0) + H5_SUBFILING_GOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "invalid number of subfiles (%d)", + num_subfiles); + + if (num_subfiles == 1) { + uint32_t u32_io_count; + + /*************************************** + * No striping - just a single subfile * + ***************************************/ + + /* + * Convert the I/O count back to a uint32_t for the vector I/O + * call until the interface can possibly be changed to use size_t + * in the future + */ + H5_CHECKED_ASSIGN(u32_io_count, uint32_t, io_count, size_t); + + if (io_type == IO_TYPE_WRITE) { + /* Make vector write call to VFD controlling subfiles */ + if (H5FD_write_vector(file_ptr->sf_file, u32_io_count, types, addrs, sizes, (const void **)bufs) < + 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to subfile failed"); + + /* + * Mirror superblock writes to the stub file so that + * legacy HDF5 applications can check what type of + * file they are reading + */ + if (H5FD__subfiling_mirror_writes_to_stub(file_ptr, u32_io_count, types, addrs, sizes, + (const void **)bufs) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "mirrored write to stub file failed"); + } + else { + /* Make vector read call to VFD controlling subfiles */ + if (H5FD_read_vector(file_ptr->sf_file, u32_io_count, types, addrs, sizes, (void **)bufs) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read from subfile failed"); + } + } + else { + uint32_t iovec_len; + size_t ioreq_count = 0; + herr_t status; + + /************************************* + * Striping across multiple subfiles * + *************************************/ + + /* + * Generate the types, addrs, sizes and bufs I/O vectors for + * this I/O request. + */ + status = generate_io_vectors( + sf_context, /* IN: Subfiling context used to look up config info */ + io_count, /* IN: Number of entries in `types`, `addrs`, `sizes` and `bufs` */ + types, /* IN: Array of memory types */ + addrs, /* IN: Array of starting file offsets */ + sizes, /* IN: Array of I/O sizes (in terms of elements) */ + bufs, /* IN: Array of I/O buffers */ + 1, /* IN: Data extent of the 'type'; byte is assumed currently */ + io_type, /* IN: Type of I/O being performed (IO_TYPE_WRITE or IO_TYPE_READ) */ + &ioreq_count, /* OUT: Number of I/O requests to be made */ + &iovec_len, /* OUT: Number of elements in I/O vector for a single I/O request */ + &io_types, /* OUT: I/O vector of memory types for each I/O entry */ + &io_addrs, /* OUT: I/O vector of file addresses for each I/O entry */ + &io_sizes, /* OUT: I/O vector of I/O sizes for each I/O entry */ + &io_bufs); /* OUT: I/O vector of buffers for each I/O entry */ + + if (status < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't initialize I/O vectors"); + + /* Nothing to do + * + * TODO: Note that this does not let the sub-filing VFD participate in + * collective calls when there is no data to write. This is not an issue + * now, as we don't do anything special with collective operations. + * However, this needs to be fixed. + */ + if (ioreq_count == 0) + H5_SUBFILING_GOTO_DONE(SUCCEED); + +#ifdef H5_SUBFILING_DEBUG + H5_subfiling_dump_iovecs(sf_context, ioreq_count, iovec_len, io_type, io_types, io_addrs, io_sizes, + io_bufs); +#endif + + /* clang-format off */ + /* + * Having now populated the I/O vectors for this I/O request and + * having determined how many I/O calls need to be made to satisfy + * the entire I/O request, loop that many times, making an I/O call + * with each set of I/O vectors. Each I/O call uses a set of I/O + * vectors with a length of up to 'number of subfiles' elements and + * each I/O call's I/O vectors are setup to ensure that the I/O is + * spread across as many subfiles as possible for each iteration. In + * the simple case of N evenly-distributed and well-aligned I/O + * requests being performed on 4 subfiles, this can be visualized as + * the following: + * + * I/O REQ. 0 I/O REQ. 1 ... I/O REQ. N-1 + * || || || + * VV VV VV + * {IOVEC[0]} {IOVEC[4]} ... {IOVEC[(N-1 * iovec_len)]} -> SUBFILE 0 + * {IOVEC[1]} {IOVEC[5]} ... {IOVEC[(N-1 * iovec_len) + 1]} -> SUBFILE 1 + * {IOVEC[2]} {IOVEC[6]} ... {IOVEC[(N-1 * iovec_len) + 2]} -> SUBFILE 2 + * {IOVEC[3]} {IOVEC[7]} ... {IOVEC[(N-1 * iovec_len) + 3]} -> SUBFILE 3 + * + * where {IOVEC[X]} represents an I/O vector composed of the entries + * at index X of io_types, io_addrs, io_sizes and io_bufs. Note that + * the entire set of I/O vectors, e.g. [ {IOVEC[0]}, {IOVEC[1]}, {IOVEC[2]}, {IOVEC[3]} ] + * from the above visualization will be sent to the underlying I/O + * concentrator VFD in a single I/O call on each iteration. That VFD is + * ultimately responsible for mapping each I/O vector to its corresponding + * subfile (here, pointed to by '->' to the right of each I/O vector). + */ + /* clang-format on */ + for (size_t ioreq_idx = 0; ioreq_idx < ioreq_count; ioreq_idx++) { + H5_flexible_const_ptr_t *io_bufs_ptr = NULL; + H5FD_mem_t *io_types_ptr = NULL; + uint32_t final_vec_len = iovec_len; + haddr_t *io_addrs_ptr = NULL; + size_t *io_sizes_ptr = NULL; + + /* Setup index into I/O vectors for this I/O operation */ + io_types_ptr = &io_types[ioreq_idx * iovec_len]; + io_addrs_ptr = &io_addrs[ioreq_idx * iovec_len]; + io_sizes_ptr = &io_sizes[ioreq_idx * iovec_len]; + io_bufs_ptr = &io_bufs[ioreq_idx * iovec_len]; + + /* Skip 0-sized I/Os */ + for (size_t vec_idx = 0; vec_idx < iovec_len; vec_idx++) + if (io_sizes_ptr[vec_idx] == 0) + final_vec_len--; + + if (io_type == IO_TYPE_WRITE) { + /* Make vector write call to VFD controlling subfiles */ + if (H5FD_write_vector(file_ptr->sf_file, final_vec_len, io_types_ptr, io_addrs_ptr, + io_sizes_ptr, (const void **)io_bufs_ptr) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to subfile failed"); + + /* + * Mirror superblock writes to the stub file so that + * legacy HDF5 applications can check what type of + * file they are reading + */ + if (H5FD__subfiling_mirror_writes_to_stub(file_ptr, final_vec_len, io_types_ptr, io_addrs_ptr, + io_sizes_ptr, (const void **)io_bufs_ptr) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "mirrored write to stub file failed"); + } + else { + if (!rank0_bcast || (file_ptr->mpi_rank == 0)) { + /* Make vector read call to VFD controlling subfiles */ + if (H5FD_read_vector(file_ptr->sf_file, final_vec_len, io_types_ptr, io_addrs_ptr, + io_sizes_ptr, (void **)io_bufs_ptr) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read from subfile failed"); + } + } + } + + if (rank0_bcast && (file_ptr->mpi_size > 1)) { + size_t size; + + assert(io_type == IO_TYPE_READ); + + extend_sizes = false; + for (size_t i = 0; i < io_count; i++) { + if (!extend_sizes) { + if ((i > 0) && (sizes[i] == 0)) { + extend_sizes = true; + size = sizes[i - 1]; + } + else { + size = sizes[i]; + } + } + + H5_CHECK_OVERFLOW(size, size_t, int); + if (MPI_SUCCESS != MPI_Bcast(bufs[i].vp, (int)size, MPI_BYTE, 0, file_ptr->comm)) + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "can't broadcast data from rank 0"); + } + } + } + +done: + /* Restore original transfer mode if we changed it */ + if (xfer_mode != H5FD_MPIO_INDEPENDENT) + if (H5CX_set_io_xfer_mode(xfer_mode) < 0) + H5_SUBFILING_DONE_ERROR(H5E_CONTEXT, H5E_CANTSET, FAIL, "can't set I/O collectivity setting"); + + free(io_bufs); + free(io_sizes); + free(io_addrs); + free(io_types); + + H5_SUBFILING_FUNC_LEAVE; +} + +/*------------------------------------------------------------------------- + * Function: H5FD__subfiling_mirror_writes_to_stub + * + * Purpose: Mirrors write calls to the Subfiling stub file so that + * legacy HDF5 applications can check what type of file they + * are reading. Only superblock I/O is mirrored to the stub + * file and only if that I/O comes from MPI rank 0. This + * means that file metadata could be missed if it comes from + * other MPI ranks (such as when using a distributed metadata + * write strategy), but, at least currently, we generally only + * care about the first few bytes of the file being properly + * written to the stub file. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__subfiling_mirror_writes_to_stub(H5FD_subfiling_t *file_ptr, uint32_t count, H5FD_mem_t types[], + haddr_t addrs[], size_t sizes[], const void *bufs[]) +{ + const void **copied_bufs = NULL; + H5FD_mem_t *copied_types = NULL; + haddr_t *copied_addrs = NULL; + size_t *copied_sizes = NULL; + H5FD_mem_t type = H5FD_MEM_DEFAULT; + size_t io_size = 0; + bool all_super_writes = true; + bool some_super_writes = false; + bool extend_types = false; + bool extend_sizes = false; + herr_t ret_value = SUCCEED; + + assert(file_ptr); + + /* Only mirror I/O from MPI rank 0 */ + if (file_ptr->mpi_rank != 0) + H5_SUBFILING_GOTO_DONE(SUCCEED); + + if (count == 0) + H5_SUBFILING_GOTO_DONE(SUCCEED); + + for (size_t i = 0; i < count; i++) { + if (!extend_types) { + if ((i > 0) && (types[i] == H5FD_MEM_NOLIST)) { + extend_types = true; + type = types[i - 1]; + } + else { + type = types[i]; + } + } + + if (type == H5FD_MEM_SUPER) + some_super_writes = true; + else + all_super_writes = false; + + /* + * If we find H5FD_MEM_NOLIST, we don't need to + * keep looking through the array entries + */ + if (extend_types) + break; + } + + if (all_super_writes) { + if (H5FD_write_vector(file_ptr->stub_file, count, types, addrs, sizes, bufs) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "couldn't write superblock information to stub file"); + } + else if (some_super_writes) { + uint32_t vec_len = 0; + + /* Copy I/O vectors and strip out non-superblock I/O */ + + if (NULL == (copied_types = malloc(count * sizeof(*copied_types)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate copy of I/O types array"); + if (NULL == (copied_addrs = malloc(count * sizeof(*copied_addrs)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate copy of I/O addresses array"); + if (NULL == (copied_sizes = malloc(count * sizeof(*copied_sizes)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate copy of I/O sizes array"); + if (NULL == (copied_bufs = malloc(count * sizeof(*copied_bufs)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate copy of I/O buffers array"); + + extend_types = false; + extend_sizes = false; + for (size_t i = 0; i < count; i++) { + if (!extend_types) { + if ((i > 0) && (types[i] == H5FD_MEM_NOLIST)) { + extend_types = true; + type = types[i - 1]; + + /* End early if none of the remaining memory types are H5FD_MEM_SUPER */ + if (type != H5FD_MEM_SUPER) + break; + } + else { + type = types[i]; + } + } + + if (!extend_sizes) { + if ((i > 0) && (sizes[i] == 0)) { + extend_sizes = true; + io_size = sizes[i - 1]; + } + else { + io_size = sizes[i]; + } + } + + if (type != H5FD_MEM_SUPER) + continue; + + copied_types[vec_len] = type; + copied_addrs[vec_len] = addrs[i]; + copied_sizes[vec_len] = io_size; + copied_bufs[vec_len] = bufs[i]; + + vec_len++; + } + + if ((vec_len > 0) && (H5FD_write_vector(file_ptr->stub_file, vec_len, copied_types, copied_addrs, + copied_sizes, copied_bufs) < 0)) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "couldn't write superblock information to stub file"); + } + +done: + free(copied_bufs); + free(copied_sizes); + free(copied_addrs); + free(copied_types); + + H5_SUBFILING_FUNC_LEAVE; +} + +/*------------------------------------------------------------------------- + * Function: generate_io_vectors + * + * Purpose: Given an array of memory types, an array of file offsets, + * an array of the number of I/O elements for each file + * offset and an array of I/O buffers, translates each (type, + * offset, number of elements, I/O buffer) tuple into a set of + * I/O vectors according to the subfiling configuration + * specified in `sf_context`. These I/O vectors are generated + * such that a set of `iovec_len` elements from each of + * `io_types`, `io_addrs`, `io_sizes` and `io_bufs` can be + * passed to H5FD_write_vector/H5FD_read_vector and that I/O + * call will span as many subfiles as possible, parallelizing + * the I/O. Then, the next set of `iovec_len` elements can be + * passed and so on, until the whole I/O request has been + * parallelized across the subfiles. Once this function + * returns, `io_types`, `io_addrs`, `io_sizes` and `io_bufs` + * will each contain `ioreq_count` sets of I/O vectors, with + * each set containing `iovec_len` elements. * * sf_context (IN) * - the subfiling context for the file * - * file_offset (IN) - * - the starting file offset for I/O + * in_count (IN) + * - the number of entries in the `types`, `file_offsets`, + * `nelemts` and `bufs` arrays + * + * types (IN) + * - the memory types for each I/O entry + * + * file_offsets (IN) + * - array of starting file offsets for I/O + * + * nelemts (IN) + * - array of the number of data elements for the I/O + * operation * - * io_nelemts (IN) - * - the number of data elements for the I/O operation + * bufs (IN) + * - array of the I/O buffers to use for each I/O entry * * dtype_extent (IN) * - the extent of the datatype of each data element for - * the I/O operation - * - * max_iovec_len (IN) - * - the maximum size for a single I/O vector in each of - * the output arrays `mem_buf_offset`, `io_block_len` - * and `sf_offset`. NOTE that this routine expects each - * of these output arrays to have enough space allocated - * for one I/O vector PER subfile. Therefore, the total - * size of each output array should be at least - * `max_iovec_len * num_subfiles`. - * - * mem_buf_offset (OUT) - * - output array of vectors (one vector for each subfile) - * containing the set of offsets into the memory buffer - * for I/O - * - * target_file_offset (OUT) - * - output array of vectors (one vector for each subfile) - * containing the set of offsets into the target file - * - * io_block_len (OUT) - * - output array of vectors (one vector for each subfile) - * containing the set of block lengths for each source - * buffer/target file offset. - * - * first_subfile_index (OUT) - * - the index of the first subfile that this I/O operation - * begins at - * - * n_subfiles_used (OUT) - * - the number of subfiles actually used for this I/O - * operation, which may be different from the total - * number of subfiles for the file - * - * max_io_req_per_subfile (OUT) - * - the maximum number of I/O requests to any particular - * subfile, or the maximum "depth" of each I/O vector - * in the output arrays. + * the I/O operation (currently assumed to be 1, meaning + * entries in `nelemts` are expressed in terms of + * bytes) + * + * io_type (IN) + * - the type of I/O being performed (IO_TYPE_WRITE or + * IO_TYPE_READ) + * + * ioreq_count (OUT) + * - the number of I/O requests needed to fully satisfy the + * I/O operation + * + * iovec_len (OUT) + * - the size of each I/O vector (in terms of array elements) + * for each I/O request to be made + * + * io_types (OUT) + * - I/O vector of memory types for the I/O operation. + * Allocated by this function and must be freed by the + * caller. + * + * io_addrs (OUT) + * - I/O vector of file addresses for the I/O operation. + * Allocated by this function and must be freed by the + * caller. + * + * io_sizes (OUT) + * - I/O vector of the I/O sizes for the I/O operation. + * Allocated by this function and must be freed by the + * caller. + * + * io_bufs (OUT) + * - I/O vector of the I/O buffers for the I/O operation. + * Allocated by this function and must be freed by the + * caller. * * Return: Non-negative on success/Negative on failure * - *------------------------------------------------------------------------- */ static herr_t -init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_nelemts, size_t dtype_extent, - size_t max_iovec_len, int64_t *mem_buf_offset, int64_t *target_file_offset, - int64_t *io_block_len, int *first_subfile_index, int *n_subfiles_used, - int64_t *max_io_req_per_subfile) +generate_io_vectors(subfiling_context_t *sf_context, size_t in_count, H5FD_mem_t types[], + haddr_t file_offsets[], size_t nelemts[], H5_flexible_const_ptr_t bufs[], + size_t dtype_extent, H5FD_subfiling_io_type_t io_type, size_t *ioreq_count, + uint32_t *iovec_len, H5FD_mem_t **io_types, haddr_t **io_addrs, size_t **io_sizes, + H5_flexible_const_ptr_t **io_bufs) +{ + H5_flexible_const_ptr_t *loc_io_bufs = NULL; + H5FD_mem_t *loc_io_types = NULL; + H5FD_mem_t mem_type = H5FD_MEM_DEFAULT; + haddr_t *loc_io_addrs = NULL; + size_t *loc_io_sizes = NULL; + size_t max_iovec_depth = 0; + size_t max_num_subfiles_touched = 0; + size_t tot_iovec_len = 0; + size_t io_size = 0; + bool extend_sizes = false; + bool extend_types = false; + herr_t ret_value = SUCCEED; + + assert(sf_context); + assert(sf_context->sf_stripe_size > 0); + assert(sf_context->sf_blocksize_per_stripe > 0); + assert(sf_context->sf_num_subfiles > 0); + assert(sf_context->topology); + assert((types) || (in_count == 0)); + assert((file_offsets) || (in_count == 0)); + assert((nelemts) || (in_count == 0)); + assert((bufs) || (in_count == 0)); + assert(dtype_extent == 1); /* For now, assume 'byte'-sized elements */ + assert(ioreq_count); + assert(iovec_len); + assert(io_types); + assert(io_addrs); + assert(io_sizes); + assert(io_bufs); + + /* Set some returned values early */ + *ioreq_count = 0; + *iovec_len = 0; + + /* Nothing to do */ + if (in_count == 0) + H5_SUBFILING_GOTO_DONE(SUCCEED); + + /* + * Do some initial pre-processing to determine how large of + * I/O vectors we will need to allocate to satisfy the + * entire I/O request + */ + get_iovec_sizes(sf_context, in_count, file_offsets, nelemts, dtype_extent, &max_iovec_depth, + &max_num_subfiles_touched); + + tot_iovec_len = in_count * max_iovec_depth * max_num_subfiles_touched; + +#ifdef H5_SUBFILING_DEBUG + H5_subfiling_log( + sf_context->sf_context_id, + "%s: I/O count: %zu, max_iovec_depth = %zu, max_num_subfiles_touched = %zu, iovec_len = %zu", + __func__, in_count, max_iovec_depth, max_num_subfiles_touched, tot_iovec_len); +#endif + + /* Allocate I/O vectors that will be returned to the caller */ + if (NULL == (loc_io_types = calloc(1, tot_iovec_len * sizeof(*loc_io_types)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate subfile I/O types vector"); + if (NULL == (loc_io_addrs = calloc(1, tot_iovec_len * sizeof(*loc_io_addrs)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate subfile I/O addresses vector"); + if (NULL == (loc_io_sizes = calloc(1, tot_iovec_len * sizeof(*loc_io_sizes)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate subfile I/O sizes vector"); + if (NULL == (loc_io_bufs = calloc(1, tot_iovec_len * sizeof(*loc_io_bufs)))) + H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate subfile I/O buffers vector"); + + /* + * Populate the I/O vectors by looping through each + * of the (type, addrs, I/O size, buf) tuples + */ + for (size_t io_idx = 0; io_idx < in_count; io_idx++) { + size_t iovec_idx; + + iovec_idx = (io_idx * max_iovec_depth * max_num_subfiles_touched); + assert(iovec_idx < tot_iovec_len); + + if (!extend_types) { + if ((io_idx > 0) && (types[io_idx] == H5FD_MEM_NOLIST)) { + extend_types = true; + mem_type = types[io_idx - 1]; + } + else { + mem_type = types[io_idx]; + } + } + + if (!extend_sizes) { + if ((io_idx > 0) && (nelemts[io_idx] == 0)) { + extend_sizes = true; + io_size = nelemts[io_idx - 1] * dtype_extent; + } + else { + io_size = nelemts[io_idx] * dtype_extent; + } + } + + if (translate_io_req_to_iovec(sf_context, iovec_idx, max_num_subfiles_touched, max_iovec_depth, + mem_type, file_offsets[io_idx], io_size, bufs[io_idx], io_type, + loc_io_types, loc_io_addrs, loc_io_sizes, loc_io_bufs) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't translate I/O request to I/O vectors"); + } + + *ioreq_count = in_count * max_iovec_depth; + H5_CHECK_OVERFLOW(max_num_subfiles_touched, size_t, uint32_t); + *iovec_len = (uint32_t)max_num_subfiles_touched; + *io_types = loc_io_types; + *io_addrs = loc_io_addrs; + *io_sizes = loc_io_sizes; + *io_bufs = loc_io_bufs; + +done: + if (ret_value < 0) { + free(loc_io_bufs); + free(loc_io_sizes); + free(loc_io_addrs); + free(loc_io_types); + } + + H5_SUBFILING_FUNC_LEAVE; +} + +/*------------------------------------------------------------------------- + * Function: get_iovec_sizes + * + * Purpose: Helper routine to determine the maximum I/O vector depth + * (in terms of array elements) and maximum number of subfiles + * touched for any particular piece of an I/O request. This + * info is used to calculate the total size of I/O vectors we + * need to allocate to satisfy an entire I/O request. + * + * Return: Maximum I/O vector depth and maximum number of subfiles + * touched (can't fail) + * + *------------------------------------------------------------------------- + */ +static void +get_iovec_sizes(subfiling_context_t *sf_context, size_t in_count, haddr_t file_offsets[], size_t nelemts[], + size_t dtype_extent, size_t *max_iovec_depth, size_t *max_num_subfiles) { int64_t stripe_size = 0; int64_t block_size = 0; - int64_t data_size = 0; + size_t loc_max_iovec_depth = 0; + size_t loc_max_num_subfiles = 0; + int num_subfiles = 0; + + assert(sf_context); + assert(file_offsets); + assert(nelemts); + assert(max_iovec_depth); + assert(max_num_subfiles); + + stripe_size = sf_context->sf_stripe_size; + block_size = sf_context->sf_blocksize_per_stripe; + num_subfiles = sf_context->sf_num_subfiles; + + for (size_t io_idx = 0; io_idx < in_count; io_idx++) { + int64_t stripe_idx; + int64_t final_stripe_idx; + int64_t cur_file_offset; + int64_t final_offset; + int64_t data_size; + int64_t first_subfile; + int64_t last_subfile; + int64_t row_stripe_idx_start; + int64_t row_stripe_idx_final; + int64_t cur_max_num_subfiles; + size_t cur_iovec_depth; + + H5_CHECKED_ASSIGN(cur_file_offset, int64_t, file_offsets[io_idx], haddr_t); + H5_CHECKED_ASSIGN(data_size, int64_t, (nelemts[io_idx] * dtype_extent), size_t); + + /* + * Calculate the following from the starting file offset: + * + * stripe_idx + * - a stripe "index" given by the file offset divided by the + * stripe size. Note that when the file offset equals or exceeds + * the block size, we simply wrap around. So, for example, if 4 + * subfiles are being used with a stripe size of 1KiB, the block + * size would be 4KiB and file offset 4096 would have a stripe + * index of 4 and reside in the same subfile as stripe index 0 + * (offsets 0-1023) + * final_offset + * - the last offset in the virtual file covered by this I/O + * operation. Simply the I/O size added to the starting file + * offset. + */ + stripe_idx = cur_file_offset / stripe_size; + final_offset = cur_file_offset + data_size; + + /* Determine which subfile the I/O request begins in */ + first_subfile = stripe_idx % num_subfiles; + + /* + * Determine the stripe "index" of the last offset in the + * virtual file and, from that, determine the subfile that + * the I/O request ends in. + */ + final_stripe_idx = final_offset / stripe_size; + last_subfile = final_stripe_idx % num_subfiles; + + /* + * Determine how "deep" the resulting I/O vectors are at + * most by calculating the maximum number of "rows" spanned + * for any particular subfile; e.g. the maximum number of + * I/O requests for any particular subfile + */ + row_stripe_idx_start = stripe_idx - first_subfile; + row_stripe_idx_final = final_stripe_idx - last_subfile; + cur_iovec_depth = (size_t)((row_stripe_idx_final - row_stripe_idx_start) / num_subfiles) + 1; + + /* + * If the I/O request "wrapped around" and ends in a subfile + * less than the subfile we started in, subtract one from the + * I/O vector length to account for "empty space". This can be + * visualized as follows: + * + * SUBFILE 0 SUBFILE 1 SUBFILE 2 SUBFILE 3 + * _______________________________________________ + * | | | XXXXX | XXXXX | ROW 0 + * | XXXXX | XXXXX | XXXXX | XXXXX | ROW 1 + * | XXXXX | XXXXX | | | ROW 2 + * | | | | | ROW ... + * | | | | | + * | | | | | + * | | | | | + * |___________|___________|___________|___________| + * + * Here, `stripe_idx` would be calculated as 2 (I/O begins in + * the 3rd stripe, or subfile index 2), `first_subfile` would be + * calculated as 2 and the starting "row" (row_stripe_idx_start) + * would be calculated as "row" index 0. `final_stripe_idx` would + * be calculated as 9, `last_subfile` would be calculated as + * (9 % 4) = 1 and the ending "row" (row_stripe_idx_final) would + * be calculated as (9 - 1) = 8. Thus, the calculated I/O vector + * length would be ((8 - 0) / 4) + 1 = 3. However, since there is + * no I/O to stripe indices 0 and 1 (residing in "row" 0 of subfile + * index 0 and 1, respectively), it can be seen that the real I/O + * vector length is 2. + */ + if (last_subfile < first_subfile) + cur_iovec_depth--; + + loc_max_iovec_depth = MAX(cur_iovec_depth, loc_max_iovec_depth); + + /* + * Determine the maximum number of subfiles this piece of the + * I/O request could touch + */ + if (data_size >= block_size) { + /* + * I/O of a size greater than the block size definitionally + * touches all subfiles at least once. + */ + cur_max_num_subfiles = (size_t)num_subfiles; + } + else if (data_size < stripe_size) { + /* + * I/O of a size smaller than the stripe size could + * touch one or two subfiles at most, depending on + * the file offset. + */ + cur_max_num_subfiles = 2; + } + else { + /* + * I/O of a size smaller than the block size, but larger + * than or equal to the stripe size must touch at least + * (data_size / stripe_size) subfiles, but could touch + * an additional subfile, depending on the file offset. + */ + cur_max_num_subfiles = (((cur_file_offset % stripe_size) + data_size - 1) / stripe_size) + 1; + } + + loc_max_num_subfiles = MAX((size_t)cur_max_num_subfiles, loc_max_num_subfiles); + } + + *max_iovec_depth = loc_max_iovec_depth; + *max_num_subfiles = loc_max_num_subfiles; +} + +/*------------------------------------------------------------------------- + * Function: translate_io_req_to_iovec + * + * Purpose: Helper routine to perform the translation between an I/O + * request (type, addr, size, buf tuple) and a set of I/O + * vectors that spans all the subfiles touched by that I/O + * request. Once finished, this function will have generated + * at most `iovec_count` sets of I/O vectors, each containing + * `iovec_len` elements, but a smaller number of I/O vector + * sets could be generated, depending on the I/O request. + * + * sf_context (IN) + * - the subfiling context for the file + * + * iovec_idx (IN) + * - the index into `io_types`, `io_addrs`, `io_sizes` and + * `io_bufs` where this function should begin filling in + * the I/O vectors + * + * iovec_len (IN) + * - the number of elements in each I/O vector generated + * + * iovec_count (IN) + * - the maximum number of I/O vectors to be generated, as + * calculated in generate_io_vectors() + * + * type (IN) + * - the memory type to use for each component of the I/O + * vectors generated + * + * addr (IN) + * - the starting file offset used to generate the I/O + * vectors + * + * io_size (IN) + * - the size of the I/O to the given file offset, which is + * used when generating the I/O vectors + * + * io_buf (IN) + * - the I/O buffer to be partitioned up while generating + * the I/O vectors + * + * io_type (IN) + * - the type of I/O being performed (IO_TYPE_WRITE or + * IO_TYPE_READ) + * + * io_types (OUT) + * - pointer to the memory types I/O vector to populate + * + * io_addrs (OUT) + * - pointer to the file offsets I/O vector to populate + * + * io_sizes (OUT) + * - pointer to the I/O sizes I/O vector to populate + * + * io_bufs (OUT) + * - pointer to the I/O buffers I/O vector to populate + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +translate_io_req_to_iovec(subfiling_context_t *sf_context, size_t iovec_idx, size_t iovec_len, + size_t iovec_count, H5FD_mem_t type, haddr_t addr, size_t io_size, + H5_flexible_const_ptr_t io_buf, H5FD_subfiling_io_type_t io_type, + H5FD_mem_t *io_types, haddr_t *io_addrs, size_t *io_sizes, + H5_flexible_const_ptr_t *io_bufs) +{ int64_t stripe_idx = 0; int64_t final_stripe_idx = 0; - int64_t curr_stripe_idx = 0; + int64_t stripe_size = 0; + int64_t block_size = 0; + int64_t file_offset = 0; int64_t offset_in_stripe = 0; int64_t offset_in_block = 0; int64_t final_offset = 0; int64_t start_length = 0; int64_t final_length = 0; - int64_t first_subfile = 0; - int64_t last_subfile = 0; + int64_t first_subfile_idx = 0; + int64_t last_subfile_idx = 0; int64_t start_row = 0; int64_t row_offset = 0; int64_t row_stripe_idx_start = 0; int64_t row_stripe_idx_final = 0; + int64_t cur_stripe_idx = 0; int64_t max_iovec_depth = 0; - int64_t curr_max_iovec_depth = 0; - int64_t total_bytes = 0; int64_t mem_offset = 0; + size_t total_bytes = 0; int num_subfiles = 0; herr_t ret_value = SUCCEED; assert(sf_context); - assert(sf_context->sf_stripe_size > 0); - assert(sf_context->sf_blocksize_per_stripe > 0); - assert(sf_context->sf_num_subfiles > 0); - assert(sf_context->topology); - assert(mem_buf_offset); - assert(target_file_offset); - assert(io_block_len); - assert(first_subfile_index); - assert(n_subfiles_used); - assert(max_io_req_per_subfile); - - *first_subfile_index = 0; - *n_subfiles_used = 0; - *max_io_req_per_subfile = 0; + assert(io_types); + assert(io_addrs); + assert(io_sizes); + assert(io_bufs); /* - * Retrieve the needed fields from the subfiling context. + * Retrieve some needed fields from the subfiling context. * * stripe_size * - the size of the data striping across the file's subfiles @@ -2723,15 +2965,13 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne * num_subfiles * - the total number of subfiles for the logical * HDF5 file - * num_io_concentrators - * - the number of I/O concentrators currently being - * used */ stripe_size = sf_context->sf_stripe_size; block_size = sf_context->sf_blocksize_per_stripe; num_subfiles = sf_context->sf_num_subfiles; - H5_CHECKED_ASSIGN(data_size, int64_t, (io_nelemts * dtype_extent), size_t); + H5_CHECKED_ASSIGN(file_offset, int64_t, addr, haddr_t); + H5_CHECK_OVERFLOW(io_size, size_t, int64_t); /* * Calculate the following from the starting file offset: @@ -2740,8 +2980,8 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne * - a stripe "index" given by the file offset divided by the * stripe size. Note that when the file offset equals or exceeds * the block size, we simply wrap around. So, for example, if 4 - * subfiles are being used with a stripe size of 1MiB, the block - * size would be 4MiB and file offset 4096 would have a stripe + * subfiles are being used with a stripe size of 1KiB, the block + * size would be 4KiB and file offset 4096 would have a stripe * index of 4 and reside in the same subfile as stripe index 0 * (offsets 0-1023) * offset_in_stripe @@ -2752,17 +2992,22 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne * subfiles * final_offset * - the last offset in the virtual file covered by this I/O - * operation. Simply the I/O size added to the starting file - * offset. + * request. Simply the I/O size minus one byte added to the + * starting file offset. */ stripe_idx = file_offset / stripe_size; offset_in_stripe = file_offset % stripe_size; offset_in_block = file_offset % block_size; - final_offset = file_offset + data_size; + final_offset = file_offset + (int64_t)(io_size > 0 ? io_size - 1 : 0); /* Determine the size of data written to the first and last stripes */ - start_length = MIN(data_size, (stripe_size - offset_in_stripe)); - final_length = (start_length == data_size ? 0 : final_offset % stripe_size); + start_length = MIN((int64_t)io_size, (stripe_size - offset_in_stripe)); + if (start_length == (int64_t)io_size) + final_length = 0; + else if (((final_offset + 1) % stripe_size) == 0) + final_length = stripe_size; + else + final_length = (final_offset + 1) % stripe_size; assert(start_length <= stripe_size); assert(final_length <= stripe_size); @@ -2776,9 +3021,9 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne * file offsets that are multiples of the block size begin a new * "row". */ - start_row = stripe_idx / num_subfiles; - first_subfile = stripe_idx % num_subfiles; - H5_CHECK_OVERFLOW(first_subfile, int64_t, int); + start_row = stripe_idx / num_subfiles; + first_subfile_idx = stripe_idx % num_subfiles; + H5_CHECK_OVERFLOW(first_subfile_idx, int64_t, int); /* * Set initial file offset for starting "row" @@ -2792,34 +3037,62 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne * the I/O request ends in. */ final_stripe_idx = final_offset / stripe_size; - last_subfile = final_stripe_idx % num_subfiles; + last_subfile_idx = final_stripe_idx % num_subfiles; /* - * Determine how "deep" the resulting I/O vectors are at - * most by calculating the maximum number of "rows" spanned - * for any particular subfile; e.g. the maximum number of - * I/O requests for any particular subfile + * Determine how "deep" the current I/O vector is at most + * by calculating the maximum number of "rows" spanned for + * any particular subfile; e.g. the maximum number of I/O + * requests for any particular subfile */ - row_stripe_idx_start = stripe_idx - first_subfile; - row_stripe_idx_final = final_stripe_idx - last_subfile; + row_stripe_idx_start = stripe_idx - first_subfile_idx; + row_stripe_idx_final = final_stripe_idx - last_subfile_idx; max_iovec_depth = ((row_stripe_idx_final - row_stripe_idx_start) / num_subfiles) + 1; - if (last_subfile < first_subfile) + /* + * If the I/O request "wrapped around" and ends in a subfile + * less than the subfile we started in, subtract one from the + * I/O vector length to account for "empty space". This can be + * visualized as follows: + * + * SUBFILE 0 SUBFILE 1 SUBFILE 2 SUBFILE 3 + * _______________________________________________ + * | | | XXXXX | XXXXX | ROW 0 + * | XXXXX | XXXXX | XXXXX | XXXXX | ROW 1 + * | XXXXX | XXXXX | | | ROW 2 + * | | | | | ROW ... + * | | | | | + * | | | | | + * | | | | | + * |___________|___________|___________|___________| + * + * Here, `stripe_idx` would be calculated as 2 (I/O begins in + * the 3rd stripe, or subfile index 2), `first_subfile` would be + * calculated as 2 and the starting "row" (row_stripe_idx_start) + * would be calculated as "row" index 0. `final_stripe_idx` would + * be calculated as 9, `last_subfile` would be calculated as + * (9 % 4) = 1 and the ending "row" (row_stripe_idx_final) would + * be calculated as (9 - 1) = 8. Thus, the calculated I/O vector + * length would be ((8 - 0) / 4) + 1 = 3. However, since there is + * no I/O to stripe indices 0 and 1 (residing in "row" 0 of subfile + * index 0 and 1, respectively), it can be seen that the real I/O + * vector length is 2. + */ + if (last_subfile_idx < first_subfile_idx) max_iovec_depth--; - /* Set returned parameters early */ - *first_subfile_index = (int)first_subfile; - *n_subfiles_used = num_subfiles; - *max_io_req_per_subfile = max_iovec_depth; - #ifdef H5_SUBFILING_DEBUG - H5_subfiling_log(sf_context->sf_context_id, - "%s: FILE OFFSET = %" PRId64 ", DATA SIZE = %zu, STRIPE SIZE = %" PRId64, __func__, - file_offset, io_nelemts, stripe_size); - H5_subfiling_log(sf_context->sf_context_id, - "%s: FIRST SUBFILE = %" PRId64 ", LAST SUBFILE = %" PRId64 ", " - "MAX IOVEC DEPTH = %" PRId64 ", START LENGTH = %" PRId64 ", FINAL LENGTH = %" PRId64, - __func__, first_subfile, last_subfile, max_iovec_depth, start_length, final_length); + H5_subfiling_log( + sf_context->sf_context_id, + "%s: TRANSLATING I/O REQUEST (MEMORY TYPE: %d, ADDR: %" PRIuHADDR ", I/O SIZE: %zu, BUF: %p)\n" + "STRIPE SIZE: %" PRId64 ", BLOCK SIZE: %" PRId64 ", NUM SUBFILES: %d\n" + "STRIPE IDX: %" PRId64 ", LAST STRIPE IDX: %" PRId64 ", FIRST SUBFILE IDX: %" PRId64 + ", LAST SUBFILE IDX: %" PRId64 "\n" + "START SEGMENT LENGTH: %" PRId64 ", LAST SEGMENT LENGTH: %" PRId64 ", MAX IOVEC DEPTH: %" PRId64, + __func__, type, addr, io_size, + (io_type == IO_TYPE_WRITE) ? (const void *)io_buf.cvp : (void *)io_buf.vp, stripe_size, block_size, + num_subfiles, stripe_idx, final_stripe_idx, first_subfile_idx, last_subfile_idx, start_length, + final_length, max_iovec_depth); #endif /* @@ -2827,131 +3100,162 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne * vector components for each. Subfiles whose data size is * zero will not have I/O requests passed to them. */ - curr_stripe_idx = stripe_idx; - curr_max_iovec_depth = max_iovec_depth; - for (int i = 0, k = (int)first_subfile; i < num_subfiles; i++) { - int64_t *_mem_buf_offset; - int64_t *_target_file_offset; - int64_t *_io_block_len; - int64_t subfile_bytes = 0; - int64_t iovec_depth; - bool is_first = false; - bool is_last = false; - size_t output_offset; - - iovec_depth = curr_max_iovec_depth; + cur_stripe_idx = stripe_idx; + for (int i = 0, subfile_idx = (int)first_subfile_idx; i < num_subfiles; i++) { + H5_flexible_const_ptr_t *_io_bufs_ptr; + H5FD_mem_t *_io_types_ptr; + haddr_t *_io_addrs_ptr; + size_t *_io_sizes_ptr; + int64_t iovec_depth; + int64_t num_full_stripes; + int64_t subfile_bytes = 0; + bool is_first = false; + bool is_last = false; + + if (total_bytes >= io_size) + break; - /* - * Setup the pointers to the next set of I/O vectors in - * the output arrays and clear those vectors - */ - output_offset = (size_t)(k)*max_iovec_len; - _mem_buf_offset = mem_buf_offset + output_offset; - _target_file_offset = target_file_offset + output_offset; - _io_block_len = io_block_len + output_offset; - - memset(_mem_buf_offset, 0, (max_iovec_len * sizeof(*_mem_buf_offset))); - memset(_target_file_offset, 0, (max_iovec_len * sizeof(*_target_file_offset))); - memset(_io_block_len, 0, (max_iovec_len * sizeof(*_io_block_len))); - - if (total_bytes == data_size) { - *n_subfiles_used = i; - goto done; - } + iovec_depth = max_iovec_depth; + num_full_stripes = iovec_depth; - if (total_bytes < data_size) { - int64_t num_full_stripes = iovec_depth; + if (subfile_idx == first_subfile_idx) { + is_first = true; - if (k == first_subfile) { - is_first = true; + /* + * Add partial segment length if not + * starting on a stripe boundary + */ + if (start_length < stripe_size) { + subfile_bytes += start_length; + num_full_stripes--; + } + } - /* - * Add partial segment length if not - * starting on a stripe boundary - */ - if (start_length < stripe_size) { - subfile_bytes += start_length; + if (subfile_idx == last_subfile_idx) { + is_last = true; + + /* + * Add partial segment length if not + * ending on a stripe boundary + */ + if (final_length < stripe_size) { + subfile_bytes += final_length; + if (num_full_stripes) num_full_stripes--; - } } + } - if (k == last_subfile) { - is_last = true; + /* Account for subfiles with uniform segments */ + if (!is_first && !is_last) { + bool thin_uniform_section = false; + if (last_subfile_idx >= first_subfile_idx) { /* - * Add partial segment length if not - * ending on a stripe boundary + * In the case where the subfile with the final data + * segment has an index value greater than or equal + * to the subfile with the first data segment, I/O + * vectors directed to a subfile with an index value + * that is greater than the last subfile or less than + * the first subfile will be "thin", or rather will + * have a vector depth of 1 less than normal, which + * will be accounted for below. This can be visualized + * with the following I/O pattern: + * + * SUBFILE 0 SUBFILE 1 SUBFILE 2 SUBFILE 3 + * _______________________________________________ + * | | XXXXX | XXXXX | XXXXX | ROW 0 + * | XXXXX | XXXXX | XXXXX | | ROW 1 + * | | | | | ROW 2 + * | | | | | ROW ... + * | | | | | + * | | | | | + * | | | | | + * |___________|___________|___________|___________| + * (thin) (thin) */ - if (final_length < stripe_size) { - subfile_bytes += final_length; - if (num_full_stripes) - num_full_stripes--; - } + thin_uniform_section = (subfile_idx > last_subfile_idx) || (subfile_idx < first_subfile_idx); + } + else { /* last_subfile_idx < first_subfile_idx */ + /* + * This can also happen when the subfile with the final + * data segment has a smaller subfile index than the + * subfile with the first data segment and the current + * subfile index falls between the two. + */ + thin_uniform_section = + ((last_subfile_idx < subfile_idx) && (subfile_idx < first_subfile_idx)); } - /* Account for subfiles with uniform segments */ - if (!is_first && !is_last) { - bool thin_uniform_section = false; - - if (last_subfile >= first_subfile) { - /* - * When a subfile has an index value that is greater - * than both the starting subfile and ending subfile - * indices, it is a "thinner" section with a smaller - * I/O vector depth. - */ - thin_uniform_section = (k > first_subfile) && (k > last_subfile); - } + if (thin_uniform_section) { + assert(iovec_depth > 1); + assert(num_full_stripes > 1); - if (last_subfile < first_subfile) { - /* - * This can also happen when the subfile with the final - * data segment has a smaller subfile index than the - * subfile with the first data segment and the current - * subfile index falls between the two. - */ - thin_uniform_section = - thin_uniform_section || ((last_subfile < k) && (k < first_subfile)); - } + iovec_depth--; + num_full_stripes--; + } + } - if (thin_uniform_section) { - assert(iovec_depth > 1); - assert(num_full_stripes > 1); + /* + * After accounting for the length of the initial + * and/or final data segments, add the combined + * size of the fully selected I/O stripes to the + * running bytes total + */ + subfile_bytes += num_full_stripes * stripe_size; + total_bytes += (size_t)subfile_bytes; - iovec_depth--; - num_full_stripes--; - } - } + /* + * Setup the pointers to the next set of I/O vectors + * in the output arrays + */ + _io_types_ptr = &io_types[iovec_idx + (size_t)i]; + _io_addrs_ptr = &io_addrs[iovec_idx + (size_t)i]; + _io_sizes_ptr = &io_sizes[iovec_idx + (size_t)i]; + _io_bufs_ptr = &io_bufs[iovec_idx + (size_t)i]; - /* - * After accounting for the length of the initial - * and/or final data segments, add the combined - * size of the fully selected I/O stripes to the - * running bytes total - */ - subfile_bytes += num_full_stripes * stripe_size; - total_bytes += subfile_bytes; - } + /* + * Fill in I/O vector with initial values. If more than 1 + * subfile is involved, these values will be adjusted below. + */ + for (size_t vec_idx = 0; vec_idx < iovec_count; vec_idx++) + *(_io_types_ptr + (vec_idx * iovec_len)) = type; + *_io_addrs_ptr = (haddr_t)(row_offset + offset_in_block); + *_io_sizes_ptr = (size_t)subfile_bytes; - _mem_buf_offset[0] = mem_offset; - _target_file_offset[0] = row_offset + offset_in_block; - _io_block_len[0] = subfile_bytes; + if (io_type == IO_TYPE_WRITE) + _io_bufs_ptr->cvp = (const char *)(io_buf.cvp) + mem_offset; + else + _io_bufs_ptr->vp = (char *)(io_buf.vp) + mem_offset; if (num_subfiles > 1) { - int64_t curr_file_offset = row_offset + offset_in_block; + int64_t cur_file_offset = row_offset + offset_in_block; + + assert(iovec_depth <= max_iovec_depth); - /* Fill the I/O vectors */ + /* Fill the I/O vectors for the current subfile */ if (is_first) { - if (is_last) { /* First + Last */ - if (iovec_fill_first_last(sf_context, iovec_depth, subfile_bytes, mem_offset, - curr_file_offset, start_length, final_length, _mem_buf_offset, - _target_file_offset, _io_block_len) < 0) + if (is_last) { + /* + * The current subfile being processed is both the first + * subfile touched by I/O and the last subfile touched by + * I/O. In this case, we may have to deal with partial + * stripe I/O in the first and last I/O segments. + */ + if (iovec_fill_first_last(sf_context, iovec_len, iovec_depth, subfile_bytes, mem_offset, + cur_file_offset, start_length, final_length, io_buf, io_type, + _io_addrs_ptr, _io_sizes_ptr, _io_bufs_ptr) < 0) H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't fill I/O vectors"); } - else { /* First ONLY */ - if (iovec_fill_first(sf_context, iovec_depth, subfile_bytes, mem_offset, curr_file_offset, - start_length, _mem_buf_offset, _target_file_offset, - _io_block_len) < 0) + else { + /* + * The current subfile being processed is the first + * subfile touched by I/O. In this case, we may have + * to deal with partial stripe I/O in the first I/O + * segment. + */ + if (iovec_fill_first(sf_context, iovec_len, iovec_depth, subfile_bytes, mem_offset, + cur_file_offset, start_length, io_buf, io_type, _io_addrs_ptr, + _io_sizes_ptr, _io_bufs_ptr) < 0) H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't fill I/O vectors"); } /* Move the memory pointer to the starting location @@ -2959,31 +3263,43 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne */ mem_offset += start_length; } - else if (is_last) { /* Last ONLY */ - if (iovec_fill_last(sf_context, iovec_depth, subfile_bytes, mem_offset, curr_file_offset, - final_length, _mem_buf_offset, _target_file_offset, _io_block_len) < 0) + else if (is_last) { + /* + * The current subfile being processed is the last subfile + * touched by I/O. In this case, we may have to deal with + * partial stripe I/O in the last I/O segment. + */ + if (iovec_fill_last(sf_context, iovec_len, iovec_depth, subfile_bytes, mem_offset, + cur_file_offset, final_length, io_buf, io_type, _io_addrs_ptr, + _io_sizes_ptr, _io_bufs_ptr) < 0) H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't fill I/O vectors"); mem_offset += stripe_size; } - else { /* Everything else (uniform) */ - if (iovec_fill_uniform(sf_context, iovec_depth, subfile_bytes, mem_offset, curr_file_offset, - _mem_buf_offset, _target_file_offset, _io_block_len) < 0) + else { + /* + * The current subfile being processed is neither the first + * nor the last subfile touched by I/O. In this case, no + * partial stripe I/O will need to be dealt with; all I/O + * segments will cover a full I/O stripe. + */ + if (iovec_fill_uniform(sf_context, iovec_len, iovec_depth, subfile_bytes, mem_offset, + cur_file_offset, io_buf, io_type, _io_addrs_ptr, _io_sizes_ptr, + _io_bufs_ptr) < 0) H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't fill I/O vectors"); mem_offset += stripe_size; } } - offset_in_block += _io_block_len[0]; + offset_in_block += (int64_t)*_io_sizes_ptr; - k++; - curr_stripe_idx++; + subfile_idx++; + cur_stripe_idx++; - if (k == num_subfiles) { - k = 0; - offset_in_block = 0; - curr_max_iovec_depth = ((final_stripe_idx - curr_stripe_idx) / num_subfiles) + 1; + if (subfile_idx == num_subfiles) { + subfile_idx = 0; + offset_in_block = 0; row_offset += block_size; } @@ -2991,13 +3307,12 @@ init_indep_io(subfiling_context_t *sf_context, int64_t file_offset, size_t io_ne assert(offset_in_block <= block_size); } - if (total_bytes != data_size) - H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, - "total bytes (%" PRId64 ") didn't match data size (%" PRId64 ")!", - total_bytes, data_size); + if (total_bytes != io_size) + H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "total bytes (%zu) didn't match data size (%zu)!", + total_bytes, io_size); done: - return ret_value; + H5_SUBFILING_FUNC_LEAVE; } /*------------------------------------------------------------------------- @@ -3020,9 +3335,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -iovec_fill_first(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t target_datasize, - int64_t start_mem_offset, int64_t start_file_offset, int64_t first_io_len, - int64_t *mem_offset_out, int64_t *target_file_offset_out, int64_t *io_block_len_out) +iovec_fill_first(subfiling_context_t *sf_context, size_t iovec_len, int64_t cur_iovec_depth, + int64_t target_datasize, int64_t start_mem_offset, int64_t start_file_offset, + int64_t first_io_len, H5_flexible_const_ptr_t buf, H5FD_subfiling_io_type_t io_type, + haddr_t *io_addrs_ptr, size_t *io_sizes_ptr, H5_flexible_const_ptr_t *io_bufs_ptr) { int64_t stripe_size; int64_t block_size; @@ -3030,10 +3346,10 @@ iovec_fill_first(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t t herr_t ret_value = SUCCEED; assert(sf_context); - assert(mem_offset_out); - assert(target_file_offset_out); - assert(io_block_len_out); - assert(iovec_depth > 0); + assert(cur_iovec_depth > 0); + assert(io_addrs_ptr); + assert(io_sizes_ptr); + assert(io_bufs_ptr); stripe_size = sf_context->sf_stripe_size; block_size = sf_context->sf_blocksize_per_stripe; @@ -3045,16 +3361,13 @@ iovec_fill_first(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t t __func__, start_mem_offset, start_file_offset, first_io_len); #endif - mem_offset_out[0] = start_mem_offset; - target_file_offset_out[0] = start_file_offset; - io_block_len_out[0] = first_io_len; + *io_addrs_ptr = (haddr_t)start_file_offset; + *io_sizes_ptr = (size_t)first_io_len; -#ifdef H5_SUBFILING_DEBUG - H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[0] = %" PRId64 ", file_offset[0] = %" PRId64 - ", io_block_len[0] = %" PRId64, - __func__, mem_offset_out[0], target_file_offset_out[0], io_block_len_out[0]); -#endif + if (io_type == IO_TYPE_WRITE) + io_bufs_ptr->cvp = (const char *)(buf.cvp) + start_mem_offset; + else + io_bufs_ptr->vp = (char *)(buf.vp) + start_mem_offset; if (first_io_len == target_datasize) H5_SUBFILING_GOTO_DONE(SUCCEED); @@ -3066,17 +3379,20 @@ iovec_fill_first(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t t total_bytes = first_io_len; - for (int64_t i = 1; i < iovec_depth; i++) { - mem_offset_out[i] = next_mem_offset; - target_file_offset_out[i] = next_file_offset; - io_block_len_out[i] = stripe_size; + for (size_t i = 1; i < (size_t)cur_iovec_depth; i++) { + *(io_addrs_ptr + (i * iovec_len)) = (haddr_t)next_file_offset; + *(io_sizes_ptr + (i * iovec_len)) = (size_t)stripe_size; + + if (io_type == IO_TYPE_WRITE) + (io_bufs_ptr + (i * iovec_len))->cvp = (const char *)(buf.cvp) + next_mem_offset; + else + (io_bufs_ptr + (i * iovec_len))->vp = (char *)(buf.vp) + next_mem_offset; #ifdef H5_SUBFILING_DEBUG H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[%" PRId64 "] = %" PRId64 ", file_offset[%" PRId64 "] = %" PRId64 - ", io_block_len[%" PRId64 "] = %" PRId64, - __func__, i, mem_offset_out[i], i, target_file_offset_out[i], i, - io_block_len_out[i]); + "%s: mem_offset[%zu] = %" PRId64 ", file_offset[%zu] = %" PRId64 + ", io_block_len[%zu] = %" PRId64, + __func__, i, next_mem_offset, i, next_file_offset, i, stripe_size); #endif next_mem_offset += block_size; @@ -3116,9 +3432,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -iovec_fill_last(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t target_datasize, - int64_t start_mem_offset, int64_t start_file_offset, int64_t last_io_len, - int64_t *mem_offset_out, int64_t *target_file_offset_out, int64_t *io_block_len_out) +iovec_fill_last(subfiling_context_t *sf_context, size_t iovec_len, int64_t cur_iovec_depth, + int64_t target_datasize, int64_t start_mem_offset, int64_t start_file_offset, + int64_t last_io_len, H5_flexible_const_ptr_t buf, H5FD_subfiling_io_type_t io_type, + haddr_t *io_addrs_ptr, size_t *io_sizes_ptr, H5_flexible_const_ptr_t *io_bufs_ptr) { int64_t stripe_size; int64_t block_size; @@ -3126,10 +3443,10 @@ iovec_fill_last(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t ta herr_t ret_value = SUCCEED; assert(sf_context); - assert(mem_offset_out); - assert(target_file_offset_out); - assert(io_block_len_out); - assert(iovec_depth > 0); + assert(cur_iovec_depth > 0); + assert(io_addrs_ptr); + assert(io_sizes_ptr); + assert(io_bufs_ptr); stripe_size = sf_context->sf_stripe_size; block_size = sf_context->sf_blocksize_per_stripe; @@ -3141,52 +3458,45 @@ iovec_fill_last(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t ta __func__, start_mem_offset, start_file_offset, last_io_len); #endif - mem_offset_out[0] = start_mem_offset; - target_file_offset_out[0] = start_file_offset; - io_block_len_out[0] = last_io_len; + *io_addrs_ptr = (haddr_t)start_file_offset; + *io_sizes_ptr = (size_t)last_io_len; - if (last_io_len == target_datasize) { -#ifdef H5_SUBFILING_DEBUG - H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[0] = %" PRId64 ", file_offset[0] = %" PRId64 - ", io_block_len[0] = %" PRId64, - __func__, mem_offset_out[0], target_file_offset_out[0], io_block_len_out[0]); -#endif + if (io_type == IO_TYPE_WRITE) + io_bufs_ptr->cvp = (const char *)(buf.cvp) + start_mem_offset; + else + io_bufs_ptr->vp = (char *)(buf.vp) + start_mem_offset; + if (last_io_len == target_datasize) H5_SUBFILING_GOTO_DONE(SUCCEED); - } - else { + + { int64_t next_mem_offset = start_mem_offset + block_size; int64_t next_file_offset = start_file_offset + block_size; - int64_t i; + size_t i; /* * If the last I/O size doesn't cover the target data * size, there is at least one full stripe preceding * the last I/O block */ - io_block_len_out[0] = stripe_size; - -#ifdef H5_SUBFILING_DEBUG - H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[0] = %" PRId64 ", file_offset[0] = %" PRId64 - ", io_block_len[0] = %" PRId64, - __func__, mem_offset_out[0], target_file_offset_out[0], io_block_len_out[0]); -#endif + *io_sizes_ptr = (size_t)stripe_size; total_bytes = stripe_size; - for (i = 1; i < iovec_depth - 1;) { - mem_offset_out[i] = next_mem_offset; - target_file_offset_out[i] = next_file_offset; - io_block_len_out[i] = stripe_size; + for (i = 1; i < (size_t)cur_iovec_depth - 1;) { + *(io_addrs_ptr + (i * iovec_len)) = (haddr_t)next_file_offset; + *(io_sizes_ptr + (i * iovec_len)) = (size_t)stripe_size; + + if (io_type == IO_TYPE_WRITE) + (io_bufs_ptr + (i * iovec_len))->cvp = (const char *)(buf.cvp) + next_mem_offset; + else + (io_bufs_ptr + (i * iovec_len))->vp = (char *)(buf.vp) + next_mem_offset; #ifdef H5_SUBFILING_DEBUG H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[%" PRId64 "] = %" PRId64 ", file_offset[%" PRId64 "] = %" PRId64 - ", io_block_len[%" PRId64 "] = %" PRId64, - __func__, i, mem_offset_out[i], i, target_file_offset_out[i], i, - io_block_len_out[i]); + "%s: mem_offset[%zu] = %" PRId64 ", file_offset[%zu] = %" PRId64 + ", io_block_len[%zu] = %" PRId64, + __func__, i, next_mem_offset, i, next_file_offset, i, stripe_size); #endif next_mem_offset += block_size; @@ -3196,16 +3506,19 @@ iovec_fill_last(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t ta i++; } - mem_offset_out[i] = next_mem_offset; - target_file_offset_out[i] = next_file_offset; - io_block_len_out[i] = last_io_len; + *(io_addrs_ptr + (i * iovec_len)) = (haddr_t)next_file_offset; + *(io_sizes_ptr + (i * iovec_len)) = (size_t)last_io_len; + + if (io_type == IO_TYPE_WRITE) + (io_bufs_ptr + (i * iovec_len))->cvp = (const char *)(buf.cvp) + next_mem_offset; + else + (io_bufs_ptr + (i * iovec_len))->vp = (char *)(buf.vp) + next_mem_offset; #ifdef H5_SUBFILING_DEBUG H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[%" PRId64 "] = %" PRId64 ", file_offset[%" PRId64 "] = %" PRId64 - ", io_block_len[%" PRId64 "] = %" PRId64, - __func__, i, mem_offset_out[i], i, target_file_offset_out[i], i, - io_block_len_out[i]); + "%s: mem_offset[%zu] = %" PRId64 ", file_offset[%zu] = %" PRId64 + ", io_block_len[%zu] = %" PRId64, + __func__, i, next_mem_offset, i, next_file_offset, i, last_io_len); #endif total_bytes += last_io_len; @@ -3244,10 +3557,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -iovec_fill_first_last(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t target_datasize, - int64_t start_mem_offset, int64_t start_file_offset, int64_t first_io_len, - int64_t last_io_len, int64_t *mem_offset_out, int64_t *target_file_offset_out, - int64_t *io_block_len_out) +iovec_fill_first_last(subfiling_context_t *sf_context, size_t iovec_len, int64_t cur_iovec_depth, + int64_t target_datasize, int64_t start_mem_offset, int64_t start_file_offset, + int64_t first_io_len, int64_t last_io_len, H5_flexible_const_ptr_t buf, + H5FD_subfiling_io_type_t io_type, haddr_t *io_addrs_ptr, size_t *io_sizes_ptr, + H5_flexible_const_ptr_t *io_bufs_ptr) { int64_t stripe_size; int64_t block_size; @@ -3255,10 +3569,10 @@ iovec_fill_first_last(subfiling_context_t *sf_context, int64_t iovec_depth, int6 herr_t ret_value = SUCCEED; assert(sf_context); - assert(mem_offset_out); - assert(target_file_offset_out); - assert(io_block_len_out); - assert(iovec_depth > 0); + assert(cur_iovec_depth > 0); + assert(io_addrs_ptr); + assert(io_sizes_ptr); + assert(io_bufs_ptr); stripe_size = sf_context->sf_stripe_size; block_size = sf_context->sf_blocksize_per_stripe; @@ -3270,16 +3584,13 @@ iovec_fill_first_last(subfiling_context_t *sf_context, int64_t iovec_depth, int6 __func__, start_mem_offset, start_file_offset, first_io_len, last_io_len); #endif - mem_offset_out[0] = start_mem_offset; - target_file_offset_out[0] = start_file_offset; - io_block_len_out[0] = first_io_len; + *io_addrs_ptr = (haddr_t)start_file_offset; + *io_sizes_ptr = (size_t)first_io_len; -#ifdef H5_SUBFILING_DEBUG - H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[0] = %" PRId64 ", file_offset[0] = %" PRId64 - ", io_block_len[0] = %" PRId64, - __func__, mem_offset_out[0], target_file_offset_out[0], io_block_len_out[0]); -#endif + if (io_type == IO_TYPE_WRITE) + io_bufs_ptr->cvp = (const char *)(buf.cvp) + start_mem_offset; + else + io_bufs_ptr->vp = (char *)(buf.vp) + start_mem_offset; if (first_io_len == target_datasize) H5_SUBFILING_GOTO_DONE(SUCCEED); @@ -3288,21 +3599,24 @@ iovec_fill_first_last(subfiling_context_t *sf_context, int64_t iovec_depth, int6 int64_t offset_in_stripe = start_file_offset % stripe_size; int64_t next_mem_offset = block_size - offset_in_stripe; int64_t next_file_offset = start_file_offset + (block_size - offset_in_stripe); - int64_t i; + size_t i; total_bytes = first_io_len; - for (i = 1; i < iovec_depth - 1;) { - mem_offset_out[i] = next_mem_offset; - target_file_offset_out[i] = next_file_offset; - io_block_len_out[i] = stripe_size; + for (i = 1; i < (size_t)cur_iovec_depth - 1;) { + *(io_addrs_ptr + (i * iovec_len)) = (haddr_t)next_file_offset; + *(io_sizes_ptr + (i * iovec_len)) = (size_t)stripe_size; + + if (io_type == IO_TYPE_WRITE) + (io_bufs_ptr + (i * iovec_len))->cvp = (const char *)(buf.cvp) + next_mem_offset; + else + (io_bufs_ptr + (i * iovec_len))->vp = (char *)(buf.vp) + next_mem_offset; #ifdef H5_SUBFILING_DEBUG H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[%" PRId64 "] = %" PRId64 ", file_offset[%" PRId64 "] = %" PRId64 - ", io_block_len[%" PRId64 "] = %" PRId64, - __func__, i, mem_offset_out[i], i, target_file_offset_out[i], i, - io_block_len_out[i]); + "%s: mem_offset[%zu] = %" PRId64 ", file_offset[%zu] = %" PRId64 + ", io_block_len[%zu] = %" PRId64, + __func__, i, next_mem_offset, i, next_file_offset, i, stripe_size); #endif next_mem_offset += block_size; @@ -3312,16 +3626,19 @@ iovec_fill_first_last(subfiling_context_t *sf_context, int64_t iovec_depth, int6 i++; } - mem_offset_out[i] = next_mem_offset; - target_file_offset_out[i] = next_file_offset; - io_block_len_out[i] = last_io_len; + *(io_addrs_ptr + (i * iovec_len)) = (haddr_t)next_file_offset; + *(io_sizes_ptr + (i * iovec_len)) = (size_t)last_io_len; + + if (io_type == IO_TYPE_WRITE) + (io_bufs_ptr + (i * iovec_len))->cvp = (const char *)(buf.cvp) + next_mem_offset; + else + (io_bufs_ptr + (i * iovec_len))->vp = (char *)(buf.vp) + next_mem_offset; #ifdef H5_SUBFILING_DEBUG H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[%" PRId64 "] = %" PRId64 ", file_offset[%" PRId64 "] = %" PRId64 - ", io_block_len[%" PRId64 "] = %" PRId64, - __func__, i, mem_offset_out[i], i, target_file_offset_out[i], i, - io_block_len_out[i]); + "%s: mem_offset[%zu] = %" PRId64 ", file_offset[%zu] = %" PRId64 + ", io_block_len[%zu] = %" PRId64, + __func__, i, next_mem_offset, i, next_file_offset, i, last_io_len); #endif total_bytes += last_io_len; @@ -3352,9 +3669,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -iovec_fill_uniform(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t target_datasize, - int64_t start_mem_offset, int64_t start_file_offset, int64_t *mem_offset_out, - int64_t *target_file_offset_out, int64_t *io_block_len_out) +iovec_fill_uniform(subfiling_context_t *sf_context, size_t iovec_len, int64_t cur_iovec_depth, + int64_t target_datasize, int64_t start_mem_offset, int64_t start_file_offset, + H5_flexible_const_ptr_t buf, H5FD_subfiling_io_type_t io_type, haddr_t *io_addrs_ptr, + size_t *io_sizes_ptr, H5_flexible_const_ptr_t *io_bufs_ptr) { int64_t stripe_size; int64_t block_size; @@ -3362,10 +3680,10 @@ iovec_fill_uniform(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t herr_t ret_value = SUCCEED; assert(sf_context); - assert(mem_offset_out); - assert(target_file_offset_out); - assert(io_block_len_out); - assert((iovec_depth > 0) || (target_datasize == 0)); + assert((cur_iovec_depth > 0) || (target_datasize == 0)); + assert(io_addrs_ptr); + assert(io_sizes_ptr); + assert(io_bufs_ptr); stripe_size = sf_context->sf_stripe_size; block_size = sf_context->sf_blocksize_per_stripe; @@ -3377,23 +3695,20 @@ iovec_fill_uniform(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t __func__, start_mem_offset, start_file_offset, stripe_size); #endif - mem_offset_out[0] = start_mem_offset; - target_file_offset_out[0] = start_file_offset; - io_block_len_out[0] = stripe_size; + *io_addrs_ptr = (haddr_t)start_file_offset; + *io_sizes_ptr = (size_t)stripe_size; -#ifdef H5_SUBFILING_DEBUG - H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[0] = %" PRId64 ", file_offset[0] = %" PRId64 - ", io_block_len[0] = %" PRId64, - __func__, mem_offset_out[0], target_file_offset_out[0], io_block_len_out[0]); -#endif + if (io_type == IO_TYPE_WRITE) + io_bufs_ptr->cvp = (const char *)(buf.cvp) + start_mem_offset; + else + io_bufs_ptr->vp = (char *)(buf.vp) + start_mem_offset; if (target_datasize == 0) { #ifdef H5_SUBFILING_DEBUG H5_subfiling_log(sf_context->sf_context_id, "%s: target_datasize = 0", __func__); #endif - io_block_len_out[0] = 0; + *io_sizes_ptr = (size_t)0; H5_SUBFILING_GOTO_DONE(SUCCEED); } @@ -3403,17 +3718,20 @@ iovec_fill_uniform(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t total_bytes = stripe_size; - for (int64_t i = 1; i < iovec_depth; i++) { - mem_offset_out[i] = next_mem_offset; - target_file_offset_out[i] = next_file_offset; - io_block_len_out[i] = stripe_size; + for (size_t i = 1; i < (size_t)cur_iovec_depth; i++) { + *(io_addrs_ptr + (i * iovec_len)) = (haddr_t)next_file_offset; + *(io_sizes_ptr + (i * iovec_len)) = (size_t)stripe_size; + + if (io_type == IO_TYPE_WRITE) + (io_bufs_ptr + (i * iovec_len))->cvp = (const char *)(buf.cvp) + next_mem_offset; + else + (io_bufs_ptr + (i * iovec_len))->vp = (char *)(buf.vp) + next_mem_offset; #ifdef H5_SUBFILING_DEBUG H5_subfiling_log(sf_context->sf_context_id, - "%s: mem_offset[%" PRId64 "] = %" PRId64 ", file_offset[%" PRId64 "] = %" PRId64 - ", io_block_len[%" PRId64 "] = %" PRId64, - __func__, i, mem_offset_out[i], i, target_file_offset_out[i], i, - io_block_len_out[i]); + "%s: mem_offset[%zu] = %" PRId64 ", file_offset[%zu] = %" PRId64 + ", io_block_len[%zu] = %" PRId64, + __func__, i, next_mem_offset, i, next_file_offset, i, stripe_size); #endif next_mem_offset += block_size; @@ -3430,3 +3748,38 @@ iovec_fill_uniform(subfiling_context_t *sf_context, int64_t iovec_depth, int64_t done: return ret_value; } + +#ifdef H5_SUBFILING_DEBUG +void +H5_subfiling_dump_iovecs(subfiling_context_t *sf_context, size_t ioreq_count, size_t iovec_len, + H5FD_subfiling_io_type_t io_type, H5FD_mem_t *io_types, haddr_t *io_addrs, + size_t *io_sizes, H5_flexible_const_ptr_t *io_bufs) +{ + assert(sf_context); + assert(io_types); + assert(io_addrs); + assert(io_sizes); + assert(io_bufs); + + H5_subfiling_log(sf_context->sf_context_id, + "%s: I/O REQUEST VECTORS (mem type, addr, size, buf):", __func__); + + for (size_t ioreq_idx = 0; ioreq_idx < ioreq_count; ioreq_idx++) { + H5_subfiling_log_nonewline(sf_context->sf_context_id, " -> I/O REQUEST %zu: ", ioreq_idx); + + H5_subfiling_log_nonewline(sf_context->sf_context_id, "["); + for (size_t i = 0; i < iovec_len; i++) { + if (i > 0) + H5_subfiling_log_nonewline(sf_context->sf_context_id, ", "); + + H5_subfiling_log_nonewline( + sf_context->sf_context_id, "(%d, %" PRIuHADDR ", %zu, %p)", + *(io_types + (ioreq_idx * iovec_len) + i), *(io_addrs + (ioreq_idx * iovec_len) + i), + *(io_sizes + (ioreq_idx * iovec_len) + i), + (io_type == IO_TYPE_WRITE) ? (const void *)(io_bufs + (ioreq_idx * iovec_len) + i)->cvp + : (void *)(io_bufs + (ioreq_idx * iovec_len) + i)->vp); + } + H5_subfiling_log_nonewline(sf_context->sf_context_id, "]\n"); + } +} +#endif diff --git a/src/H5FDsubfiling/H5subfiling_common.c b/src/H5FDsubfiling/H5subfiling_common.c index f9cc0dc..1127ae0 100644 --- a/src/H5FDsubfiling/H5subfiling_common.c +++ b/src/H5FDsubfiling/H5subfiling_common.c @@ -3156,4 +3156,37 @@ done: return; } + +void +H5_subfiling_log_nonewline(int64_t sf_context_id, const char *fmt, ...) +{ + subfiling_context_t *sf_context = NULL; + va_list log_args; + + va_start(log_args, fmt); + + /* Retrieve the subfiling object for the newly-created context ID */ + if (NULL == (sf_context = H5_get_subfiling_object(sf_context_id))) { + printf("%s: couldn't get subfiling object from context ID\n", __func__); + goto done; + } + + H5FD_ioc_begin_thread_exclusive(); + + if (sf_context->sf_logfile) { + vfprintf(sf_context->sf_logfile, fmt, log_args); + fflush(sf_context->sf_logfile); + } + else { + vprintf(fmt, log_args); + fflush(stdout); + } + + H5FD_ioc_end_thread_exclusive(); + +done: + va_end(log_args); + + return; +} #endif diff --git a/src/H5FDsubfiling/H5subfiling_common.h b/src/H5FDsubfiling/H5subfiling_common.h index 395183a..156902a 100644 --- a/src/H5FDsubfiling/H5subfiling_common.h +++ b/src/H5FDsubfiling/H5subfiling_common.h @@ -284,7 +284,10 @@ H5_DLL herr_t H5_subfiling_validate_config(const H5FD_subfiling_params_t *subf_c H5_DLL herr_t H5_subfiling_terminate(void); +#ifdef H5_SUBFILING_DEBUG H5_DLL void H5_subfiling_log(int64_t sf_context_id, const char *fmt, ...); +H5_DLL void H5_subfiling_log_nonewline(int64_t sf_context_id, const char *fmt, ...); +#endif #ifdef __cplusplus } diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index cc0bf2f..5485f6b 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -354,6 +354,9 @@ H5_DLL hid_t H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_ * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Fcreate} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Fcreate_async(const char *app_file, const char *app_func, unsigned app_line, @@ -450,14 +453,20 @@ H5_DLL hid_t H5Fcreate_async(const char *filename, unsigned flags, hid_t fcpl_id * * \version 1.10.0 The #H5F_ACC_SWMR_WRITE and #H5F_ACC_SWMR_READ flags were added. * + * \since 1.0.0 + * * \see H5Fclose() * + * */ H5_DLL hid_t H5Fopen(const char *filename, unsigned flags, hid_t fapl_id); /** * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Fopen} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Fopen_async(const char *app_file, const char *app_func, unsigned app_line, @@ -489,12 +498,17 @@ H5_DLL hid_t H5Fopen_async(const char *filename, unsigned flags, hid_t access_pl * active \p file_id. E.g., one cannot close a file with H5Fclose() on * \p file_id then use H5Freopen() on \p file_id to reopen it. * + * \since 1.0.0 + * */ H5_DLL hid_t H5Freopen(hid_t file_id); /** * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Freopen} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Freopen_async(const char *app_file, const char *app_func, unsigned app_line, hid_t file_id, @@ -533,12 +547,17 @@ H5_DLL hid_t H5Freopen_async(hid_t file_id, hid_t es_id); * that, the OS is responsible for ensuring that the data is * actually flushed to disk. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Fflush(hid_t object_id, H5F_scope_t scope); /** * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Fflush} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Fflush_async(const char *app_file, const char *app_func, unsigned app_line, hid_t object_id, @@ -586,6 +605,8 @@ H5_DLL herr_t H5Fflush_async(hid_t object_id, H5F_scope_t scope, hid_t es_id); * before calling H5Fclose. It is generally recommended to do so in all * cases. * + * \since 1.0.0 + * * \see H5Fopen() * */ @@ -594,6 +615,9 @@ H5_DLL herr_t H5Fclose(hid_t file_id); * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Fclose} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Fclose_async(const char *app_file, const char *app_func, unsigned app_line, hid_t file_id, @@ -656,6 +680,8 @@ H5_DLL herr_t H5Fdelete(const char *filename, hid_t fapl_id); * The creation property list identifier should be released with * H5Pclose(). * + * \since 1.0.0 + * */ H5_DLL hid_t H5Fget_create_plist(hid_t file_id); /** @@ -669,6 +695,8 @@ H5_DLL hid_t H5Fget_create_plist(hid_t file_id); * \details H5Fget_access_plist() returns the file access property list * identifier of the specified file. * + * \since 1.0.0 + * */ H5_DLL hid_t H5Fget_access_plist(hid_t file_id); /** @@ -749,6 +777,8 @@ H5_DLL herr_t H5Fget_fileno(hid_t file_id, unsigned long *fileno); * of objects to be counted. #H5F_OBJ_LOCAL restricts the * search to objects opened through current file identifier. * + * \since 1.6.0 + * */ H5_DLL ssize_t H5Fget_obj_count(hid_t file_id, unsigned types); /** @@ -842,6 +872,8 @@ H5_DLL herr_t H5Fget_vfd_handle(hid_t file_id, hid_t fapl, void **file_handle); * proper value to pass for \p plist is #H5P_DEFAULT, indicating the * default file mount property list. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Fmount(hid_t loc, const char *name, hid_t child, hid_t plist); /** @@ -864,6 +896,8 @@ H5_DLL herr_t H5Fmount(hid_t loc, const char *name, hid_t child, hid_t plist); * parent; if it was opened after the mount then it is the root group * of the child. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Funmount(hid_t loc, const char *name); /** @@ -1067,6 +1101,8 @@ H5_DLL herr_t H5Fset_mdc_config(hid_t file_id, const H5AC_cache_config_t *config * See the overview of the metadata cache in the special topics section of the user manual for * details on the metadata cache and its adaptive resize algorithms. * + * \since 1.8.0 + * */ H5_DLL herr_t H5Fget_mdc_hit_rate(hid_t file_id, double *hit_rate_ptr); /** @@ -1098,6 +1134,8 @@ H5_DLL herr_t H5Fget_mdc_hit_rate(hid_t file_id, double *hit_rate_ptr); * Current size can exceed maximum size under certain conditions. See the overview of the * metadata cache in the special topics section of the user manual for a discussion of this. * + * \since 1.8.0 + * */ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, size_t *max_size_ptr, size_t *min_clean_size_ptr, size_t *cur_size_ptr, int *cur_num_entries_ptr); @@ -1127,6 +1165,8 @@ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, size_t *max_size_ptr, size_t *min_c * you should not be using this API call. * \endparblock * + * \since 1.8.0 + * */ H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); /** diff --git a/src/H5Gpublic.h b/src/H5Gpublic.h index cc04680..4c0e2de 100644 --- a/src/H5Gpublic.h +++ b/src/H5Gpublic.h @@ -214,6 +214,9 @@ H5_DLL hid_t H5Gopen2(hid_t loc_id, const char *name, hid_t gapl_id); * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Gopen} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Gopen_async(const char *app_file, const char *app_func, unsigned app_line, hid_t loc_id, @@ -273,6 +276,9 @@ H5_DLL herr_t H5Gget_info(hid_t loc_id, H5G_info_t *ginfo); * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Gget_info} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Gget_info_async(const char *app_file, const char *app_func, unsigned app_line, hid_t loc_id, @@ -317,6 +323,9 @@ H5_DLL herr_t H5Gget_info_by_name(hid_t loc_id, const char *name, H5G_info_t *gi * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Gget_info_by_name} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Gget_info_by_name_async(const char *app_file, const char *app_func, unsigned app_line, @@ -377,6 +386,9 @@ H5_DLL herr_t H5Gget_info_by_idx(hid_t loc_id, const char *group_name, H5_index_ * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Gget_info_by_idx} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Gget_info_by_idx_async(const char *app_file, const char *app_func, unsigned app_line, @@ -468,6 +480,9 @@ H5_DLL herr_t H5Gclose(hid_t group_id); * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Gclose} + * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Gclose_async(const char *app_file, const char *app_func, unsigned app_line, hid_t group_id, @@ -683,6 +698,8 @@ H5_DLL hid_t H5Gopen1(hid_t loc_id, const char *name); * \version 1.8.0 Function deprecated in this release. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Glink(hid_t cur_loc_id, H5G_link_t type, const char *cur_name, const char *new_name); /** @@ -716,9 +733,11 @@ H5_DLL herr_t H5Glink(hid_t cur_loc_id, H5G_link_t type, const char *cur_name, c * current_name is \Code{./foo}, \p new_name is \Code{./x/y/bar}, and a * request is made for \Code{./x/y/bar}, then the actual object looked * up is \Code{./x/y/./foo}. - + * * \version 1.8.0 Function deprecated in this release. * + * \since 1.6.0 + * */ H5_DLL herr_t H5Glink2(hid_t cur_loc_id, const char *cur_name, H5G_link_t type, hid_t new_loc_id, const char *new_name); @@ -748,6 +767,8 @@ H5_DLL herr_t H5Glink2(hid_t cur_loc_id, const char *cur_name, H5G_link_t type, * * \version 1.8.0 Function deprecated in this release. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Gmove(hid_t src_loc_id, const char *src_name, const char *dst_name); /** @@ -779,6 +800,8 @@ H5_DLL herr_t H5Gmove(hid_t src_loc_id, const char *src_name, const char *dst_na * * \version 1.8.0 Function deprecated in this release. * + * \since 1.6.0 + * */ H5_DLL herr_t H5Gmove2(hid_t src_loc_id, const char *src_name, hid_t dst_loc_id, const char *dst_name); /** @@ -820,6 +843,8 @@ H5_DLL herr_t H5Gmove2(hid_t src_loc_id, const char *src_name, hid_t dst_loc_id, * * \version 1.8.0 Function deprecated in this release. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Gunlink(hid_t loc_id, const char *name); /** @@ -858,6 +883,8 @@ H5_DLL herr_t H5Gunlink(hid_t loc_id, const char *name); * * \version 1.8.0 Function deprecated in this release. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Gget_linkval(hid_t loc_id, const char *name, size_t size, char *buf /*out*/); /** @@ -898,6 +925,8 @@ H5_DLL herr_t H5Gget_linkval(hid_t loc_id, const char *name, size_t size, char * * * \version 1.8.0 Function deprecated in this release. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Gset_comment(hid_t loc_id, const char *name, const char *comment); /** @@ -943,6 +972,8 @@ H5_DLL herr_t H5Gset_comment(hid_t loc_id, const char *name, const char *comment * * \version 1.8.0 Function deprecated in this release. * + * \since 1.0.0 + * */ H5_DLL int H5Gget_comment(hid_t loc_id, const char *name, size_t bufsize, char *buf); /** @@ -1008,6 +1039,8 @@ H5_DLL int H5Gget_comment(hid_t loc_id, const char *name, size_t bufsize, char * * * \version 1.8.0 Function deprecated in this release. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Giterate(hid_t loc_id, const char *name, int *idx, H5G_iterate_t op, void *op_data); /** @@ -1029,6 +1062,8 @@ H5_DLL herr_t H5Giterate(hid_t loc_id, const char *name, int *idx, H5G_iterate_t * * \version 1.8.0 Function deprecated in this release. * + * \since 1.6.0 + * */ H5_DLL herr_t H5Gget_num_objs(hid_t loc_id, hsize_t *num_objs); /** @@ -1086,6 +1121,8 @@ H5_DLL herr_t H5Gget_num_objs(hid_t loc_id, hsize_t *num_objs); * \version 1.6.1 Two new fields were added to the \ref H5G_stat_t struct in * this release. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Gget_objinfo(hid_t loc_id, const char *name, hbool_t follow_link, H5G_stat_t *statbuf /*out*/); diff --git a/src/H5Idbg.c b/src/H5Idbg.c index 4c7d1ec..7910b1c 100644 --- a/src/H5Idbg.c +++ b/src/H5Idbg.c @@ -83,13 +83,13 @@ H5I__id_dump_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) fprintf(stderr, " id = %" PRIdHID "\n", info->id); fprintf(stderr, " count = %u\n", info->count); - fprintf(stderr, " obj = 0x%8p\n", info->object); + fprintf(stderr, " obj = 0x%8p\n", info->u.c_object); fprintf(stderr, " marked = %d\n", info->marked); /* Get the group location, so we get get the name */ switch (type) { case H5I_GROUP: { - const H5VL_object_t *vol_obj = (const H5VL_object_t *)info->object; + const H5VL_object_t *vol_obj = (const H5VL_object_t *)info->u.c_object; object = H5VL_object_data(vol_obj); if (H5_VOL_NATIVE == vol_obj->connector->cls->value) @@ -98,7 +98,7 @@ H5I__id_dump_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) } case H5I_DATASET: { - const H5VL_object_t *vol_obj = (const H5VL_object_t *)info->object; + const H5VL_object_t *vol_obj = (const H5VL_object_t *)info->u.c_object; object = H5VL_object_data(vol_obj); if (H5_VOL_NATIVE == vol_obj->connector->cls->value) @@ -107,13 +107,10 @@ H5I__id_dump_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) } case H5I_DATATYPE: { - const H5T_t *dt = (const H5T_t *)info->object; + H5T_t *dt = info->u.object; - H5_GCC_CLANG_DIAG_OFF("cast-qual") - object = (void *)H5T_get_actual_type((H5T_t *)dt); - H5_GCC_CLANG_DIAG_ON("cast-qual") - - path = H5T_nameof(object); + object = H5T_get_actual_type((H5T_t *)dt); + path = H5T_nameof(object); break; } diff --git a/src/H5Idevelop.h b/src/H5Idevelop.h index d0ff200..2eaa61a 100644 --- a/src/H5Idevelop.h +++ b/src/H5Idevelop.h @@ -128,6 +128,8 @@ extern "C" { * \note The H5Iregister_future() function is primarily targeted at VOL connector * authors and is _not_ designed for general-purpose application use. * + * \since 1.14.0 + * */ H5_DLL hid_t H5Iregister_future(H5I_type_t type, const void *object, H5I_future_realize_func_t realize_cb, H5I_future_discard_func_t discard_cb); diff --git a/src/H5Iint.c b/src/H5Iint.c index 7d8b4ac..1c53214 100644 --- a/src/H5Iint.c +++ b/src/H5Iint.c @@ -369,17 +369,16 @@ H5I__mark_node(void *_info, void H5_ATTR_UNUSED *key, void *_udata) */ if (udata->force || (info->count - (!udata->app_ref * info->app_count)) <= 1) { /* Check if this is an un-realized future object */ - H5_GCC_CLANG_DIAG_OFF("cast-qual") if (info->is_future) { /* Discard the future object */ - if ((info->discard_cb)((void *)info->object) < 0) { + if ((info->discard_cb)(info->u.object) < 0) { if (udata->force) { #ifdef H5I_DEBUG if (H5DEBUG(I)) { fprintf(H5DEBUG(I), "H5I: discard type=%d obj=%p " "failure ignored\n", - (int)udata->type_info->cls->type, info->object); + (int)udata->type_info->cls->type, info->u.c_object); } #endif /* H5I_DEBUG */ @@ -395,14 +394,14 @@ H5I__mark_node(void *_info, void H5_ATTR_UNUSED *key, void *_udata) else { /* Check for a 'free' function and call it, if it exists */ if (udata->type_info->cls->free_func && - (udata->type_info->cls->free_func)((void *)info->object, H5_REQUEST_NULL) < 0) { + (udata->type_info->cls->free_func)(info->u.object, H5_REQUEST_NULL) < 0) { if (udata->force) { #ifdef H5I_DEBUG if (H5DEBUG(I)) { fprintf(H5DEBUG(I), "H5I: free type=%d obj=%p " "failure ignored\n", - (int)udata->type_info->cls->type, info->object); + (int)udata->type_info->cls->type, info->u.c_object); } #endif /* H5I_DEBUG */ @@ -415,7 +414,6 @@ H5I__mark_node(void *_info, void H5_ATTR_UNUSED *key, void *_udata) mark = true; } } - H5_GCC_CLANG_DIAG_ON("cast-qual") /* Remove ID if requested */ if (mark) { @@ -524,7 +522,7 @@ H5I__register(H5I_type_t type, const void *object, bool app_ref, H5I_future_real info->id = new_id; info->count = 1; /* initial reference count */ info->app_count = !!app_ref; - info->object = object; + info->u.c_object = object; info->is_future = (NULL != realize_cb); info->realize_cb = realize_cb; info->discard_cb = discard_cb; @@ -633,7 +631,7 @@ H5I_register_using_existing_id(H5I_type_t type, void *object, bool app_ref, hid_ info->id = existing_id; info->count = 1; /* initial reference count*/ info->app_count = !!app_ref; - info->object = object; + info->u.object = object; /* This API call is only used by the native VOL connector, which is * not asynchronous. */ @@ -677,12 +675,10 @@ H5I_subst(hid_t id, const void *new_object) HGOTO_ERROR(H5E_ID, H5E_NOTFOUND, NULL, "can't get ID ref count"); /* Get the old object pointer to return */ - H5_GCC_CLANG_DIAG_OFF("cast-qual") - ret_value = (void *)info->object; - H5_GCC_CLANG_DIAG_ON("cast-qual") + ret_value = info->u.object; /* Set the new object pointer for the ID */ - info->object = new_object; + info->u.c_object = new_object; done: FUNC_LEAVE_NOAPI(ret_value) @@ -709,12 +705,9 @@ H5I_object(hid_t id) FUNC_ENTER_NOAPI_NOERR /* General lookup of the ID */ - if (NULL != (info = H5I__find_id(id))) { + if (NULL != (info = H5I__find_id(id))) /* Get the object pointer to return */ - H5_GCC_CLANG_DIAG_OFF("cast-qual") - ret_value = (void *)info->object; - H5_GCC_CLANG_DIAG_ON("cast-qual") - } + ret_value = info->u.object; FUNC_LEAVE_NOAPI(ret_value) } /* end H5I_object() */ @@ -742,12 +735,9 @@ H5I_object_verify(hid_t id, H5I_type_t type) assert(type >= 1 && (int)type < H5I_next_type_g); /* Verify that the type of the ID is correct & lookup the ID */ - if (type == H5I_TYPE(id) && NULL != (info = H5I__find_id(id))) { + if (type == H5I_TYPE(id) && NULL != (info = H5I__find_id(id))) /* Get the object pointer to return */ - H5_GCC_CLANG_DIAG_OFF("cast-qual") - ret_value = (void *)info->object; - H5_GCC_CLANG_DIAG_ON("cast-qual") - } + ret_value = info->u.object; FUNC_LEAVE_NOAPI(ret_value) } /* H5I_object_verify() */ @@ -898,9 +888,7 @@ H5I__remove_common(H5I_type_info_t *type_info, hid_t id) if (type_info->last_id_info == info) type_info->last_id_info = NULL; - H5_GCC_CLANG_DIAG_OFF("cast-qual") - ret_value = (void *)info->object; - H5_GCC_CLANG_DIAG_ON("cast-qual") + ret_value = info->u.object; if (!H5I_marking_s) info = H5FL_FREE(H5I_id_info_t, info); @@ -1000,8 +988,7 @@ H5I__dec_ref(hid_t id, void **request) /* Get the ID's type */ type_info = H5I_type_info_array_g[H5I_TYPE(id)]; - H5_GCC_CLANG_DIAG_OFF("cast-qual") - if (!type_info->cls->free_func || (type_info->cls->free_func)((void *)info->object, request) >= 0) { + if (!type_info->cls->free_func || (type_info->cls->free_func)(info->u.object, request) >= 0) { /* Remove the node from the type */ if (NULL == H5I__remove_common(type_info, id)) HGOTO_ERROR(H5E_ID, H5E_CANTDELETE, (-1), "can't remove ID node"); @@ -1009,7 +996,6 @@ H5I__dec_ref(hid_t id, void **request) } /* end if */ else ret_value = -1; - H5_GCC_CLANG_DIAG_ON("cast-qual") } /* end if */ else { --(info->count); @@ -1482,9 +1468,7 @@ H5I__iterate_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) /* The stored object pointer might be an H5VL_object_t, in which * case we'll need to get the wrapped object struct (H5F_t *, etc.). */ - H5_GCC_CLANG_DIAG_OFF("cast-qual") - object = H5I__unwrap((void *)info->object, type); - H5_GCC_CLANG_DIAG_ON("cast-qual") + object = H5I__unwrap(info->u.object, type); /* Invoke callback function */ cb_ret_val = (*udata->user_func)((void *)object, info->id, udata->user_udata); @@ -1607,14 +1591,13 @@ H5I__find_id(hid_t id) } /* Check if this is a future ID */ - H5_GCC_CLANG_DIAG_OFF("cast-qual") if (id_info && id_info->is_future) { hid_t actual_id = H5I_INVALID_HID; /* ID for actual object */ void *future_object; /* Pointer to the future object */ void *actual_object; /* Pointer to the actual object */ /* Invoke the realize callback, to get the actual object */ - if ((id_info->realize_cb)((void *)id_info->object, &actual_id) < 0) + if ((id_info->realize_cb)(id_info->u.object, &actual_id) < 0) HGOTO_DONE(NULL); /* Verify that we received a valid ID, of the same type */ @@ -1624,10 +1607,10 @@ H5I__find_id(hid_t id) HGOTO_DONE(NULL); /* Swap the actual object in for the future object */ - future_object = (void *)id_info->object; + future_object = id_info->u.object; actual_object = H5I__remove_common(type_info, actual_id); assert(actual_object); - id_info->object = actual_object; + id_info->u.object = actual_object; /* Discard the future object */ if ((id_info->discard_cb)(future_object) < 0) @@ -1639,7 +1622,6 @@ H5I__find_id(hid_t id) id_info->realize_cb = NULL; id_info->discard_cb = NULL; } - H5_GCC_CLANG_DIAG_ON("cast-qual") /* Set return value */ ret_value = id_info; @@ -1674,9 +1656,7 @@ H5I__find_id_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) assert(udata); /* Get a pointer to the VOL connector's data */ - H5_GCC_CLANG_DIAG_OFF("cast-qual") - object = H5I__unwrap((void *)info->object, type); - H5_GCC_CLANG_DIAG_ON("cast-qual") + object = H5I__unwrap(info->u.object, type); /* Check for a match */ if (object == udata->object) { diff --git a/src/H5Ipkg.h b/src/H5Ipkg.h index 6207962..c42c61d 100644 --- a/src/H5Ipkg.h +++ b/src/H5Ipkg.h @@ -56,10 +56,13 @@ /* ID information structure used */ typedef struct H5I_id_info_t { - hid_t id; /* ID for this info */ - unsigned count; /* Ref. count for this ID */ - unsigned app_count; /* Ref. count of application visible IDs */ - const void *object; /* Pointer associated with the ID */ + hid_t id; /* ID for this info */ + unsigned count; /* Ref. count for this ID */ + unsigned app_count; /* Ref. count of application visible IDs */ + union { + const void *c_object; /* Const pointer associated with the ID */ + void *object; /* Pointer associated with the ID */ + } u; /* Future ID info */ bool is_future; /* Whether this ID represents a future object */ diff --git a/src/H5Ipublic.h b/src/H5Ipublic.h index ce78ae3..3e429ec 100644 --- a/src/H5Ipublic.h +++ b/src/H5Ipublic.h @@ -125,6 +125,8 @@ extern "C" { * will be a reference to. This pointer will be stored by the library * and returned via a call to H5Iobject_verify(). * + * \since 1.8.0 + * */ H5_DLL hid_t H5Iregister(H5I_type_t type, const void *object); /** @@ -148,6 +150,8 @@ H5_DLL hid_t H5Iregister(H5I_type_t type, const void *object); * * \see H5Iregister() * + * \since 1.8.0 + * */ H5_DLL void *H5Iobject_verify(hid_t id, H5I_type_t type); /** @@ -178,6 +182,8 @@ H5_DLL void *H5Iobject_verify(hid_t id, H5I_type_t type); * The pointer returned by H5Iregister() must be deallocated by the user * to avoid memory leaks. * + * \since 1.8.0 + * */ H5_DLL void *H5Iremove_verify(hid_t id, H5I_type_t type); /** @@ -202,6 +208,8 @@ H5_DLL void *H5Iremove_verify(hid_t id, H5I_type_t type); * is valid identifier. Validity can be determined with a call to * H5Iis_valid(). * + * \since 1.0.0 + * */ H5_DLL H5I_type_t H5Iget_type(hid_t id); /** @@ -311,7 +319,7 @@ H5_DLL ssize_t H5Iget_name(hid_t id, char *name /*out*/, size_t size); * safely closed or decremented and the HDF5 object will be closed * when the reference count for that that object drops to zero. * - * \since 1.6.2 + * \since 1.6.3 * */ H5_DLL int H5Iinc_ref(hid_t id); @@ -357,7 +365,7 @@ H5_DLL int H5Iinc_ref(hid_t id); * safely closed or decremented and the HDF5 object will be closed * when the reference count for that object drops to zero. * - * \since 1.6.2 + * \since 1.6.3 * */ H5_DLL int H5Idec_ref(hid_t id); @@ -381,7 +389,7 @@ H5_DLL int H5Idec_ref(hid_t id); * The function H5Iis_valid() is used to determine whether a specific * object identifier is valid. * - * \since 1.6.2 + * \since 1.6.3 * */ H5_DLL int H5Iget_ref(hid_t id); @@ -415,6 +423,8 @@ H5_DLL int H5Iget_ref(hid_t id); * pointer which was passed in to the H5Iregister() function. The \p * free_func function should return 0 on success and -1 on failure. * + * \since 1.8.0 + * */ H5_DLL H5I_type_t H5Iregister_type(size_t hash_size, unsigned reserved, H5I_free_t free_func); /** @@ -439,6 +449,8 @@ H5_DLL H5I_type_t H5Iregister_type(size_t hash_size, unsigned reserved, H5I_free * identifiers will be entirely unchanged. If the force flag is true, * all identifiers of this type will be deleted. * + * \since 1.8.0 + * */ H5_DLL herr_t H5Iclear_type(H5I_type_t type, hbool_t force); /** @@ -462,6 +474,8 @@ H5_DLL herr_t H5Iclear_type(H5I_type_t type, hbool_t force); * reused when new types are registered, it is a good idea to set the * variable holding the value of the destroyed type to #H5I_UNINIT. * + * \since 1.8.0 + * */ H5_DLL herr_t H5Idestroy_type(H5I_type_t type); /** @@ -481,6 +495,8 @@ H5_DLL herr_t H5Idestroy_type(H5I_type_t type); * reference count is to be incremented. This identifier must have * been created by a call to H5Iregister_type(). * + * \since 1.8.0 + * */ H5_DLL int H5Iinc_type_ref(H5I_type_t type); /** @@ -501,6 +517,8 @@ H5_DLL int H5Iinc_type_ref(H5I_type_t type); * reference count is to be decremented. This identifier must have * been created by a call to H5Iregister_type(). * + * \since 1.8.0 + * */ H5_DLL int H5Idec_type_ref(H5I_type_t type); /** @@ -520,6 +538,8 @@ H5_DLL int H5Idec_type_ref(H5I_type_t type); * reference count is to be retrieved. This identifier must have been * created by a call to H5Iregister_type(). * + * \since 1.8.0 + * */ H5_DLL int H5Iget_type_ref(H5I_type_t type); /** @@ -561,6 +581,8 @@ H5_DLL int H5Iget_type_ref(H5I_type_t type); * The \p key parameter will be passed to the search function as a * parameter. It can be used to further define the search at run-time. * + * \since 1.8.0 + * */ H5_DLL void *H5Isearch(H5I_type_t type, H5I_search_func_t func, void *key); /** @@ -613,6 +635,8 @@ H5_DLL herr_t H5Iiterate(H5I_type_t type, H5I_iterate_func_t op, void *op_data); * exist, or it has been destroyed, \p num_members is returned with * the value 0. * + * \since 1.8.0 + * */ H5_DLL herr_t H5Inmembers(H5I_type_t type, hsize_t *num_members); /** diff --git a/src/H5Pprivate.h b/src/H5Pprivate.h index fa70e38..36c2457 100644 --- a/src/H5Pprivate.h +++ b/src/H5Pprivate.h @@ -175,7 +175,7 @@ H5_DLL herr_t H5P_insert(H5P_genplist_t *plist, const char *name, size_t size, v H5_DLL herr_t H5P_remove(H5P_genplist_t *plist, const char *name); H5_DLL htri_t H5P_exist_plist(const H5P_genplist_t *plist, const char *name); H5_DLL htri_t H5P_class_isa(const H5P_genclass_t *pclass1, const H5P_genclass_t *pclass2); -H5_DLL char *H5P_get_class_name(H5P_genclass_t *pclass); +H5_DLL char *H5P_get_class_name(H5P_genclass_t *pclass) H5_ATTR_MALLOC; /* Internal helper routines */ H5_DLL herr_t H5P_get_nprops_pclass(const H5P_genclass_t *pclass, size_t *nprops, bool recurse); diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 8410a82..1b5d2f4 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -3370,7 +3370,7 @@ H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, siz * file driver remains registered. * * - * \since 1.4.0 + * \since 1.2.0 * */ H5_DLL hid_t H5Pget_driver(hid_t plist_id); @@ -3410,6 +3410,8 @@ H5_DLL hid_t H5Pget_driver(hid_t plist_id); * described this function only in the virtual file driver * documentation. * + * \since 1.4.0 + * */ H5_DLL const void *H5Pget_driver_info(hid_t plist_id); /** @@ -3436,7 +3438,7 @@ H5_DLL const void *H5Pget_driver_info(hid_t plist_id); * string is simply returned. The caller can then allocate a buffer * of the appropriate size and call this routine again. * - * \version 1.14.0 Function publicized in this release. + * \since 1.14.0 * */ H5_DLL ssize_t H5Pget_driver_config_str(hid_t fapl_id, char *config_buf, size_t buf_size); @@ -4274,6 +4276,8 @@ H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size * described this function only in the virtual file driver * documentation. * + * \since 1.4.0 + * */ H5_DLL herr_t H5Pset_driver(hid_t plist_id, hid_t driver_id, const void *driver_info); /** @@ -4297,7 +4301,7 @@ H5_DLL herr_t H5Pset_driver(hid_t plist_id, hid_t driver_id, const void *driver_ * registered, an attempt will be made to load the driver as a * plugin. * - * \version 1.14.0 Function publicized in this release. + * \since 1.14.0 * */ H5_DLL herr_t H5Pset_driver_by_name(hid_t plist_id, const char *driver_name, const char *driver_config); @@ -4322,7 +4326,7 @@ H5_DLL herr_t H5Pset_driver_by_name(hid_t plist_id, const char *driver_name, con * registered, an attempt will be made to load the driver as a * plugin. * - * \version 1.14.0 Function publicized in this release. + * \since 1.14.0 * */ H5_DLL herr_t H5Pset_driver_by_value(hid_t plist_id, H5FD_class_value_t driver_value, @@ -4796,6 +4800,8 @@ H5_DLL herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool * * The default value for garbage collecting references is off. * + * \since 1.2.0 + * */ H5_DLL herr_t H5Pset_gc_references(hid_t fapl_id, unsigned gc_ref); /** @@ -5584,8 +5590,7 @@ H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective) * \details H5Pget_mpi_params() gets the MPI communicator and info stored in * the file access property list \p fapl_id. * - * \todo When was this introduced? - * + * \since 1.12.0 */ H5_DLL herr_t H5Pget_mpi_params(hid_t fapl_id, MPI_Comm *comm, MPI_Info *info); @@ -5602,7 +5607,7 @@ H5_DLL herr_t H5Pget_mpi_params(hid_t fapl_id, MPI_Comm *comm, MPI_Info *info); * \details H5Pset_mpi_params() sets the MPI communicator and info stored in * the file access property list \p fapl_id. * - * \todo When was this introduced? + * \since 1.12.0 * */ H5_DLL herr_t H5Pset_mpi_params(hid_t fapl_id, MPI_Comm comm, MPI_Info info); @@ -7146,7 +7151,7 @@ H5_DLL herr_t H5Pget_chunk_cache(hid_t dapl_id, size_t *rdcc_nslots /*out*/, siz * file location behavior and for notes on the use of the * HDF5_EXTFILE_PREFIX environment variable. * - * \since 1.10.0, 1.8.17 + * \since 1.8.17 * */ H5_DLL ssize_t H5Pget_efile_prefix(hid_t dapl_id, char *prefix /*out*/, size_t size); @@ -7491,7 +7496,7 @@ H5_DLL herr_t H5Pset_chunk_cache(hid_t dapl_id, size_t rdcc_nslots, size_t rdcc_ * \note On Windows, the prefix must be an ASCII string since the Windows * standard C library's I/O functions cannot handle UTF-8 file names. * - * \since 1.10.0, 1.8.17 + * \since 1.8.17 * */ H5_DLL herr_t H5Pset_efile_prefix(hid_t dapl_id, const char *prefix); @@ -7618,6 +7623,8 @@ H5_DLL herr_t H5Pset_virtual_view(hid_t dapl_id, H5D_vds_view_t view); * \p left, \p middle, and \p right, as set by the H5Pset_btree_ratios() * function. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Pget_btree_ratios(hid_t plist_id, double *left /*out*/, double *middle /*out*/, double *right /*out*/); @@ -7640,6 +7647,8 @@ H5_DLL herr_t H5Pget_btree_ratios(hid_t plist_id, double *left /*out*/, double * * \version 1.6.0 The return type changed from \p hsize_t to \p size_t. * \version 1.4.0 The return type changed to \p hsize_t. * + * \since 1.0.0 + * */ H5_DLL size_t H5Pget_buffer(hid_t plist_id, void **tconv /*out*/, void **bkg /*out*/); /** @@ -7830,6 +7839,8 @@ H5_DLL herr_t H5Pget_vlen_mem_manager(hid_t plist_id, H5MM_allocate_t *alloc_fun * * All ratios are real numbers between 0 and 1, inclusive. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Pset_btree_ratios(hid_t plist_id, double left, double middle, double right); @@ -7871,6 +7882,8 @@ H5_DLL herr_t H5Pset_btree_ratios(hid_t plist_id, double left, double middle, do * \version 1.6.0 The \p size parameter has changed from type hsize_t to \c size_t. * \version 1.4.0 The \p size parameter has changed to type hsize_t. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Pset_buffer(hid_t plist_id, size_t size, void *tconv, void *bkg); @@ -8582,7 +8595,7 @@ H5_DLL herr_t H5Pget_create_intermediate_group(hid_t plist_id, unsigned *crt_int * * \details H5Pset_create_intermediate_group() * - * \since + * \since 1.8.0 * */ H5_DLL herr_t H5Pset_create_intermediate_group(hid_t plist_id, unsigned crt_intmd); @@ -9922,6 +9935,8 @@ H5_DLL herr_t H5Pset_mcdt_search_cb(hid_t plist_id, H5O_mcdt_search_cb_t func, v * The #H5P_prp_cb2_t is as follows: * \snippet this H5P_prp_cb2_t_snip * + * \since 1.4.0 + * */ /* Function prototypes */ @@ -10036,6 +10051,8 @@ H5_DLL herr_t H5Pregister1(hid_t cls_id, const char *name, size_t size, void *de * The #H5P_prp_cb2_t is as follows: * \snippet this H5P_prp_cb2_t_snip * + * \since 1.4.0 + * */ H5_DLL herr_t H5Pinsert1(hid_t plist_id, const char *name, size_t size, void *value, H5P_prp_set_func_t prp_set, H5P_prp_get_func_t prp_get, @@ -10135,6 +10152,8 @@ H5_DLL herr_t H5Pencode1(hid_t plist_id, void *buf, size_t *nalloc); * deprecated in this release. * \version 1.6.4 \p filter parameter type changed to unsigned. * + * \since 1.0.0 + * */ H5_DLL H5Z_filter_t H5Pget_filter1(hid_t plist_id, unsigned filter, unsigned int *flags /*out*/, size_t *cd_nelmts /*out*/, unsigned cd_values[] /*out*/, size_t namelen, @@ -10189,7 +10208,9 @@ H5_DLL H5Z_filter_t H5Pget_filter1(hid_t plist_id, unsigned filter, unsigned int * lists. * \version 1.8.0 Function H5Pget_filter_by_id() renamed to * H5Pget_filter_by_id1() and deprecated in this release. - * \version 1.6.0 Function introduced in this release. + * + * \since 1.6.0 + * */ H5_DLL herr_t H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out*/, size_t *cd_nelmts /*out*/, unsigned cd_values[] /*out*/, size_t namelen, @@ -10218,6 +10239,8 @@ H5_DLL herr_t H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, unsigned int * \version 1.6.4 \p boot, \p freelist, \p stab, \p shhdr parameter types * changed to unsigned. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Pget_version(hid_t plist_id, unsigned *boot /*out*/, unsigned *freelist /*out*/, unsigned *stab /*out*/, unsigned *shhdr /*out*/); @@ -10239,6 +10262,8 @@ H5_DLL herr_t H5Pget_version(hid_t plist_id, unsigned *boot /*out*/, unsigned *f * * \details Maps to the function H5Pset_file_space_strategy(). * + * \since 1.10.0 + * */ H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); /** @@ -10257,6 +10282,7 @@ H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, * * \details Maps to the function H5Pget_file_space_strategy() * + * \since 1.10.0 * */ H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); diff --git a/src/H5Rpublic.h b/src/H5Rpublic.h index 3f63d59..a28262c 100644 --- a/src/H5Rpublic.h +++ b/src/H5Rpublic.h @@ -143,6 +143,8 @@ extern "C" { * H5Rdestroy() should be used to release the resource from the * reference. * + * \since 1.12.0 + * */ H5_DLL herr_t H5Rcreate_object(hid_t loc_id, const char *name, hid_t oapl_id, H5R_ref_t *ref_ptr); @@ -179,6 +181,8 @@ H5_DLL herr_t H5Rcreate_object(hid_t loc_id, const char *name, hid_t oapl_id, H5 * H5Rdestroy() should be used to release the resource from the * reference. * + * \since 1.12.0 + * */ H5_DLL herr_t H5Rcreate_region(hid_t loc_id, const char *name, hid_t space_id, hid_t oapl_id, H5R_ref_t *ref_ptr); @@ -216,6 +220,8 @@ H5_DLL herr_t H5Rcreate_region(hid_t loc_id, const char *name, hid_t space_id, h * H5Rdestroy() should be used to release the resource from the * reference. * + * \since 1.12.0 + * */ H5_DLL herr_t H5Rcreate_attr(hid_t loc_id, const char *name, const char *attr_name, hid_t oapl_id, H5R_ref_t *ref_ptr); @@ -237,6 +243,8 @@ H5_DLL herr_t H5Rcreate_attr(hid_t loc_id, const char *name, const char *attr_na * \ref H5R_ref_t is defined in H5Rpublic.h as: * \snippet this H5R_ref_t_snip * + * \since 1.12.0 + * */ H5_DLL herr_t H5Rdestroy(H5R_ref_t *ref_ptr); @@ -265,6 +273,8 @@ H5_DLL herr_t H5Rdestroy(H5R_ref_t *ref_ptr); * \ref H5R_ref_t is defined in H5Rpublic.h as: * \snippet this H5R_ref_t_snip * + * \since 1.12.0 + * */ H5_DLL H5R_type_t H5Rget_type(const H5R_ref_t *ref_ptr); @@ -287,6 +297,8 @@ H5_DLL H5R_type_t H5Rget_type(const H5R_ref_t *ref_ptr); * \ref H5R_ref_t is defined in H5Rpublic.h as: * \snippet this H5R_ref_t_snip * + * \since 1.12.0 + * */ H5_DLL htri_t H5Requal(const H5R_ref_t *ref1_ptr, const H5R_ref_t *ref2_ptr); @@ -305,6 +317,8 @@ H5_DLL htri_t H5Requal(const H5R_ref_t *ref1_ptr, const H5R_ref_t *ref2_ptr); * \p src_ref_ptr points to the reference to copy, and \p dst_ref_ptr is the * pointer to the destination reference. * + * \since 1.12.0 + * */ H5_DLL herr_t H5Rcopy(const H5R_ref_t *src_ref_ptr, H5R_ref_t *dst_ref_ptr); @@ -339,6 +353,8 @@ H5_DLL herr_t H5Rcopy(const H5R_ref_t *src_ref_ptr, H5R_ref_t *dst_ref_ptr); * the appropriate close function, such as H5Oclose() or H5Dclose() * for datasets. * + * \since 1.12.0 + * */ H5_DLL hid_t H5Ropen_object(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t oapl_id); @@ -346,6 +362,9 @@ H5_DLL hid_t H5Ropen_object(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t oapl_id); * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Ropen_object} + * + * \since 1.14.0 + * */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Ropen_object_async(const char *app_file, const char *app_func, unsigned app_line, @@ -385,6 +404,8 @@ H5_DLL hid_t H5Ropen_object_async(unsigned app_line, H5R_ref_t *ref_ptr, hid_t r * Use H5Sclose() to release the dataspace identifier returned by * this function when the identifier is no longer needed. * + * \since 1.12.0 + * */ H5_DLL hid_t H5Ropen_region(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t oapl_id); @@ -392,6 +413,9 @@ H5_DLL hid_t H5Ropen_region(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t oapl_id); * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Ropen_region} + * + * \since 1.14.0 + * */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Ropen_region_async(const char *app_file, const char *app_func, unsigned app_line, @@ -427,6 +451,8 @@ H5_DLL hid_t H5Ropen_region_async(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t oapl_ * The attribute opened with this function should be closed with * H5Aclose() when it is no longer needed. * + * \since 1.12.0 + * */ H5_DLL hid_t H5Ropen_attr(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t aapl_id); @@ -434,6 +460,8 @@ H5_DLL hid_t H5Ropen_attr(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t aapl_id); * -------------------------------------------------------------------------- * \ingroup ASYNC * \async_variant_of{H5Ropen_attr} + * + * \since 1.14.0 */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Ropen_attr_async(const char *app_file, const char *app_func, unsigned app_line, @@ -469,6 +497,8 @@ H5_DLL hid_t H5Ropen_attr_async(H5R_ref_t *ref_ptr, hid_t rapl_id, hid_t aapl_id * referenced object type are as followed (defined in H5Opublic.h): * \snippet H5Opublic.h H5O_type_t_snip * + * \since 1.12.0 + * */ H5_DLL herr_t H5Rget_obj_type3(H5R_ref_t *ref_ptr, hid_t rapl_id, H5O_type_t *obj_type); @@ -498,6 +528,8 @@ H5_DLL herr_t H5Rget_obj_type3(H5R_ref_t *ref_ptr, hid_t rapl_id, H5O_type_t *ob * passed in for size in the second call to H5Rget_file_name(), * which will retrieve the actual name. * + * \since 1.12.0 + * */ H5_DLL ssize_t H5Rget_file_name(const H5R_ref_t *ref_ptr, char *name, size_t size); @@ -544,6 +576,8 @@ H5_DLL ssize_t H5Rget_file_name(const H5R_ref_t *ref_ptr, char *name, size_t siz * if there are multiple links pointing to it. This function may * return any one of these paths. * + * \since 1.12.0 + * */ H5_DLL ssize_t H5Rget_obj_name(H5R_ref_t *ref_ptr, hid_t rapl_id, char *name, size_t size); @@ -571,6 +605,8 @@ H5_DLL ssize_t H5Rget_obj_name(H5R_ref_t *ref_ptr, hid_t rapl_id, char *name, si * be passed in for size in the second call to H5Rget_attr_name(), * which will retrieve the actual name. * + * \since 1.12.0 + * */ H5_DLL ssize_t H5Rget_attr_name(const H5R_ref_t *ref_ptr, char *name, size_t size); @@ -663,7 +699,8 @@ H5_DLL ssize_t H5Rget_attr_name(const H5R_ref_t *ref_ptr, char *name, size_t siz * * \version 1.8.0 Function H5Rget_obj_type() renamed to H5Rget_obj_type1() and * deprecated in this release. - * \since 1.6.0 + * + * \since 1.2.0 * */ H5_DLL H5G_obj_t H5Rget_obj_type1(hid_t id, H5R_type_t ref_type, const void *ref); @@ -705,7 +742,7 @@ H5_DLL H5G_obj_t H5Rget_obj_type1(hid_t id, H5R_type_t ref_type, const void *ref * * \version 1.10.0 Function H5Rdereference() renamed to H5Rdereference1() and * deprecated in this release. - * \since 1.8.0 + * \since 1.0.0 * */ H5_DLL hid_t H5Rdereference1(hid_t obj_id, H5R_type_t ref_type, const void *ref); @@ -742,7 +779,7 @@ H5_DLL hid_t H5Rdereference1(hid_t obj_id, H5R_type_t ref_type, const void *ref) * dataset region references and should be set to -1 if the reference * is an object reference, #H5R_OBJECT. * - * \since 1.8.0 + * \since 1.0.0 */ H5_DLL herr_t H5Rcreate(void *ref, hid_t loc_id, const char *name, H5R_type_t ref_type, hid_t space_id); @@ -873,6 +910,8 @@ H5_DLL hid_t H5Rdereference2(hid_t obj_id, hid_t oapl_id, H5R_type_t ref_type, c * Use H5Sclose() to release the dataspace identifier returned by this * function when the identifier is no longer needed. * + * \since 1.0.0 + * */ H5_DLL hid_t H5Rget_region(hid_t dataset, H5R_type_t ref_type, const void *ref); diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index 7a79d44..b9e24be 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -870,7 +870,7 @@ H5_DLL herr_t H5T__enum_insert(const H5T_t *dt, const char *name, const void *va H5_DLL herr_t H5T__get_member_value(const H5T_t *dt, unsigned membno, void *value); /* Field functions (for both compound & enumerated types) */ -H5_DLL char *H5T__get_member_name(H5T_t const *dt, unsigned membno); +H5_DLL char *H5T__get_member_name(H5T_t const *dt, unsigned membno) H5_ATTR_MALLOC; H5_DLL herr_t H5T__sort_value(const H5T_t *dt, int *map); H5_DLL herr_t H5T__sort_name(const H5T_t *dt, int *map); diff --git a/src/H5Tpublic.h b/src/H5Tpublic.h index 1a16301..a117075 100644 --- a/src/H5Tpublic.h +++ b/src/H5Tpublic.h @@ -1044,7 +1044,7 @@ H5_DLLVAR hid_t H5T_NATIVE_UINT_FAST64_g; * * \see H5Tclose() * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL hid_t H5Tcreate(H5T_class_t type, size_t size); @@ -1069,6 +1069,8 @@ H5_DLL hid_t H5Tcreate(H5T_class_t type, size_t size); * The returned datatype identifier should be released with H5Tclose() * to prevent resource leaks. * + * \since 1.0.0 + * */ H5_DLL hid_t H5Tcopy(hid_t type_id); /** @@ -1084,6 +1086,8 @@ H5_DLL hid_t H5Tcopy(hid_t type_id); * through this datatype identifier is illegal. Failure to release * a datatype with this call will result in resource leaks. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Tclose(hid_t type_id); /** @@ -1091,6 +1095,8 @@ H5_DLL herr_t H5Tclose(hid_t type_id); * * \brief Asynchronous version of H5Tclose(). * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Tclose_async(const char *app_file, const char *app_func, unsigned app_line, hid_t type_id, @@ -1111,7 +1117,7 @@ H5_DLL herr_t H5Tclose_async(hid_t type_id, hid_t es_id); * \details H5Tequal() determines whether two datatype identifiers refer to * the same datatype. * - * \since 1.6 or earlier + * \since 1.0.0 * */ H5_DLL htri_t H5Tequal(hid_t type1_id, hid_t type2_id); @@ -1130,6 +1136,8 @@ H5_DLL htri_t H5Tequal(hid_t type1_id, hid_t type2_id); * inadvertently change or delete a predefined type. Once a datatype * is locked it can never be unlocked. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Tlock(hid_t type_id); /** @@ -1185,6 +1193,8 @@ H5_DLL herr_t H5Tcommit2(hid_t loc_id, const char *name, hid_t type_id, hid_t lc * * \brief Asynchronous version of H5Tcommit2(). * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL herr_t H5Tcommit_async(const char *app_file, const char *app_func, unsigned app_line, hid_t loc_id, @@ -1223,6 +1233,8 @@ H5_DLL hid_t H5Topen2(hid_t loc_id, const char *name, hid_t tapl_id); * * \brief Asynchronous version of H5Topen2(). * + * \since 1.12.0 + * */ #ifndef H5_DOXYGEN H5_DLL hid_t H5Topen_async(const char *app_file, const char *app_func, unsigned app_line, hid_t loc_id, @@ -1322,7 +1334,7 @@ H5_DLL hid_t H5Tget_create_plist(hid_t type_id); * * \version 1.8.0 Fortran API was added * - * \since 1.6 or earlier + * \since 1.0.0 * */ H5_DLL htri_t H5Tcommitted(hid_t type_id); @@ -1380,6 +1392,8 @@ H5_DLL herr_t H5Tencode(hid_t obj_id, void *buf, size_t *nalloc); * with H5Tclose() when the identifier is no longer needed so that * resource leaks will not develop. * + * \since 1.2.0 + * */ H5_DLL hid_t H5Tdecode(const void *buf); /** @@ -1403,7 +1417,7 @@ H5_DLL hid_t H5Tdecode(const void *buf); * * \return \herr_t * - * \since 1.10.0 C function introduced with this release. + * \since 1.10.0 * * \see H5Dflush() * H5Drefresh() @@ -1439,7 +1453,7 @@ H5_DLL herr_t H5Tflush(hid_t type_id); * datatype. The reopened datatype is automatically re-registered * with the same identifier. * - * \since 1.2.0 + * \since 1.10.0 * */ H5_DLL herr_t H5Trefresh(hid_t type_id); @@ -1469,7 +1483,7 @@ H5_DLL herr_t H5Trefresh(hid_t type_id); * datatypes; a compound datatype can have a member which is a * compound datatype. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tinsert(hid_t parent_id, const char *name, size_t offset, hid_t member_id); @@ -1485,7 +1499,7 @@ H5_DLL herr_t H5Tinsert(hid_t parent_id, const char *name, size_t offset, hid_t * \details H5Tpack() recursively removes padding from within a compound * datatype to make it more efficient (space-wise) to store that data. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tpack(hid_t type_id); @@ -1631,6 +1645,8 @@ H5_DLL herr_t H5Tenum_valueof(hid_t type, const char *name, void *value /*out*/) * the array being of the string or character base type.\n * To create a variable-length string datatype, see \ref_vlen_strings. * + * \since 1.2.0 + * */ H5_DLL hid_t H5Tvlen_create(hid_t base_id); @@ -1719,6 +1735,8 @@ H5_DLL int H5Tget_array_dims2(hid_t type_id, hsize_t dims[]); * maximum size of an opaque datatype tag, was added in * H5Tpublic.h. * + * \since 1.2.0 + * */ H5_DLL herr_t H5Tset_tag(hid_t type, const char *tag); /** @@ -1737,6 +1755,8 @@ H5_DLL herr_t H5Tset_tag(hid_t type, const char *tag); * \attention The tag is returned via a pointer to an allocated string, which * the caller must free. * + * \since 1.2.0 + * */ H5_DLL char *H5Tget_tag(hid_t type); @@ -1758,6 +1778,8 @@ H5_DLL char *H5Tget_tag(hid_t type); * with H5Tclose() when the identifier is no longer needed so that * resource leaks will not develop. * + * \since 1.2.0 + * */ H5_DLL hid_t H5Tget_super(hid_t type); /** @@ -1781,6 +1803,8 @@ H5_DLL hid_t H5Tget_super(hid_t type); * be readable and modifiable only on the originating computing * platform; it will not be portable to other platforms. * + * \since 1.0.0 + * */ H5_DLL H5T_class_t H5Tget_class(hid_t type_id); /** @@ -1835,7 +1859,7 @@ H5_DLL htri_t H5Tdetect_class(hid_t type_id, H5T_class_t cls); * * \see H5Tset_size() * - * \since 1.2.0 + * \since 1.0.0 */ H5_DLL size_t H5Tget_size(hid_t type_id); /** @@ -1860,7 +1884,7 @@ H5_DLL size_t H5Tget_size(hid_t type_id); * but all other members have byte order #H5T_ORDER_LE, H5Tget_order() * will return #H5T_ORDER_LE for the compound datatype. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL H5T_order_t H5Tget_order(hid_t type_id); @@ -1880,7 +1904,7 @@ H5_DLL H5T_order_t H5Tget_order(hid_t type_id); * unless padding is present, is 8 times larger than the value * returned by H5Tget_size(). * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL size_t H5Tget_precision(hid_t type_id); @@ -1909,7 +1933,7 @@ H5_DLL size_t H5Tget_precision(hid_t type_id); * 3: [0x22] [ pad] [ pad] [0x11] * \endcode * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL int H5Tget_offset(hid_t type_id); @@ -1928,7 +1952,7 @@ H5_DLL int H5Tget_offset(hid_t type_id); * most-significant bit padding. Valid padding types are: * \snippet this H5T_pad_t_snip * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tget_pad(hid_t type_id, H5T_pad_t *lsb /*out*/, H5T_pad_t *msb /*out*/); @@ -1945,7 +1969,7 @@ H5_DLL herr_t H5Tget_pad(hid_t type_id, H5T_pad_t *lsb /*out*/, H5T_pad_t *msb / * Valid types are: * \snippet this H5T_sign_t_snip * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL H5T_sign_t H5Tget_sign(hid_t type_id); @@ -1969,7 +1993,7 @@ H5_DLL H5T_sign_t H5Tget_sign(hid_t type_id); * datatype. Bits are numbered with the least significant bit number * zero. Any (or even all) of the arguments can be null pointers. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tget_fields(hid_t type_id, size_t *spos /*out*/, size_t *epos /*out*/, size_t *esize /*out*/, @@ -1985,7 +2009,7 @@ H5_DLL herr_t H5Tget_fields(hid_t type_id, size_t *spos /*out*/, size_t *epos /* * * \details H5Tget_ebias() retrieves the exponent bias of a floating-point type. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL size_t H5Tget_ebias(hid_t type_id); @@ -2004,7 +2028,7 @@ H5_DLL size_t H5Tget_ebias(hid_t type_id); * floating-point datatype. Valid normalization types are: * \snippet this H5T_norm_t_snip * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL H5T_norm_t H5Tget_norm(hid_t type_id); @@ -2023,7 +2047,7 @@ H5_DLL H5T_norm_t H5Tget_norm(hid_t type_id); * bits in floating-point datatypes. Valid padding types are: * \snippet this H5T_pad_t_snip * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL H5T_pad_t H5Tget_inpad(hid_t type_id); @@ -2044,7 +2068,7 @@ H5_DLL H5T_pad_t H5Tget_inpad(hid_t type_id); * values returned are: * \str_pad_type * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL H5T_str_t H5Tget_strpad(hid_t type_id); @@ -2061,7 +2085,7 @@ H5_DLL H5T_str_t H5Tget_strpad(hid_t type_id); * \details H5Tget_nmembers() retrieves the number of fields in a compound * datatype or the number of members of an enumeration datatype. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL int H5Tget_nmembers(hid_t type_id); @@ -2088,7 +2112,7 @@ H5_DLL int H5Tget_nmembers(hid_t type_id); * the field. The caller must subsequently free the buffer with * H5free_memory(). * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL char *H5Tget_member_name(hid_t type_id, unsigned membno); @@ -2110,7 +2134,7 @@ H5_DLL char *H5Tget_member_name(hid_t type_id, unsigned membno); * Fields are stored in no particular order with index values of 0 * through N-1, where N is the value returned by H5Tget_nmembers() . * - * \since 1.2.0 + * \since 1.4.0 * */ H5_DLL int H5Tget_member_index(hid_t type_id, const char *name); @@ -2134,7 +2158,7 @@ H5_DLL int H5Tget_member_index(hid_t type_id, const char *name); * * \version 1.6.4 \p member_no parameter type changed to unsigned. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL size_t H5Tget_member_offset(hid_t type_id, unsigned membno); @@ -2155,7 +2179,7 @@ H5_DLL size_t H5Tget_member_offset(hid_t type_id, unsigned membno); * Valid class identifiers, as defined in H5Tpublic.h, are: * \snippet this H5T_class_t_snip * - * \since 1.2.0 + * \since 1.4.0 * */ H5_DLL H5T_class_t H5Tget_member_class(hid_t type_id, unsigned membno); @@ -2176,7 +2200,7 @@ H5_DLL H5T_class_t H5Tget_member_class(hid_t type_id, unsigned membno); * * \version 1.6.4 \p membno parameter type changed to unsigned. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL hid_t H5Tget_member_type(hid_t type_id, unsigned membno); @@ -2203,7 +2227,7 @@ H5_DLL hid_t H5Tget_member_type(hid_t type_id, unsigned membno); * of that base type. If the size is unknown, you can determine it * with H5Tget_size(). * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tget_member_value(hid_t type_id, unsigned membno, void *value /*out*/); @@ -2221,7 +2245,7 @@ H5_DLL herr_t H5Tget_member_value(hid_t type_id, unsigned membno, void *value /* * Valid character set types are: * \csets * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL H5T_cset_t H5Tget_cset(hid_t type_id); @@ -2378,7 +2402,7 @@ H5_DLL hid_t H5Tget_native_type(hid_t type_id, H5T_direction_t direction); * * \see H5Tget_size() * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_size(hid_t type_id, size_t size); @@ -2417,7 +2441,7 @@ H5_DLL herr_t H5Tset_size(hid_t type_id, size_t size); * have the same byte order. * \li Opaque datatypes: Byte order can be set but has no effect. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_order(hid_t type_id, H5T_order_t order); @@ -2447,7 +2471,7 @@ H5_DLL herr_t H5Tset_order(hid_t type_id, H5T_order_t order); * locations and sizes of the sign, mantissa, and exponent fields * first. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_precision(hid_t type_id, size_t prec); @@ -2481,7 +2505,7 @@ H5_DLL herr_t H5Tset_precision(hid_t type_id, size_t prec); * * The offset of an #H5T_STRING cannot be set to anything but zero. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_offset(hid_t type_id, size_t offset); @@ -2517,7 +2541,7 @@ H5_DLL herr_t H5Tset_pad(hid_t type_id, H5T_pad_t lsb, H5T_pad_t msb); * \details H5Tset_sign() sets the sign property for an integer type: * \sign_prop * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_sign(hid_t type_id, H5T_sign_t sign); @@ -2544,7 +2568,7 @@ H5_DLL herr_t H5Tset_sign(hid_t type_id, H5T_sign_t sign); * Fields are not allowed to extend beyond the number of bits of * precision, nor are they allowed to overlap with one another. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_fields(hid_t type_id, size_t spos, size_t epos, size_t esize, size_t mpos, size_t msize); @@ -2560,7 +2584,7 @@ H5_DLL herr_t H5Tset_fields(hid_t type_id, size_t spos, size_t epos, size_t esiz * * \details H5Tset_ebias() sets the exponent bias of a floating-point type. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_ebias(hid_t type_id, size_t ebias); @@ -2578,7 +2602,7 @@ H5_DLL herr_t H5Tset_ebias(hid_t type_id, size_t ebias); * datatype. Valid normalization types are: * \snippet this H5T_norm_t_snip * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_norm(hid_t type_id, H5T_norm_t norm); @@ -2599,7 +2623,7 @@ H5_DLL herr_t H5Tset_norm(hid_t type_id, H5T_norm_t norm); * padding types are: * \snippet this H5T_pad_t_snip * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_inpad(hid_t type_id, H5T_pad_t pad); @@ -2632,7 +2656,7 @@ H5_DLL herr_t H5Tset_inpad(hid_t type_id, H5T_pad_t pad); * string datatype while H5Pset_char_encoding() sets the character * set used for an HDF5 link or attribute name. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_cset(hid_t type_id, H5T_cset_t cset); @@ -2664,7 +2688,7 @@ H5_DLL herr_t H5Tset_cset(hid_t type_id, H5T_cset_t cset); * When converting from a shorter string to a longer string, the * longer string is padded on the end by appending nulls or spaces. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL herr_t H5Tset_strpad(hid_t type_id, H5T_str_t strpad); @@ -2708,6 +2732,8 @@ H5_DLL herr_t H5Tset_strpad(hid_t type_id, H5T_str_t strpad); * \version 1.6.3 \p nelmts parameter type changed to size_t. * \version 1.4.0 \p nelmts parameter type changed to hsize_t. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Tconvert(hid_t src_id, hid_t dst_id, size_t nelmts, void *buf, void *background, hid_t plist_id); @@ -2829,7 +2855,7 @@ H5_DLL herr_t H5Tcommit1(hid_t loc_id, const char *name, hid_t type_id); * \version 1.8.0 Function H5Topen() renamed to H5Topen1() and deprecated in * this release. * - * \since 1.2.0 + * \since 1.0.0 * */ H5_DLL hid_t H5Topen1(hid_t loc_id, const char *name); @@ -2864,6 +2890,7 @@ H5_DLL hid_t H5Topen1(hid_t loc_id, const char *name); * * \version 1.8.0 Function H5Tarray_create() renamed to H5Tarray_create1() * and deprecated in this release. + * * \since 1.4.0 * */ @@ -2892,7 +2919,8 @@ H5_DLL hid_t H5Tarray_create1(hid_t base_id, int ndims, const hsize_t dim[/* ndi * * \version 1.8.0 Function H5Tarray_create() renamed to H5Tarray_create1() * and deprecated in this release. - * \since 1.2.0 + * + * \since 1.4.0 * */ H5_DLL int H5Tget_array_dims1(hid_t type_id, hsize_t dims[], int perm[]); diff --git a/src/H5Zdevelop.h b/src/H5Zdevelop.h index 736c283..fb3b71b 100644 --- a/src/H5Zdevelop.h +++ b/src/H5Zdevelop.h @@ -145,6 +145,9 @@ typedef herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space * The return value from the filter is the number of bytes in the * output buffer. If an error occurs then the function should return * zero and leave all pointer arguments unchanged. + * + * \since 1.0.0 + * */ //! typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], @@ -379,6 +382,8 @@ extern "C" { * a new #H5Z_class_t struct and new set local and can apply * callback functions. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Zregister(const void *cls); /** diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index d906e3c..44d91c0 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -304,7 +304,7 @@ H5_DLL htri_t H5Zfilter_avail(H5Z_filter_t id); * such as H5Pset_szip(), that might require a particular filter * configuration. * - * \since 1.6.3 + * \since 1.6.0 */ H5_DLL herr_t H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_config_flags); diff --git a/src/H5private.h b/src/H5private.h index 3aaa0d5..5663c00 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -497,8 +497,16 @@ #define H5_DIAG_DO_PRAGMA(x) _Pragma(#x) #define H5_DIAG_PRAGMA(x) H5_DIAG_DO_PRAGMA(GCC diagnostic x) +/* Allow suppression of compiler diagnostics unless H5_SHOW_ALL_WARNINGS is + * defined (enabled with '--enable-show-all-warnings' configure option). + */ +#ifndef H5_SHOW_ALL_WARNINGS #define H5_DIAG_OFF(x) H5_DIAG_PRAGMA(push) H5_DIAG_PRAGMA(ignored H5_DIAG_JOINSTR("-W", x)) #define H5_DIAG_ON(x) H5_DIAG_PRAGMA(pop) +#else +#define H5_DIAG_OFF(x) +#define H5_DIAG_ON(x) +#endif /* Macros for enabling/disabling particular GCC-only warnings. * These pragmas are only implemented usefully in gcc 4.6+ diff --git a/src/H5public.h b/src/H5public.h index a40ca00..9eb0d93 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -429,6 +429,9 @@ extern "C" { * H5open() before an application issues any other function calls to * the HDF5 library, as there are no damaging side effects in calling * it more than once. + * + * \since 1.0.0 + * */ H5_DLL herr_t H5open(void); /** @@ -468,6 +471,9 @@ H5_DLL herr_t H5atclose(H5_atclose_func_t func, void *ctx); * generally called when the application calls exit(), but may be * called earlier in the event of an emergency shutdown or out of a * desire to free all resources used by the HDF5 library. + * + * \since 1.0.0 + * */ H5_DLL herr_t H5close(void); /** @@ -487,6 +493,9 @@ H5_DLL herr_t H5close(void); * before any other HDF5 function calls, and must be called each * time the library is loaded/linked into the application (the first * time and after it's been unloaded). + * + * \since 1.0.0 + * */ H5_DLL herr_t H5dont_atexit(void); /** @@ -507,6 +516,9 @@ H5_DLL herr_t H5dont_atexit(void); * * \note The library automatically garbage collects all the free lists when the * application ends. + * + * \since 1.4.0 + * */ H5_DLL herr_t H5garbage_collect(void); /** @@ -559,7 +571,7 @@ H5_DLL herr_t H5garbage_collect(void); * \version 1.8.3 Function changed in this release to set factory free list * memory limits. * - * \since 1.6.0 + * \since 1.4.0 */ H5_DLL herr_t H5set_free_list_limits(int reg_global_lim, int reg_list_lim, int arr_global_lim, int arr_list_lim, int blk_global_lim, int blk_list_lim); @@ -595,6 +607,8 @@ H5_DLL herr_t H5get_free_list_sizes(size_t *reg_size, size_t *arr_size, size_t * * of the version of the HDF5 library which is linked to the * application. * + * \since 1.0.0 + * */ H5_DLL herr_t H5get_libversion(unsigned *majnum, unsigned *minnum, unsigned *relnum); /** @@ -643,6 +657,8 @@ H5_DLL herr_t H5get_libversion(unsigned *majnum, unsigned *minnum, unsigned *rel * informational warning is printed but the application is allowed to * run. * + * \since 1.0.0 + * */ H5_DLL herr_t H5check_version(unsigned majnum, unsigned minnum, unsigned relnum); /** @@ -675,6 +691,9 @@ H5_DLL herr_t H5is_library_terminating(hbool_t *is_terminating); * data structures with a mutex. In certain circumstances, it may be * useful to determine, at run-time, whether the linked HDF5 library * was built with the thread-safety feature enabled. + * + * \since 1.10.0 + * */ H5_DLL herr_t H5is_library_threadsafe(hbool_t *is_ts); /** diff --git a/test/API/README.md b/test/API/README.md index aec6eaa..9ee1963 100644 --- a/test/API/README.md +++ b/test/API/README.md @@ -42,7 +42,7 @@ Currently unsupported These API tests currently only support usage with the native HDF5 VOL connector and HDF5 VOL connectors that can be loaded dynamically as a plugin. For information on how to build a VOL -connector in this manner, refer to section 2.3 of the [HDF5 VOL Connector Author Guide](https://portal.hdfgroup.org/display/HDF5/HDF5+VOL+Connector+Authors+Guide?preview=/53610813/59903039/vol_connector_author_guide.pdf). +connector in this manner, refer to section 2.3 of the [HDF5 VOL Connector Author Guide](https://docs.hdfgroup.org/hdf5/v1_14/_v_o_l__connector.html). TODO: section on building VOL connectors alongside HDF5 for use with tests @@ -84,4 +84,4 @@ following will appear in the test output: ### Help and Support -For help with building or using the HDF5 API tests, please contact the [HDF Help Desk](https://portal.hdfgroup.org/display/support/The+HDF+Help+Desk). +For help with building or using the HDF5 API tests, please contact the [HDF Help Desk](https://help.hdfgroup.org/). diff --git a/test/event_set.c b/test/event_set.c index 52aa6ba..9e659aa 100644 --- a/test/event_set.c +++ b/test/event_set.c @@ -22,8 +22,8 @@ static const char *FILENAME[] = {"event_set_1", NULL}; static hid_t connector_ids_g[EVENT_SET_NUM_CONNECTOR_IDS]; -herr_t fake_wait_request_wait(void *req, uint64_t timeout, H5VL_request_status_t *status); -herr_t fake_wait_request_free(void *req); +herr_t fake_wait_request_wait(void *req, uint64_t timeout, H5VL_request_status_t *status); +H5_ATTR_CONST herr_t fake_wait_request_free(void *req); /* A VOL class struct that describes a VOL class with no * functionality, other than a wait that returns success. diff --git a/testpar/t_subfiling_vfd.c b/testpar/t_subfiling_vfd.c index 45cb363..4f109cb 100644 --- a/testpar/t_subfiling_vfd.c +++ b/testpar/t_subfiling_vfd.c @@ -40,7 +40,7 @@ #define PATH_MAX 4096 #endif -#define DEFAULT_DEFLATE_LEVEL 9 +#define DEFAULT_DEFLATE_LEVEL 4 #define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0]) @@ -99,6 +99,7 @@ static void test_create_and_close(void); static void test_ioc_only_fail(void); static void test_config_file(void); static void test_stripe_sizes(void); +static void test_iovec_translation(void); static void test_selection_strategies(void); static void test_read_different_stripe_size(void); static void test_subfiling_precreate_rank_0(void); @@ -111,6 +112,7 @@ static test_func tests[] = { test_ioc_only_fail, test_config_file, test_stripe_sizes, + test_iovec_translation, test_selection_strategies, test_read_different_stripe_size, test_subfiling_precreate_rank_0, @@ -888,6 +890,697 @@ test_stripe_sizes(void) #undef SUBF_NITER /* + * Test the I/O vector translation code by writing with some + * different specific I/O patterns + */ +#define SUBF_FILENAME "test_subfiling_iovec_translation.h5" +static void +test_iovec_translation(void) +{ + H5FD_subfiling_params_t cfg; + const void *c_write_buf; + h5_stat_t file_info; + int64_t stripe_size; + haddr_t write_addr; + size_t nbytes; + size_t buf_size; + herr_t status; + hid_t file_id; + H5FD_t *file_ptr = NULL; + FILE *subfile_ptr = NULL; + void *write_buf = NULL; + void *read_buf = NULL; + char *tmp_filename = NULL; + hid_t dxpl_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + bool skip = false; + int num_subfiles; + int num_digits; + + curr_nerrors = nerrors; + + if (MAINPROCESS) + TESTING_2("I/O vector translation"); + + /* + * Don't run this test if subfiling configuration + * environment variables have been set since we + * want to use fixed configurations for testing. + */ + if (getenv(H5FD_SUBFILING_STRIPE_SIZE) || getenv(H5FD_SUBFILING_IOC_PER_NODE)) + skip = true; + + /* I/O only needs to be done from a single rank */ + if (MAINPROCESS && !skip) { + + /* Use a fixed configuration for these tests */ + stripe_size = 1048576; + num_subfiles = 4; + num_digits = (int)(log10(num_subfiles) + 1); + + /* Allocate enough buffer space for up to 2 "subfile blocks" of I/O */ + buf_size = (size_t)(2 * stripe_size * num_subfiles); + write_buf = malloc(buf_size); + VRFY(write_buf, "malloc succeeded"); + read_buf = malloc(buf_size); + VRFY(read_buf, "malloc succeeded"); + + c_write_buf = write_buf; + + tmp_filename = malloc(PATH_MAX); + VRFY(tmp_filename, "malloc succeeded"); + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + /* Set selection I/O mode on DXPL */ + VRFY((H5Pset_selection_io(dxpl_id, H5D_SELECTION_IO_MODE_ON) >= 0), "H5Pset_selection_io succeeded"); + + cfg.ioc_selection = SELECT_IOC_ONE_PER_NODE; + cfg.stripe_size = stripe_size; + cfg.stripe_count = 4; + + fapl_id = create_subfiling_ioc_fapl(MPI_COMM_SELF, MPI_INFO_NULL, true, &cfg, + H5FD_IOC_DEFAULT_THREAD_POOL_SIZE); + VRFY((fapl_id >= 0), "FAPL creation succeeded"); + + /* Set independent I/O on DXPL */ + VRFY((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_INDEPENDENT) >= 0), "H5Pset_dxpl_mpio succeeded"); + + /* + * Test the case where the index value of the last subfile + * touched by I/O is greater than or equal to the index + * value of the first subfile touched by I/O, and this results + * in "thin" I/O segments directed to the subfiles with index + * values greater than the index values of the first and + * last subfiles. This might appear as the following I/O + * pattern: + * + * SUBFILE 0 SUBFILE 1 SUBFILE 2 SUBFILE 3 + * _______________________________________________ + * | XXXXX | XXXXX | XXXXX | XXXXX | ROW 0 + * | XXXXX | XXXXX | | | ROW 1 + * | | | | | ROW 2 + * | | | | | ROW ... + * | | | | | + * | | | | | + * | | | | | + * |___________|___________|___________|___________| + */ + + /* Create/truncate the file */ + file_id = H5Fcreate(SUBF_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0), "H5Fcreate succeeded"); + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + + /* Retrieve file info to get the file inode for later use */ + VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded"); + + /* Re-open file through H5FDopen for direct writes */ + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + nbytes = (size_t)(6 * stripe_size); + memset(write_buf, 255, nbytes); + memset(read_buf, 0, buf_size); + + write_addr = 0; + + /* Set EOA for following write call */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Write according to the above pattern */ + status = H5FDwrite(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, c_write_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + /* Close and re-open the file */ + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + /* + * Set EOA for following read call (since we wrote over any + * superblock information in the file) + */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Read the written bytes and verify */ + status = H5FDread(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, read_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + VRFY((0 == memcmp(write_buf, read_buf, nbytes)), "memcmp succeeded"); + + /* Verify the size of each subfile */ + for (int i = 0; i < num_subfiles; i++) { + h5_stat_size_t subfile_size; + h5_stat_t subfile_info; + + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, i + 1, num_subfiles); + + /* Ensure file exists */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr, "fopen on subfile succeeded"); + VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded"); + + /* Check file size */ + VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded"); + subfile_size = (h5_stat_size_t)subfile_info.st_size; + + if (i <= 1) { + /* + * Subfiles with index values <= 1 should have full + * I/O segments (2 * stripe size) written to them. + */ + VRFY((subfile_size == 2 * cfg.stripe_size), "File size verification succeeded"); + } + else { + /* + * Subfiles with index values > 1 should have "thin" + * I/O segments (1 * stripe size) written to them. + */ + VRFY((subfile_size == cfg.stripe_size), "File size verification succeeded"); + } + } + + /* Verify that there aren't too many subfiles */ + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, num_subfiles + 1, num_subfiles); + + /* Ensure file doesn't exist */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr == NULL, "fopen on subfile correctly failed"); + + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + + /* + * Test the case where the index value of the last subfile + * touched by I/O is greater than or equal to the index + * value of the first subfile touched by I/O, and this results + * in "thin" I/O segments directed to the subfiles with index + * values less than the index values of the first and + * last subfiles. This might appear as the following I/O + * pattern: + * + * SUBFILE 0 SUBFILE 1 SUBFILE 2 SUBFILE 3 + * _______________________________________________ + * | | XXXXX | XXXXX | XXXXX | ROW 0 + * | XXXXX | XXXXX | XXXXX | XXXXX | ROW 1 + * | | | | | ROW 2 + * | | | | | ROW ... + * | | | | | + * | | | | | + * | | | | | + * |___________|___________|___________|___________| + */ + + /* Create/truncate the file */ + file_id = H5Fcreate(SUBF_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0), "H5Fcreate succeeded"); + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + + /* Retrieve file info to get the file inode for later use */ + VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded"); + + /* Re-open file through H5FDopen for direct writes */ + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + nbytes = (size_t)(7 * stripe_size); + memset(write_buf, 255, nbytes); + memset(read_buf, 0, buf_size); + + write_addr = (haddr_t)stripe_size; + + /* Set EOA for following write call */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Write according to the above pattern */ + status = H5FDwrite(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, c_write_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + /* Close and re-open the file */ + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + /* + * Set EOA for following read call (since we wrote over any + * superblock information in the file) + */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Read the written bytes and verify */ + status = H5FDread(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, read_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + VRFY((0 == memcmp(write_buf, read_buf, nbytes)), "memcmp succeeded"); + + /* Verify the size of each subfile */ + for (int i = 0; i < num_subfiles; i++) { + h5_stat_size_t subfile_size; + h5_stat_t subfile_info; + + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, i + 1, num_subfiles); + + /* Ensure file exists */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr, "fopen on subfile succeeded"); + VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded"); + + /* Check file size */ + VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded"); + subfile_size = (h5_stat_size_t)subfile_info.st_size; + + /* + * Every subfile should be (2 * stripe size) bytes due to + * space allocated in the file for subfile index 0 + */ + VRFY((subfile_size == 2 * cfg.stripe_size), "File size verification succeeded"); + } + + /* Verify that there aren't too many subfiles */ + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, num_subfiles + 1, num_subfiles); + + /* Ensure file doesn't exist */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr == NULL, "fopen on subfile correctly failed"); + + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + + /* + * Test the case where the index value of the last subfile + * touched by I/O is less than the index value of the first + * subfile touched by I/O, and this results in "thin" I/O + * segments directed to the subfiles with index values that + * fall between the values of the first and last subfiles. + * This might appear as the following I/O pattern: + * + * SUBFILE 0 SUBFILE 1 SUBFILE 2 SUBFILE 3 + * _______________________________________________ + * | | | XXXXX | XXXXX | ROW 0 + * | XXXXX | XXXXX | XXXXX | XXXXX | ROW 1 + * | XXXXX | | | | ROW 2 + * | | | | | ROW ... + * | | | | | + * | | | | | + * | | | | | + * |___________|___________|___________|___________| + */ + + /* Create/truncate the file */ + file_id = H5Fcreate(SUBF_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0), "H5Fcreate succeeded"); + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + + /* Retrieve file info to get the file inode for later use */ + VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded"); + + /* Re-open file through H5FDopen for direct writes */ + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + nbytes = (size_t)(7 * stripe_size); + memset(write_buf, 255, nbytes); + memset(read_buf, 0, buf_size); + + write_addr = (haddr_t)(2 * stripe_size); + + /* Set EOA for following write call */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Write according to the above pattern */ + status = H5FDwrite(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, c_write_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + /* Close and re-open the file */ + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + /* + * Set EOA for following read call (since we wrote over any + * superblock information in the file) + */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Read the written bytes and verify */ + status = H5FDread(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, read_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + VRFY((0 == memcmp(write_buf, read_buf, nbytes)), "memcmp succeeded"); + + /* Verify the size of each subfile */ + for (int i = 0; i < num_subfiles; i++) { + h5_stat_size_t subfile_size; + h5_stat_t subfile_info; + + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, i + 1, num_subfiles); + + /* Ensure file exists */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr, "fopen on subfile succeeded"); + VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded"); + + /* Check file size */ + VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded"); + subfile_size = (h5_stat_size_t)subfile_info.st_size; + + /* + * Subfile index 0 should be (3 * stripe size) bytes due to + * space allocated in the file, while others should be + * (2 * stripe size) bytes. + */ + if (i == 0) { + VRFY((subfile_size == 3 * cfg.stripe_size), "File size verification succeeded"); + } + else { + VRFY((subfile_size == 2 * cfg.stripe_size), "File size verification succeeded"); + } + } + + /* Verify that there aren't too many subfiles */ + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, num_subfiles + 1, num_subfiles); + + /* Ensure file doesn't exist */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr == NULL, "fopen on subfile correctly failed"); + + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + + /* + * Test the case where I/O is 2 stripe sizes in total, but + * is offset from a stripe boundary by a single byte, causing + * the I/O to cross 3 subfiles. + */ + + /* Create/truncate the file */ + file_id = H5Fcreate(SUBF_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0), "H5Fcreate succeeded"); + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + + /* Retrieve file info to get the file inode for later use */ + VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded"); + + /* Re-open file through H5FDopen for direct writes */ + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + nbytes = (size_t)(2 * stripe_size); + memset(write_buf, 255, nbytes); + memset(read_buf, 0, buf_size); + + write_addr = (haddr_t)1; + + /* Set EOA for following write call */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Write according to the above pattern */ + status = H5FDwrite(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, c_write_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + /* Close and re-open the file */ + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + /* + * Set EOA for following read call (since we wrote over any + * superblock information in the file) + */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Read the written bytes and verify */ + status = H5FDread(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, read_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + VRFY((0 == memcmp(write_buf, read_buf, nbytes)), "memcmp succeeded"); + + /* Verify the size of each subfile */ + for (int i = 0; i < num_subfiles; i++) { + h5_stat_size_t subfile_size; + h5_stat_t subfile_info; + + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, i + 1, num_subfiles); + + /* Ensure file exists */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr, "fopen on subfile succeeded"); + VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded"); + + /* Check file size */ + VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded"); + subfile_size = (h5_stat_size_t)subfile_info.st_size; + + /* + * Subfiles indexed 0 and 1 should both be (1 * stripe size) + * bytes (Subfile index 0 was written to with an offset of 1 + * byte, but that space will still be allocated in the file). + * Subfile index 2 should have a single byte written to it and + * Subfile index 3 should have nothing written to it. + */ + if (i == 2) { + VRFY((subfile_size == 1), "File size verification succeeded"); + } + else if (i == 3) { + VRFY((subfile_size == 0), "File size verification succeeded"); + } + else { + VRFY((subfile_size == cfg.stripe_size), "File size verification succeeded"); + } + } + + /* Verify that there aren't too many subfiles */ + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, num_subfiles + 1, num_subfiles); + + /* Ensure file doesn't exist */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr == NULL, "fopen on subfile correctly failed"); + + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + + /* + * Test the case where I/O is 2 stripe sizes in total, but + * is offset from a stripe boundary by (stripe size - 1) bytes, + * causing the I/O to start at the last byte of a subfile and + * cross 3 subfiles. + */ + + /* Create/truncate the file */ + file_id = H5Fcreate(SUBF_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0), "H5Fcreate succeeded"); + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + + /* Retrieve file info to get the file inode for later use */ + VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded"); + + /* Re-open file through H5FDopen for direct writes */ + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + nbytes = (size_t)(2 * stripe_size); + memset(write_buf, 255, nbytes); + memset(read_buf, 0, buf_size); + + write_addr = (haddr_t)(stripe_size - 1); + + /* Set EOA for following write call */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Write according to the above pattern */ + status = H5FDwrite(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, c_write_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + /* Close and re-open the file */ + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + /* + * Set EOA for following read call (since we wrote over any + * superblock information in the file) + */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Read the written bytes and verify */ + status = H5FDread(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, read_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + VRFY((0 == memcmp(write_buf, read_buf, nbytes)), "memcmp succeeded"); + + /* Verify the size of each subfile */ + for (int i = 0; i < num_subfiles; i++) { + h5_stat_size_t subfile_size; + h5_stat_t subfile_info; + + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, i + 1, num_subfiles); + + /* Ensure file exists */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr, "fopen on subfile succeeded"); + VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded"); + + /* Check file size */ + VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded"); + subfile_size = (h5_stat_size_t)subfile_info.st_size; + + /* + * Subfiles indexed 0 and 1 should both be (1 * stripe size) + * bytes (Subfile index 0 was written to with an offset of + * stripe size - 1 bytes, but that space will still be allocated + * in the file). Subfile index 2 should be (1 * stripe size) - 1 + * bytes. Subfile index 3 should have nothing written to it. + */ + if (i == 2) { + VRFY((subfile_size == cfg.stripe_size - 1), "File size verification succeeded"); + } + else if (i == 3) { + VRFY((subfile_size == 0), "File size verification succeeded"); + } + else { + VRFY((subfile_size == cfg.stripe_size), "File size verification succeeded"); + } + } + + /* Verify that there aren't too many subfiles */ + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, num_subfiles + 1, num_subfiles); + + /* Ensure file doesn't exist */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr == NULL, "fopen on subfile correctly failed"); + + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + + /* + * Test the case where I/O is 2 stripe sizes + 1 byte in total + * and starts aligned to a stripe boundary, causing the I/O + * to cross 3 subfiles. + */ + + /* Create/truncate the file */ + file_id = H5Fcreate(SUBF_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0), "H5Fcreate succeeded"); + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + + /* Retrieve file info to get the file inode for later use */ + VRFY((HDstat(SUBF_FILENAME, &file_info) >= 0), "HDstat succeeded"); + + /* Re-open file through H5FDopen for direct writes */ + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + nbytes = (size_t)((2 * stripe_size) + 1); + memset(write_buf, 255, nbytes); + memset(read_buf, 0, buf_size); + + write_addr = (haddr_t)0; + + /* Set EOA for following write call */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Write according to the above pattern */ + status = H5FDwrite(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, c_write_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + /* Close and re-open the file */ + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + file_ptr = H5FDopen(SUBF_FILENAME, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + VRFY(file_ptr, "H5FDopen succeeded"); + + /* + * Set EOA for following read call (since we wrote over any + * superblock information in the file) + */ + VRFY((H5FDset_eoa(file_ptr, H5FD_MEM_DEFAULT, write_addr + nbytes) >= 0), "H5FDset_eoa succeeded"); + + /* Read the written bytes and verify */ + status = H5FDread(file_ptr, H5FD_MEM_DRAW, dxpl_id, write_addr, nbytes, read_buf); + VRFY((status >= 0), "H5FDwrite succeeded"); + + VRFY((0 == memcmp(write_buf, read_buf, nbytes)), "memcmp succeeded"); + + /* Verify the size of each subfile */ + for (int i = 0; i < num_subfiles; i++) { + h5_stat_size_t subfile_size; + h5_stat_t subfile_info; + + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, i + 1, num_subfiles); + + /* Ensure file exists */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr, "fopen on subfile succeeded"); + VRFY((fclose(subfile_ptr) >= 0), "fclose on subfile succeeded"); + + /* Check file size */ + VRFY((HDstat(tmp_filename, &subfile_info) >= 0), "HDstat succeeded"); + subfile_size = (h5_stat_size_t)subfile_info.st_size; + + /* + * Subfiles indexed 0 and 1 should both be (1 * stripe size) + * bytes. Subfile index 2 should have a single byte written to + * it and Subfile index 3 should have nothing written to it. + */ + if (i == 2) { + VRFY((subfile_size == 1), "File size verification succeeded"); + } + else if (i == 3) { + VRFY((subfile_size == 0), "File size verification succeeded"); + } + else { + VRFY((subfile_size == cfg.stripe_size), "File size verification succeeded"); + } + } + + /* Verify that there aren't too many subfiles */ + snprintf(tmp_filename, PATH_MAX, H5FD_SUBFILING_FILENAME_TEMPLATE, SUBF_FILENAME, + (uint64_t)file_info.st_ino, num_digits, num_subfiles + 1, num_subfiles); + + /* Ensure file doesn't exist */ + subfile_ptr = fopen(tmp_filename, "r"); + VRFY(subfile_ptr == NULL, "fopen on subfile correctly failed"); + + VRFY((H5FDclose(file_ptr) >= 0), "H5FDclose succeeded"); + + free(write_buf); + write_buf = NULL; + free(read_buf); + write_buf = NULL; + + free(tmp_filename); + + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); + + H5E_BEGIN_TRY + { + H5Fdelete(SUBF_FILENAME, fapl_id); + } + H5E_END_TRY + + VRFY((H5Pclose(fapl_id) >= 0), "FAPL close succeeded"); + } + + mpi_code_g = MPI_Barrier(comm_g); + VRFY((mpi_code_g == MPI_SUCCESS), "MPI_Barrier succeeded"); + + if (skip) { + if (MAINPROCESS) + SKIPPED(); + } + else + CHECK_PASSED(); +} +#undef SUBF_FILENAME + +/* * Test the different I/O Concentator selection strategies * for the Subfiling VFD */ @@ -2360,11 +3053,33 @@ main(int argc, char **argv) if (MAINPROCESS) puts(""); + if (MAINPROCESS) + printf(" Re-running tests with compression enabled\n"); + +#ifdef H5_HAVE_FILTER_DEFLATE + enable_compression = true; + for (size_t i = 0; i < ARRAY_SIZE(tests); i++) { + if (MPI_SUCCESS == (mpi_code_g = MPI_Barrier(comm_g))) { + (*tests[i])(); + } + else { + if (MAINPROCESS) + MESG("MPI_Barrier failed"); + nerrors++; + } + } + enable_compression = false; +#else + if (MAINPROCESS) + SKIPPED(); +#endif + /* * Set any unset Subfiling environment variables and re-run * the tests as a quick smoke check of whether those are * working correctly */ + if (stripe_size_g < 0) { int64_t stripe_size; char tmp[64]; @@ -2488,26 +3203,6 @@ main(int argc, char **argv) num_iocs_g = mpi_size; if (MAINPROCESS) - printf(" Re-running tests with compression enabled\n"); - -#ifdef H5_HAVE_FILTER_DEFLATE - enable_compression = true; - for (size_t i = 0; i < ARRAY_SIZE(tests); i++) { - if (MPI_SUCCESS == (mpi_code_g = MPI_Barrier(comm_g))) { - (*tests[i])(); - } - else { - if (MAINPROCESS) - MESG("MPI_Barrier failed"); - nerrors++; - } - } - enable_compression = false; -#else - if (MAINPROCESS) - SKIPPED(); -#endif - if (MAINPROCESS) printf("\nRe-running tests with environment variables set\n"); for (size_t i = 0; i < ARRAY_SIZE(tests); i++) { @@ -2523,6 +3218,7 @@ main(int argc, char **argv) if (MAINPROCESS) printf("\n Re-running tests with compression enabled\n"); + #ifdef H5_HAVE_FILTER_DEFLATE enable_compression = true; for (size_t i = 0; i < ARRAY_SIZE(tests); i++) { @@ -2540,6 +3236,7 @@ main(int argc, char **argv) if (MAINPROCESS) SKIPPED(); #endif + if (nerrors) goto exit; diff --git a/tools/libtest/CMakeLists.txt b/tools/libtest/CMakeLists.txt index f6d8912..3578560 100644 --- a/tools/libtest/CMakeLists.txt +++ b/tools/libtest/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_LIBTEST C) #----------------------------------------------------------------------------- add_executable (h5tools_test_utils ${HDF5_TOOLS_LIBTEST_SOURCE_DIR}/h5tools_test_utils.c) target_compile_options(h5tools_test_utils PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_include_directories(h5tools_test_utils PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") +target_include_directories(h5tools_test_utils PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (BUILD_STATIC_LIBS) TARGET_C_PROPERTIES (h5tools_test_utils STATIC) target_link_libraries (h5tools_test_utils PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${HDF5_TEST_LIB_TARGET}) diff --git a/tools/src/h5copy/CMakeLists.txt b/tools/src/h5copy/CMakeLists.txt index efd38cd..5b79a85 100644 --- a/tools/src/h5copy/CMakeLists.txt +++ b/tools/src/h5copy/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_SRC_H5COPY C) # -------------------------------------------------------------------- if (BUILD_STATIC_LIBS) add_executable (h5copy ${HDF5_TOOLS_SRC_H5COPY_SOURCE_DIR}/h5copy.c) - target_include_directories (h5copy PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5copy PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5copy PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5copy STATIC) target_link_libraries (h5copy PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -18,7 +18,7 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5copy-shared ${HDF5_TOOLS_SRC_H5COPY_SOURCE_DIR}/h5copy.c) - target_include_directories (h5copy-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5copy-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5copy-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5copy-shared SHARED) target_link_libraries (h5copy-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) diff --git a/tools/src/h5diff/CMakeLists.txt b/tools/src/h5diff/CMakeLists.txt index f01d1aa..8de7c61 100644 --- a/tools/src/h5diff/CMakeLists.txt +++ b/tools/src/h5diff/CMakeLists.txt @@ -10,7 +10,7 @@ if (BUILD_STATIC_LIBS) ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/h5diff_main.c ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/h5diff_common.h ) - target_include_directories (h5diff PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5diff PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5diff PRIVATE "${HDF5_CMAKE_C_FLAGS}") #target_compile_definitions (h5diff PRIVATE H5_TOOLS_DEBUG) TARGET_C_PROPERTIES (h5diff STATIC) @@ -26,7 +26,7 @@ if (BUILD_SHARED_LIBS) ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/h5diff_main.c ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/h5diff_common.h ) - target_include_directories (h5diff-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5diff-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5diff-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") #target_compile_definitions (h5diff-shared PRIVATE H5_TOOLS_DEBUG) TARGET_C_PROPERTIES (h5diff-shared SHARED) @@ -54,7 +54,7 @@ if (H5_HAVE_PARALLEL) ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/h5diff_common.c ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/ph5diff_main.c ) - target_include_directories (ph5diff PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (ph5diff PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(ph5diff PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (ph5diff STATIC) target_link_libraries (ph5diff PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} "$<$:MPI::MPI_C>") @@ -67,7 +67,7 @@ if (H5_HAVE_PARALLEL) ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/h5diff_common.c ${HDF5_TOOLS_SRC_H5DIFF_SOURCE_DIR}/ph5diff_main.c ) - target_include_directories (ph5diff-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (ph5diff-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(ph5diff-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (ph5diff-shared SHARED) target_link_libraries (ph5diff-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} "$<$:MPI::MPI_C>") diff --git a/tools/src/h5dump/CMakeLists.txt b/tools/src/h5dump/CMakeLists.txt index f382020..3284949 100644 --- a/tools/src/h5dump/CMakeLists.txt +++ b/tools/src/h5dump/CMakeLists.txt @@ -15,7 +15,7 @@ if (BUILD_STATIC_LIBS) ${HDF5_TOOLS_SRC_H5DUMP_SOURCE_DIR}/h5dump_ddl.h ${HDF5_TOOLS_SRC_H5DUMP_SOURCE_DIR}/h5dump_xml.h ) - target_include_directories (h5dump PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5dump PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5dump PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5dump STATIC) target_link_libraries (h5dump PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -36,7 +36,7 @@ if (BUILD_SHARED_LIBS) ${HDF5_TOOLS_SRC_H5DUMP_SOURCE_DIR}/h5dump_ddl.h ${HDF5_TOOLS_SRC_H5DUMP_SOURCE_DIR}/h5dump_xml.h ) - target_include_directories (h5dump-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5dump-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5dump-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5dump-shared SHARED) target_link_libraries (h5dump-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) diff --git a/tools/src/h5format_convert/CMakeLists.txt b/tools/src/h5format_convert/CMakeLists.txt index d1e2158..2a7e3cb 100644 --- a/tools/src/h5format_convert/CMakeLists.txt +++ b/tools/src/h5format_convert/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_SRC_H5FC C) # -------------------------------------------------------------------- if (BUILD_STATIC_LIBS) add_executable (h5format_convert ${HDF5_TOOLS_SRC_H5FC_SOURCE_DIR}/h5format_convert.c) - target_include_directories (h5format_convert PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5format_convert PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5format_convert PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5format_convert STATIC) target_link_libraries (h5format_convert PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -17,7 +17,7 @@ if (BUILD_STATIC_LIBS) endif () if (BUILD_SHARED_LIBS) add_executable (h5format_convert-shared ${HDF5_TOOLS_SRC_H5FC_SOURCE_DIR}/h5format_convert.c) - target_include_directories (h5format_convert-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5format_convert-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5format_convert-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5format_convert-shared SHARED) target_link_libraries (h5format_convert-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) diff --git a/tools/src/h5import/CMakeLists.txt b/tools/src/h5import/CMakeLists.txt index b2337cd..f8268a1 100644 --- a/tools/src/h5import/CMakeLists.txt +++ b/tools/src/h5import/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_SRC_H5IMPORT C) # -------------------------------------------------------------------- if (BUILD_STATIC_LIBS) add_executable (h5import ${HDF5_TOOLS_SRC_H5IMPORT_SOURCE_DIR}/h5import.c ${HDF5_TOOLS_SRC_H5IMPORT_SOURCE_DIR}/h5import.h) - target_include_directories (h5import PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5import PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5import STATIC) target_link_libraries (h5import PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) #set_target_properties (h5import PROPERTIES COMPILE_DEFINITIONS H5DEBUGIMPORT) @@ -18,7 +18,7 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5import-shared ${HDF5_TOOLS_SRC_H5IMPORT_SOURCE_DIR}/h5import.c ${HDF5_TOOLS_SRC_H5IMPORT_SOURCE_DIR}/h5import.h) - target_include_directories (h5import-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5import-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5import-shared SHARED) target_link_libraries (h5import-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) #set_target_properties (h5import-shared PROPERTIES COMPILE_DEFINITIONS H5DEBUGIMPORT) diff --git a/tools/src/h5import/h5import.c b/tools/src/h5import/h5import.c index 2d3574a..663f68a 100644 --- a/tools/src/h5import/h5import.c +++ b/tools/src/h5import/h5import.c @@ -26,48 +26,48 @@ #endif /* Local function declarations */ -static int gtoken(char *s); -static int process(struct Options *opt); -static int processConfigurationFile(char *infile, struct Input *in); -static int mapKeywordToIndex(char *key); -static int parsePathInfo(struct path_info *path, char *strm); -static int parseDimensions(struct Input *in, char *strm); -static int getInputSize(struct Input *in, int ival); -static int getInputClass(struct Input *in, char *strm); -static int getInputClassType(struct Input *in, char *strm); -static int getInputByteOrder(struct Input *in, FILE *strm); -static int InputClassStrToInt(char *temp); -static int getRank(struct Input *in, FILE *strm); -static int getDimensionSizes(struct Input *in, FILE *strm); -static int getOutputSize(struct Input *in, FILE *strm); -static int getOutputClass(struct Input *in, FILE *strm); -static int OutputClassStrToInt(char *temp); -static int getOutputArchitecture(struct Input *in, FILE *strm); -static int OutputArchStrToInt(const char *temp); -static int getOutputByteOrder(struct Input *in, FILE *strm); -static int OutputByteOrderStrToInt(const char *temp); -static int getChunkedDimensionSizes(struct Input *in, FILE *strm); -static int getCompressionType(struct Input *in, FILE *strm); -static int CompressionTypeStrToInt(char *temp); -static int getCompressionParameter(struct Input *in, FILE *strm); -static int getExternalFilename(struct Input *in, FILE *strm); -static int getMaximumDimensionSizes(struct Input *in, FILE *strm); -static int processDataFile(char *infile, struct Input *in, hid_t file_id); -static int readIntegerData(FILE *strm, struct Input *in); -static int readFloatData(FILE *strm, struct Input *in); -static int allocateIntegerStorage(struct Input *in); -static int allocateFloatStorage(struct Input *in); -static int readUIntegerData(FILE *strm, struct Input *in); -static int allocateUIntegerStorage(struct Input *in); -static int validateConfigurationParameters(struct Input *in); -static int processStrData(FILE *strm, struct Input *in, hid_t file_id); -static int processStrHDFData(FILE *strm, struct Input *in, hid_t file_id); -uint16_t swap_uint16(uint16_t val); -int16_t swap_int16(int16_t val); -uint32_t swap_uint32(uint32_t val); -int32_t swap_int32(int32_t val); -int64_t swap_int64(int64_t val); -uint64_t swap_uint64(uint64_t val); +static int gtoken(char *s); +static int process(struct Options *opt); +static int processConfigurationFile(char *infile, struct Input *in); +static int mapKeywordToIndex(char *key); +static int parsePathInfo(struct path_info *path, char *strm); +static int parseDimensions(struct Input *in, char *strm); +static int getInputSize(struct Input *in, int ival); +static int getInputClass(struct Input *in, char *strm); +static int getInputClassType(struct Input *in, char *strm); +static int getInputByteOrder(struct Input *in, FILE *strm); +static int InputClassStrToInt(char *temp); +static int getRank(struct Input *in, FILE *strm); +static int getDimensionSizes(struct Input *in, FILE *strm); +static int getOutputSize(struct Input *in, FILE *strm); +static int getOutputClass(struct Input *in, FILE *strm); +static int OutputClassStrToInt(char *temp); +static int getOutputArchitecture(struct Input *in, FILE *strm); +static int OutputArchStrToInt(const char *temp); +static int getOutputByteOrder(struct Input *in, FILE *strm); +static int OutputByteOrderStrToInt(const char *temp); +static int getChunkedDimensionSizes(struct Input *in, FILE *strm); +static int getCompressionType(struct Input *in, FILE *strm); +static int CompressionTypeStrToInt(char *temp); +static int getCompressionParameter(struct Input *in, FILE *strm); +static int getExternalFilename(struct Input *in, FILE *strm); +static int getMaximumDimensionSizes(struct Input *in, FILE *strm); +static int processDataFile(char *infile, struct Input *in, hid_t file_id); +static int readIntegerData(FILE *strm, struct Input *in); +static int readFloatData(FILE *strm, struct Input *in); +static int allocateIntegerStorage(struct Input *in); +static int allocateFloatStorage(struct Input *in); +static int readUIntegerData(FILE *strm, struct Input *in); +static int allocateUIntegerStorage(struct Input *in); +static int validateConfigurationParameters(struct Input *in); +static int processStrData(FILE *strm, struct Input *in, hid_t file_id); +static int processStrHDFData(FILE *strm, struct Input *in, hid_t file_id); +H5_ATTR_CONST uint16_t swap_uint16(uint16_t val); +H5_ATTR_CONST int16_t swap_int16(int16_t val); +H5_ATTR_CONST uint32_t swap_uint32(uint32_t val); +H5_ATTR_CONST int32_t swap_int32(int32_t val); +H5_ATTR_CONST int64_t swap_int64(int64_t val); +H5_ATTR_CONST uint64_t swap_uint64(uint64_t val); int main(int argc, char *argv[]) diff --git a/tools/src/h5jam/CMakeLists.txt b/tools/src/h5jam/CMakeLists.txt index 8642d6f..7efd0d8 100644 --- a/tools/src/h5jam/CMakeLists.txt +++ b/tools/src/h5jam/CMakeLists.txt @@ -6,14 +6,14 @@ project (HDF5_TOOLS_SRC_H5JAM C) # -------------------------------------------------------------------- if (BUILD_STATIC_LIBS) add_executable (h5jam ${HDF5_TOOLS_SRC_H5JAM_SOURCE_DIR}/h5jam.c) - target_include_directories (h5jam PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5jam PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5jam STATIC) target_link_libraries (h5jam PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) set_target_properties (h5jam PROPERTIES FOLDER tools) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5jam") add_executable (h5unjam ${HDF5_TOOLS_SRC_H5JAM_SOURCE_DIR}/h5unjam.c) - target_include_directories (h5unjam PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5unjam PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5unjam STATIC) target_link_libraries (h5unjam PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) set_target_properties (h5unjam PROPERTIES FOLDER tools) @@ -27,14 +27,14 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5jam-shared ${HDF5_TOOLS_SRC_H5JAM_SOURCE_DIR}/h5jam.c) - target_include_directories (h5jam-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5jam-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5jam-shared SHARED) target_link_libraries (h5jam-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) set_target_properties (h5jam-shared PROPERTIES FOLDER tools) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5jam-shared") add_executable (h5unjam-shared ${HDF5_TOOLS_SRC_H5JAM_SOURCE_DIR}/h5unjam.c) - target_include_directories (h5unjam-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5unjam-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5unjam-shared SHARED) target_link_libraries (h5unjam-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) set_target_properties (h5unjam-shared PROPERTIES FOLDER tools) diff --git a/tools/src/h5ls/CMakeLists.txt b/tools/src/h5ls/CMakeLists.txt index 9499253..67122a1 100644 --- a/tools/src/h5ls/CMakeLists.txt +++ b/tools/src/h5ls/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_SRC_H5LS C) #----------------------------------------------------------------------------- if (BUILD_STATIC_LIBS) add_executable (h5ls ${HDF5_TOOLS_SRC_H5LS_SOURCE_DIR}/h5ls.c) - target_include_directories (h5ls PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5ls PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5ls PRIVATE "${HDF5_CMAKE_C_FLAGS}") #target_compile_definitions(h5ls PRIVATE H5_TOOLS_DEBUG) TARGET_C_PROPERTIES (h5ls STATIC) @@ -19,7 +19,7 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5ls-shared ${HDF5_TOOLS_SRC_H5LS_SOURCE_DIR}/h5ls.c) - target_include_directories (h5ls-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5ls-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5ls-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") #target_compile_definitions(h5ls-shared PRIVATE H5_TOOLS_DEBUG) TARGET_C_PROPERTIES (h5ls-shared SHARED) diff --git a/tools/src/h5repack/CMakeLists.txt b/tools/src/h5repack/CMakeLists.txt index 360fb0f..ea1ee80 100644 --- a/tools/src/h5repack/CMakeLists.txt +++ b/tools/src/h5repack/CMakeLists.txt @@ -17,7 +17,7 @@ set (REPACK_COMMON_SOURCES if (BUILD_STATIC_LIBS) add_executable (h5repack ${REPACK_COMMON_SOURCES} ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_main.c) - target_include_directories (h5repack PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5repack PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5repack PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5repack STATIC) target_link_libraries (h5repack PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -29,7 +29,7 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5repack-shared ${REPACK_COMMON_SOURCES} ${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR}/h5repack_main.c) - target_include_directories (h5repack-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5repack-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5repack-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5repack-shared SHARED) target_link_libraries (h5repack-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) diff --git a/tools/src/h5stat/CMakeLists.txt b/tools/src/h5stat/CMakeLists.txt index c0c0b32..c3aef5f 100644 --- a/tools/src/h5stat/CMakeLists.txt +++ b/tools/src/h5stat/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_SRC_H5STAT C) # -------------------------------------------------------------------- if (BUILD_STATIC_LIBS) add_executable (h5stat ${HDF5_TOOLS_SRC_H5STAT_SOURCE_DIR}/h5stat.c) - target_include_directories (h5stat PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5stat PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5stat PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5stat STATIC) target_link_libraries (h5stat PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -18,7 +18,7 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5stat-shared ${HDF5_TOOLS_SRC_H5STAT_SOURCE_DIR}/h5stat.c) - target_include_directories (h5stat-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5stat-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5stat-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5stat-shared SHARED) target_link_libraries (h5stat-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) diff --git a/tools/src/misc/CMakeLists.txt b/tools/src/misc/CMakeLists.txt index d2b1f51..62bd443 100644 --- a/tools/src/misc/CMakeLists.txt +++ b/tools/src/misc/CMakeLists.txt @@ -7,7 +7,7 @@ project (HDF5_TOOLS_SRC_MISC C) #-- Misc Executables if (BUILD_STATIC_LIBS) add_executable (h5debug ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5debug.c) - target_include_directories (h5debug PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5debug PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5debug PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5debug STATIC) target_link_libraries (h5debug PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -15,7 +15,7 @@ if (BUILD_STATIC_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5debug") add_executable (h5repart ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5repart.c) - target_include_directories (h5repart PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5repart PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5repart PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5repart STATIC) target_link_libraries (h5repart PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -23,7 +23,7 @@ if (BUILD_STATIC_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5repart") add_executable (h5mkgrp ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5mkgrp.c) - target_include_directories (h5mkgrp PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5mkgrp PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5mkgrp PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5mkgrp STATIC) target_link_libraries (h5mkgrp PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -31,7 +31,7 @@ if (BUILD_STATIC_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5mkgrp") add_executable (h5clear ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5clear.c) - target_include_directories (h5clear PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5clear PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5clear PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5clear STATIC) target_link_libraries (h5clear PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -39,7 +39,7 @@ if (BUILD_STATIC_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5clear") add_executable (h5delete ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5delete.c) - target_include_directories (h5delete PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5delete PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5delete PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5delete STATIC) target_link_libraries (h5delete PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -56,7 +56,7 @@ if (BUILD_STATIC_LIBS) endif () if (BUILD_SHARED_LIBS) add_executable (h5debug-shared ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5debug.c) - target_include_directories (h5debug-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5debug-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5debug-shared SHARED) target_compile_options(h5debug-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") target_link_libraries (h5debug-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) @@ -64,7 +64,7 @@ if (BUILD_SHARED_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5debug-shared") add_executable (h5repart-shared ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5repart.c) - target_include_directories (h5repart-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5repart-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5repart-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5repart-shared SHARED) target_link_libraries (h5repart-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) @@ -72,7 +72,7 @@ if (BUILD_SHARED_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5repart-shared") add_executable (h5mkgrp-shared ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5mkgrp.c) - target_include_directories (h5mkgrp-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5mkgrp-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5mkgrp-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5mkgrp-shared SHARED) target_link_libraries (h5mkgrp-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) @@ -80,7 +80,7 @@ if (BUILD_SHARED_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5mkgrp-shared") add_executable (h5clear-shared ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5clear.c) - target_include_directories (h5clear-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5clear-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5clear-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5clear-shared SHARED) target_link_libraries (h5clear-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) @@ -88,7 +88,7 @@ if (BUILD_SHARED_LIBS) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5clear-shared") add_executable (h5delete-shared ${HDF5_TOOLS_SRC_MISC_SOURCE_DIR}/h5delete.c) - target_include_directories (h5delete-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5delete-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5delete-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5delete-shared SHARED) target_link_libraries (h5delete-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) diff --git a/tools/test/h5copy/CMakeTests.cmake b/tools/test/h5copy/CMakeTests.cmake index fb02d81..b4daa87 100644 --- a/tools/test/h5copy/CMakeTests.cmake +++ b/tools/test/h5copy/CMakeTests.cmake @@ -163,8 +163,8 @@ COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ -i ./testfiles/${infile} -o ./testfiles/${testname}.out.h5 -v -s ${psparam} -d ${pdparam} ) set_tests_properties (H5COPY-${testname}-prefill PROPERTIES DEPENDS H5COPY-${testname}-clear-objects) - if ("" MATCHES "${HDF5_DISABLE_TESTS_REGEX}") - set_tests_properties ( PROPERTIES DISABLED true) + if ("H5COPY-${testname}-prefill" MATCHES "${HDF5_DISABLE_TESTS_REGEX}") + set_tests_properties (H5COPY-${testname}-prefill PROPERTIES DISABLED true) endif () add_test ( diff --git a/tools/test/h5jam/CMakeLists.txt b/tools/test/h5jam/CMakeLists.txt index 160ecdf..e6017fd 100644 --- a/tools/test/h5jam/CMakeLists.txt +++ b/tools/test/h5jam/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_TEST_H5JAM C) # -------------------------------------------------------------------- if (HDF5_BUILD_GENERATORS AND BUILD_STATIC_LIBS) add_executable (h5jamgentest ${HDF5_TOOLS_TEST_H5JAM_SOURCE_DIR}/h5jamgentest.c) - target_include_directories (h5jamgentest PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5jamgentest PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (h5jamgentest STATIC) target_link_libraries (h5jamgentest PRIVATE ${HDF5_LIB_TARGET}) set_target_properties (h5jamgentest PROPERTIES FOLDER generator/tools) @@ -22,7 +22,7 @@ if (HDF5_BUILD_GENERATORS AND BUILD_STATIC_LIBS) endif () add_executable (getub ${HDF5_TOOLS_TEST_H5JAM_SOURCE_DIR}/getub.c) -target_include_directories (getub PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") +target_include_directories (getub PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (BUILD_STATIC_LIBS) TARGET_C_PROPERTIES (getub STATIC) target_link_libraries (getub PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) @@ -33,7 +33,7 @@ endif () set_target_properties (getub PROPERTIES FOLDER tools) add_executable (tellub ${HDF5_TOOLS_TEST_H5JAM_SOURCE_DIR}/tellub.c) -target_include_directories (tellub PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") +target_include_directories (tellub PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (BUILD_STATIC_LIBS) TARGET_C_PROPERTIES (tellub STATIC) target_link_libraries (tellub PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) diff --git a/tools/test/h5repack/CMakeLists.txt b/tools/test/h5repack/CMakeLists.txt index 3e7c6c2..5c0075c 100644 --- a/tools/test/h5repack/CMakeLists.txt +++ b/tools/test/h5repack/CMakeLists.txt @@ -6,7 +6,7 @@ project (HDF5_TOOLS_TEST_H5REPACK C) # -------------------------------------------------------------------- add_executable (testh5repack_detect_szip ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testh5repack_detect_szip.c) target_include_directories (testh5repack_detect_szip - PRIVATE "${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR};${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>" + PRIVATE "${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR};${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>" ) if (BUILD_STATIC_LIBS) TARGET_C_PROPERTIES (testh5repack_detect_szip STATIC) @@ -29,7 +29,7 @@ set (REPACK_COMMON_SOURCES ) add_executable (h5repacktest ${REPACK_COMMON_SOURCES} ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/h5repacktst.c) target_include_directories (h5repacktest - PRIVATE "${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR};${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_TEST_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>" + PRIVATE "${HDF5_TOOLS_SRC_H5REPACK_SOURCE_DIR};${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_TEST_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>" ) if (BUILD_STATIC_LIBS) TARGET_C_PROPERTIES (h5repacktest STATIC) diff --git a/utils/mirror_vfd/mirror_writer.c b/utils/mirror_vfd/mirror_writer.c index a5a1d27..f156965 100644 --- a/utils/mirror_vfd/mirror_writer.c +++ b/utils/mirror_vfd/mirror_writer.c @@ -29,7 +29,6 @@ /* in detailed logging */ #define HEXDUMP_WRITEDATA 0 /* Toggle whether to print bytes to write */ /* in detailed logging */ -#define LISTENQ 80 /* max pending Driver requests */ #define MW_SESSION_MAGIC 0x88F36B32u #define MW_SOCK_COMM_MAGIC 0xDF10A157u diff --git a/utils/tools/h5dwalk/CMakeLists.txt b/utils/tools/h5dwalk/CMakeLists.txt index 9e4eb6d..f0611e0 100644 --- a/utils/tools/h5dwalk/CMakeLists.txt +++ b/utils/tools/h5dwalk/CMakeLists.txt @@ -7,9 +7,9 @@ project (HDF5_UTILS_TOOLS_H5DWALK C) if (BUILD_STATIC_LIBS) add_executable (h5dwalk ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) # add_custom_target(generate_demo ALL -# DEPENDS "${HDF5_TOOLS_DIR}/test/demo_destfiles.test" +# DEPENDS "${HDF5_TOOLS_ROOT_DIR}/test/demo_destfiles.test" # ) - target_include_directories (h5dwalk PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5dwalk PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5dwalk PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5dwalk STATIC) target_link_libraries (h5dwalk PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${MFU_LIBRARY} "$<$:MPI::MPI_C>") @@ -21,7 +21,7 @@ endif () if (BUILD_SHARED_LIBS) add_executable (h5dwalk-shared ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) - target_include_directories (h5dwalk-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5dwalk-shared PRIVATE "${HDF5_TOOLS_ROOT_DIR}/lib;${HDF5_SRC_INCLUDE_DIRS};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") target_compile_options(h5dwalk-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") TARGET_C_PROPERTIES (h5dwalk-shared SHARED) target_link_libraries (h5dwalk-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${MFU_LIBRARY} "$<$:MPI::MPI_C>") diff --git a/utils/tools/test/h5dwalk/CMakeLists.txt b/utils/tools/test/h5dwalk/CMakeLists.txt index 530bed8..a27190c 100644 --- a/utils/tools/test/h5dwalk/CMakeLists.txt +++ b/utils/tools/test/h5dwalk/CMakeLists.txt @@ -3,9 +3,9 @@ project (HDF5_TOOLS_TEST_H5DWALK) if (HDF5_BUILD_PARALLEL_TOOLS) add_custom_command( - OUTPUT ${HDF5_TOOLS_DIR}/test/demo_destfiles.test + OUTPUT ${HDF5_TOOLS_ROOT_DIR}/test/demo_destfiles.test COMMAND bash -c ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh - ARGS ${HDF5_TOOLS_DIR}/test ${CMAKE_BINARY_DIR}/bin + ARGS ${HDF5_TOOLS_ROOT_DIR}/test ${CMAKE_BINARY_DIR}/bin DEPENDS ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh ) endif () -- cgit v0.12