From 78375882485a99a81caa933928ed08d7a38ef88b Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Sun, 6 Mar 2022 15:33:00 -0800 Subject: VFD SWMR: normalization with develop (#1472) Much normalization with develop. Still needs tools changes wrt VFD plugins. --- .gitattributes | 14 +- .github/CODEOWNERS | 30 +- .github/FUNDING.yml | 3 + .github/workflows/codespell.yml | 15 + .gitmodules | 0 CMakeFilters.cmake | 10 +- CMakeInstallation.cmake | 18 +- CMakeLists.txt | 53 +- MANIFEST | 64 +- README.md | 85 + README.txt | 80 - bin/bbrelease | 10 +- bin/buildhdf5 | 2 +- bin/cmakehdf5 | 4 +- bin/genparser | 2 +- bin/h5cc.in | 6 +- bin/h5vers | 12 +- bin/newer | 2 +- bin/release | 10 +- bin/runtest | 6 +- bin/snapshot | 10 +- bin/trace | 4 +- bin/warnhist | 2 +- c++/src/H5AbstractDs.cpp | 2 +- c++/src/H5Attribute.cpp | 2 +- c++/src/H5DataSet.cpp | 2 +- c++/src/H5DataSpace.cpp | 2 +- c++/src/H5DataType.cpp | 4 +- c++/src/H5DcreatProp.cpp | 2 +- c++/src/H5File.cpp | 2 +- c++/src/H5Group.cpp | 2 +- c++/src/H5Location.cpp | 2 +- c++/src/H5Location.h | 4 +- c++/src/H5PredType.cpp | 2 +- c++/src/H5PropList.cpp | 2 +- c++/src/cpp_doc_config | 2 +- c++/test/CMakeVFDTests.cmake | 16 +- c++/test/tattr.cpp | 8 +- c++/test/testhdf5.cpp | 2 +- c++/test/th5s.cpp | 2 +- c++/test/titerate.cpp | 9 +- c++/test/tobject.cpp | 2 +- c++/test/ttypes.cpp | 2 +- c++/test/tvlstr.cpp | 10 +- config/BlankForm | 4 +- config/cmake/CTestCustom.cmake | 2 + config/cmake/ConfigureChecks.cmake | 2 +- config/cmake/H5pubconf.h.in | 5 +- config/cmake/HDF5Macros.cmake | 41 + config/cmake/HDF5PluginMacros.cmake | 4 +- config/cmake/HDF5UseFortran.cmake | 4 +- config/cmake/HDF5_Examples.cmake.in | 4 +- config/cmake/HDFCXXCompilerFlags.cmake | 2 +- config/cmake/README.md.cmake.in | 80 + config/cmake/README.txt.cmake.in | 80 - config/cmake/cacheinit.cmake | 22 +- config/cmake/hdf5-config.cmake.in | 6 +- config/cmake/libh5cc.in | 23 +- config/cmake/scripts/HDF5config.cmake | 4 +- config/cmake_ext_mod/ConfigureChecks.cmake | 100 +- config/cmake_ext_mod/HDFLibMacros.cmake | 20 +- config/cmake_ext_mod/HDFMacros.cmake | 10 +- config/cmake_ext_mod/HDFTests.c | 4 +- config/cmake_ext_mod/HDFUseFortran.cmake | 2 +- config/cmake_ext_mod/NSIS.template.in | 4 +- config/cmake_ext_mod/runTest.cmake | 10 +- config/conclude.am | 19 +- config/gnu-cxxflags | 2 +- config/gnu-flags | 2 +- config/sanitizer/code-coverage.cmake | 15 +- config/sanitizer/sanitizers.cmake | 11 +- config/solaris | 6 +- configure.ac | 188 +- doxygen/CMakeLists.txt | 2 +- doxygen/aliases | 2 +- doxygen/dox/About.dox | 2 - doxygen/dox/Glossary.dox | 565 ++ doxygen/dox/H5AC_cache_config_t.dox | 8 +- doxygen/dox/Overview.dox | 38 +- doxygen/examples/DebuggingHDF5Applications.html | 2 +- doxygen/examples/Filters.html | 8 +- doxygen/examples/H5.format.1.0.html | 24 +- doxygen/examples/H5.format.1.1.html | 18 +- doxygen/examples/H5.format.2.0.html | 6 +- doxygen/examples/H5.format.html | 6 +- doxygen/examples/ThreadSafeLibrary.html | 2 +- doxygen/examples/VFL.html | 4 +- doxygen/hdf5_navtree_hacks.js | 2 +- doxygen/hdf5doxy_layout.xml | 2 +- doxygen/img/HDF5.png | Bin 0 -> 10660 bytes examples/CMakeLists.txt | 42 +- examples/CMakeTests.cmake | 40 +- examples/Makefile.am | 6 +- examples/gaussians.c | 2 +- examples/h5_drivers.c | 2 +- examples/h5_extlink.c | 4 +- examples/h5_mount.c | 2 +- examples/h5_shared_mesg.c | 4 +- examples/h5_vds-percival-unlim-maxmin.c | 2 +- examples/ph5_filtered_writes.c | 490 + examples/ph5_filtered_writes_no_sel.c | 370 + examples/ph5example.c | 14 +- examples/testh5cc.sh.in | 2 +- fortran/examples/compound.f90 | 14 +- fortran/examples/h5_cmprss.f90 | 2 +- fortran/examples/h5_crtatt.f90 | 2 +- fortran/examples/h5_extend.f90 | 4 +- fortran/examples/refobjexample.f90 | 2 +- fortran/examples/testh5fc.sh.in | 2 +- fortran/src/H5Af.c | 8 +- fortran/src/H5Aff.F90 | 14 +- fortran/src/H5Df.c | 14 +- fortran/src/H5Dff.F90 | 18 +- fortran/src/H5Ff.c | 4 +- fortran/src/H5Fff.F90 | 2 +- fortran/src/H5Gf.c | 8 +- fortran/src/H5Gff.F90 | 32 +- fortran/src/H5Lff.F90 | 6 +- fortran/src/H5Pf.c | 30 +- fortran/src/H5Pff.F90 | 16 +- fortran/src/H5Sf.c | 8 +- fortran/src/H5Sff.F90 | 2 +- fortran/src/H5Tf.c | 6 +- fortran/src/H5Tff.F90 | 4 +- fortran/src/H5_buildiface.F90 | 8 +- fortran/src/H5_ff.F90 | 4 +- fortran/src/H5f90global.F90 | 4 +- fortran/src/H5f90proto.h | 2 +- fortran/src/h5fc.in | 6 +- fortran/test/H5_test_buildiface.F90 | 2 +- fortran/test/fflush1.F90 | 2 +- fortran/test/fflush2.F90 | 4 +- fortran/test/t.c | 2 +- fortran/test/tH5A.F90 | 2 +- fortran/test/tH5A_1_8.F90 | 17 +- fortran/test/tH5D.F90 | 16 +- fortran/test/tH5F.F90 | 40 +- fortran/test/tH5G_1_8.F90 | 2 +- fortran/test/tH5MISC_1_8.F90 | 2 +- fortran/test/tH5P.F90 | 6 +- fortran/test/tH5P_F03.F90 | 12 +- fortran/test/tH5R.F90 | 2 +- fortran/test/tH5S.F90 | 16 +- fortran/test/tH5Sselect.F90 | 12 +- fortran/test/tH5T.F90 | 18 +- fortran/test/tH5VL.F90 | 4 +- fortran/test/tH5Z.F90 | 2 +- hl/c++/test/ptableTest.cpp | 6 +- hl/examples/ex_image1.c | 4 +- hl/fortran/src/H5DSff.F90 | 2 +- hl/fortran/src/H5HL_buildiface.F90 | 2 +- hl/fortran/src/H5LTff.F90 | 4 +- hl/fortran/src/H5TBfc.c | 4 +- hl/fortran/src/H5TBff.F90 | 2 +- hl/fortran/test/tsttable.F90 | 8 +- hl/src/H5DO.c | 2 +- hl/src/H5DS.c | 8 +- hl/src/H5LTpublic.h | 2 +- hl/test/test_ds.c | 28 +- hl/test/test_file_image.c | 7 +- hl/test/test_h5do_compat.c | 2 +- hl/test/test_image.c | 10 +- hl/test/test_ld.c | 6 +- hl/test/test_packet.c | 2 +- hl/test/test_packet_vlen.c | 18 +- hl/test/test_table.c | 2 +- hl/tools/gif2h5/decompress.c | 4 +- hl/tools/gif2h5/gif.h | 2 +- hl/tools/gif2h5/gif2mem.c | 2 +- hl/tools/gif2h5/gifread.c | 44 +- hl/tools/gif2h5/h52giftest.sh.in | 2 +- hl/tools/gif2h5/writehdf.c | 6 +- hl/tools/h5watch/CMakeLists.txt | 23 - hl/tools/h5watch/CMakeTests.cmake | 68 +- hl/tools/h5watch/Makefile.am | 3 +- hl/tools/h5watch/extend_dset.c | 6 +- hl/tools/h5watch/h5watch.c | 14 +- hl/tools/h5watch/swmr_check_compat_vfd.c | 55 - hl/tools/h5watch/testh5watch.sh.in | 7 +- java/examples/datasets/H5Ex_D_Sofloat.java | 13 +- java/examples/datasets/H5Ex_D_UnlimitedMod.java | 2 +- java/examples/datasets/JavaDatasetExample.sh.in | 6 +- java/examples/datasets/Makefile.am | 2 +- .../examples/datatypes/H5Ex_T_ObjectReference.java | 275 +- .../datatypes/H5Ex_T_ObjectReferenceAttribute.java | 335 +- java/examples/datatypes/JavaDatatypeExample.sh.in | 6 +- java/examples/datatypes/Makefile.am | 2 +- java/examples/groups/H5Ex_G_Corder.java | 2 +- java/examples/groups/JavaGroupExample.sh.in | 6 +- java/examples/groups/Makefile.am | 2 +- java/examples/intro/JavaIntroExample.sh.in | 6 +- java/examples/intro/Makefile.am | 2 +- .../testfiles/examples.datasets.H5Ex_D_Sofloat.txt | 8 +- java/lib/ext/slf4j-nop-1.7.25.jar | Bin 4007 -> 0 bytes java/lib/ext/slf4j-nop-1.7.33.jar | Bin 0 -> 4020 bytes java/lib/ext/slf4j-simple-1.7.25.jar | Bin 15257 -> 0 bytes java/lib/ext/slf4j-simple-1.7.33.jar | Bin 0 -> 15400 bytes java/lib/slf4j-api-1.7.25.jar | Bin 41203 -> 0 bytes java/lib/slf4j-api-1.7.33.jar | Bin 0 -> 41473 bytes java/src/Makefile.am | 2 +- java/src/hdf/hdf5lib/H5.java | 32 +- .../hdf5lib/exceptions/HDF5LibraryException.java | 2 +- java/src/hdf/hdf5lib/package-info.java | 4 +- java/src/hdf/hdf5lib/structs/H5F_info2_t.java | 2 +- java/src/jni/h5util.c | 157 +- java/test/CMakeLists.txt | 1 + java/test/Makefile.am | 3 +- java/test/TestH5.java | 4 +- java/test/TestH5Ocopy.java | 300 +- java/test/TestH5OcopyOld.java | 397 + java/test/TestH5P.java | 4 +- java/test/TestH5Pfapl.java | 8 +- java/test/TestH5Plist.java | 10 +- java/test/junit.sh.in | 28 +- java/test/testfiles/JUnit-TestH5Ocopy.txt | 3 +- java/test/testfiles/JUnit-TestH5OcopyOld.txt | 10 + m4/aclocal_fc.f90 | 2 +- m4/aclocal_fc.m4 | 6 +- release_docs/HISTORY-1_0-1_8_0_rc3.txt | 82 +- release_docs/HISTORY-1_10.txt | 32 +- release_docs/HISTORY-1_10_0-1_12_0.txt | 2 +- release_docs/HISTORY-1_13.txt | 1758 ++++ release_docs/HISTORY-1_8.txt | 50 +- release_docs/HISTORY-1_8_0-1_10_0.txt | 26 +- release_docs/INSTALL_Cygwin.txt | 4 +- release_docs/RELEASE.txt | 1455 +-- release_docs/USING_CMake_Examples.txt | 7 +- release_docs/USING_HDF5_CMake.txt | 19 +- src/CMakeLists.txt | 1 + src/H5.c | 332 +- src/H5A.c | 8 +- src/H5AC.c | 125 +- src/H5ACdbg.c | 4 +- src/H5ACmpio.c | 111 +- src/H5ACpkg.h | 2 +- src/H5ACproxy_entry.c | 3 +- src/H5ACpublic.h | 12 +- src/H5Adense.c | 32 +- src/H5Adeprec.c | 2 +- src/H5Aint.c | 76 +- src/H5Apublic.h | 4 +- src/H5B.c | 3 - src/H5B2.c | 3 - src/H5C.c | 268 +- src/H5CS.c | 4 +- src/H5CX.c | 94 +- src/H5CXprivate.h | 2 + src/H5Cdbg.c | 4 +- src/H5Cimage.c | 12 +- src/H5Cmpio.c | 2 +- src/H5Cpkg.h | 13 +- src/H5Cprefetched.c | 2 +- src/H5Cprivate.h | 24 +- src/H5Cquery.c | 2 +- src/H5D.c | 7 +- src/H5Dchunk.c | 452 +- src/H5Dcompact.c | 136 +- src/H5Dcontig.c | 51 +- src/H5Dearray.c | 4 +- src/H5Defl.c | 28 +- src/H5Dfarray.c | 2 +- src/H5Dfill.c | 2 +- src/H5Dint.c | 173 +- src/H5Dio.c | 124 +- src/H5Dlayout.c | 2 +- src/H5Dmpio.c | 5484 +++++++++--- src/H5Dpkg.h | 66 +- src/H5Dpublic.h | 17 +- src/H5Dscatgath.c | 4 +- src/H5Dselect.c | 199 +- src/H5Dsingle.c | 8 +- src/H5Dvirtual.c | 94 +- src/H5E.c | 147 +- src/H5EA.c | 3 - src/H5EAdbg.c | 6 +- src/H5EApkg.h | 2 +- src/H5EAtest.c | 2 +- src/H5ES.c | 55 + src/H5ESdevelop.h | 2 + src/H5ESint.c | 138 +- src/H5ESlist.c | 13 +- src/H5ESpkg.h | 5 +- src/H5ESprivate.h | 1 + src/H5ESpublic.h | 13 +- src/H5Eint.c | 3 +- src/H5Epkg.h | 2 +- src/H5F.c | 4 +- src/H5FA.c | 5 +- src/H5FAcache.c | 2 +- src/H5FAdblkpage.c | 2 +- src/H5FApkg.h | 4 +- src/H5FAprivate.h | 4 +- src/H5FAtest.c | 2 +- src/H5FD.c | 222 +- src/H5FDcore.c | 80 +- src/H5FDcore.h | 3 +- src/H5FDdevelop.h | 6 + src/H5FDdirect.c | 136 +- src/H5FDdirect.h | 6 +- src/H5FDfamily.c | 246 +- src/H5FDfamily.h | 3 +- src/H5FDhdfs.c | 35 +- src/H5FDhdfs.h | 6 +- src/H5FDint.c | 394 +- src/H5FDlog.c | 138 +- src/H5FDlog.h | 3 +- src/H5FDmirror.c | 36 +- src/H5FDmirror.h | 5 +- src/H5FDmirror_priv.h | 10 +- src/H5FDmpi.c | 90 +- src/H5FDmpio.c | 471 +- src/H5FDmpio.h | 8 +- src/H5FDmulti.c | 483 +- src/H5FDmulti.h | 2 +- src/H5FDperform.c | 58 + src/H5FDprivate.h | 41 +- src/H5FDpublic.h | 139 + src/H5FDros3.c | 45 +- src/H5FDros3.h | 6 +- src/H5FDs3comms.h | 6 +- src/H5FDsec2.c | 98 +- src/H5FDsec2.h | 3 +- src/H5FDspace.c | 15 +- src/H5FDsplitter.c | 441 +- src/H5FDsplitter.h | 5 +- src/H5FDstdio.c | 4 +- src/H5FDstdio.h | 2 +- src/H5FDvfd_swmr.c | 31 +- src/H5FDvfd_swmr.h | 3 +- src/H5FDwindows.c | 2 +- src/H5FL.c | 169 +- src/H5FLprivate.h | 8 +- src/H5FS.c | 41 +- src/H5FSint.c | 25 - src/H5FSprivate.h | 5 +- src/H5FSsection.c | 65 +- src/H5Fcwfs.c | 6 +- src/H5Fefc.c | 2 +- src/H5Fint.c | 86 +- src/H5Fmount.c | 2 +- src/H5Fmpi.c | 196 +- src/H5Fpkg.h | 28 +- src/H5Fprivate.h | 10 +- src/H5Fpublic.h | 7 +- src/H5Fquery.c | 23 + src/H5Fsuper.c | 6 +- src/H5Fsuper_cache.c | 4 +- src/H5Fvfd_swmr.c | 16 +- src/H5Gcache.c | 4 +- src/H5Gcompact.c | 2 +- src/H5Gint.c | 72 +- src/H5Gnode.c | 2 +- src/H5Gpublic.h | 2 +- src/H5HF.c | 5 +- src/H5HFcache.c | 4 +- src/H5HFdbg.c | 4 +- src/H5HFdblock.c | 2 +- src/H5HFspace.c | 2 +- src/H5HG.c | 3 - src/H5HL.c | 3 - src/H5I.c | 2 +- src/H5Iint.c | 41 +- src/H5Ipublic.h | 2 +- src/H5Lint.c | 47 +- src/H5Lpublic.h | 16 +- src/H5M.c | 64 +- src/H5MF.c | 69 +- src/H5MFaggr.c | 41 +- src/H5MFdbg.c | 2 +- src/H5MFsection.c | 44 +- src/H5MP.c | 3 - src/H5MV.c | 3 - src/H5MVprivate.h | 2 +- src/H5Oainfo.c | 2 +- src/H5Oattr.c | 2 +- src/H5Oattribute.c | 10 +- src/H5Obogus.c | 4 +- src/H5Obtreek.c | 2 +- src/H5Ocache.c | 2 +- src/H5Ocache_image.c | 2 +- src/H5Ocont.c | 2 +- src/H5Ocopy.c | 2 +- src/H5Ocopy_ref.c | 200 +- src/H5Odrvinfo.c | 2 +- src/H5Odtype.c | 40 +- src/H5Oefl.c | 2 +- src/H5Ofill.c | 4 +- src/H5Ofsinfo.c | 4 +- src/H5Oginfo.c | 2 +- src/H5Oint.c | 27 +- src/H5Olayout.c | 2 +- src/H5Olinfo.c | 2 +- src/H5Olink.c | 2 +- src/H5Omessage.c | 8 +- src/H5Omtime.c | 4 +- src/H5Oname.c | 2 +- src/H5Onull.c | 2 +- src/H5Opkg.h | 2 +- src/H5Opline.c | 4 +- src/H5Oprivate.h | 2 +- src/H5Opublic.h | 26 +- src/H5Orefcount.c | 2 +- src/H5Osdspace.c | 2 +- src/H5Oshmesg.c | 2 +- src/H5Ostab.c | 2 +- src/H5Ounknown.c | 2 +- src/H5P.c | 11 +- src/H5PB.c | 32 +- src/H5PBpkg.h | 16 +- src/H5PBprivate.h | 4 +- src/H5PL.c | 14 +- src/H5PLint.c | 113 +- src/H5PLpath.c | 35 +- src/H5PLplugin_cache.c | 140 +- src/H5PLprivate.h | 32 +- src/H5PLpublic.h | 6 +- src/H5Pdxpl.c | 16 +- src/H5Pencdec.c | 2 +- src/H5Pfapl.c | 601 +- src/H5Pint.c | 369 +- src/H5Pmodule.h | 3 +- src/H5Pocpl.c | 14 +- src/H5Ppkg.h | 3 + src/H5Pprivate.h | 13 +- src/H5Ppublic.h | 366 +- src/H5RS.c | 5 +- src/H5Rint.c | 110 +- src/H5Rprivate.h | 2 + src/H5S.c | 85 +- src/H5SL.c | 82 +- src/H5SLprivate.h | 1 + src/H5SM.c | 7 +- src/H5Sall.c | 34 +- src/H5Shyper.c | 146 +- src/H5Smpio.c | 2 +- src/H5Snone.c | 34 +- src/H5Spkg.h | 20 +- src/H5Spoint.c | 38 +- src/H5Sprivate.h | 62 +- src/H5Sselect.c | 34 +- src/H5Stest.c | 12 +- src/H5T.c | 441 +- src/H5TS.c | 10 +- src/H5Tbit.c | 10 +- src/H5Tcommit.c | 2 +- src/H5Tconv.c | 48 +- src/H5Tfields.c | 2 +- src/H5Tnative.c | 6 +- src/H5Topaque.c | 2 +- src/H5Tpkg.h | 2 +- src/H5Tpublic.h | 8 +- src/H5VLcallback.c | 80 +- src/H5VLint.c | 201 +- src/H5VLnative.c | 6 +- src/H5VLnative_introspect.c | 2 +- src/H5VLnative_token.c | 4 +- src/H5VLpassthru.c | 15 +- src/H5VM.c | 2 +- src/H5WB.c | 2 +- src/H5Z.c | 160 +- src/H5Znbit.c | 31 +- src/H5Zscaleoffset.c | 12 +- src/H5Ztrans.c | 6 +- src/H5detect.c | 4 +- src/H5mpi.c | 235 +- src/H5private.h | 181 +- src/H5public.h | 65 +- src/H5retry_private.h | 2 +- src/H5system.c | 2 +- src/H5timer.c | 15 +- src/H5trace.c | 66 +- src/Makefile.am | 2 +- test/CMakeLists.txt | 142 +- test/CMakeTests.cmake | 15 +- test/CMakeVFDTests.cmake | 35 +- test/Makefile.am | 30 +- test/SWMR_POSIX_Order_UG.txt | 4 +- test/SWMR_UseCase_UG.txt | 6 +- test/ShellTests.cmake | 39 +- test/accum.c | 6 +- test/accum_swmr_reader.c | 2 +- test/app_ref.c | 28 +- test/atomic_writer.c | 2 +- test/big.c | 10 +- test/btree2.c | 140 +- test/cache.c | 71 +- test/cache_api.c | 57 +- test/cache_common.c | 162 +- test/cache_common.h | 4 +- test/cache_image.c | 33 +- test/cache_tagging.c | 52 +- test/chunk_info.c | 6 +- test/cmpd_dset.c | 14 +- test/cork.c | 23 +- test/dangle.c | 16 +- test/del_many_dense_attrs.c | 4 +- test/direct_chunk.c | 14 +- test/dsets.c | 81 +- test/dt_arith.c | 26 +- test/dtransform.c | 21 +- test/dtypes.c | 112 +- test/earray.c | 27 +- test/enc_dec_plist.c | 4 +- test/enc_dec_plist_cross_platform.c | 2 +- test/enum.c | 2 +- test/error_test.c | 20 +- test/event_set.c | 513 +- test/external.c | 10 +- test/external_common.c | 8 +- test/family_v16-000000.h5 | Bin 0 -> 5120 bytes test/family_v16-000001.h5 | Bin 0 -> 5120 bytes test/family_v16-000002.h5 | Bin 0 -> 5120 bytes test/family_v16-000003.h5 | Bin 0 -> 4048 bytes test/family_v16_00000.h5 | Bin 5120 -> 0 bytes test/family_v16_00001.h5 | Bin 5120 -> 0 bytes test/family_v16_00002.h5 | Bin 5120 -> 0 bytes test/family_v16_00003.h5 | Bin 4048 -> 0 bytes test/farray.c | 12 +- test/fheap.c | 156 +- test/file_image.c | 13 +- test/filenotclosed.c | 2 +- test/fillval.c | 9 +- test/filter_fail.c | 4 +- test/filter_plugin1_dsets.c | 2 +- test/filter_plugin3_dsets.c | 2 +- test/flush1.c | 4 +- test/flush2.c | 4 +- test/flushrefresh.c | 18 +- test/freespace.c | 28 +- test/gen_bounds.c | 12 +- test/gen_new_array.c | 4 +- test/gen_old_group.c | 2 +- test/gen_plist.c | 2 +- test/genall5.c | 2 +- test/getname.c | 10 +- test/gheap.c | 2 +- test/h5test.c | 402 +- test/h5test.h | 71 +- test/lheap.c | 40 +- test/links.c | 192 +- test/links_env.c | 16 +- test/mf.c | 245 +- test/mirror_vfd.c | 2 +- test/mount.c | 4 +- test/mtime.c | 103 +- test/ntypes.c | 16 +- test/null_vfd_plugin.c | 162 + test/null_vfd_plugin.h | 33 + test/objcopy.c | 53 +- test/ohdr.c | 32 +- test/page_buffer.c | 44 +- test/reserved.c | 8 +- test/ros3.c | 2 +- test/s3comms.c | 26 +- test/set_extent.c | 57 +- test/stab.c | 22 +- test/swmr.c | 58 +- test/swmr_addrem_writer.c | 2 +- test/swmr_check_compat_vfd.c | 53 - test/swmr_generator.c | 4 +- test/swmr_reader.c | 2 +- test/swmr_remove_reader.c | 2 +- test/swmr_remove_writer.c | 2 +- test/swmr_sparse_reader.c | 3 +- test/swmr_sparse_writer.c | 2 +- test/swmr_start_write.c | 6 +- test/swmr_writer.c | 2 +- test/tarray.c | 6 +- test/tattr.c | 273 +- test/tcoords.c | 2 +- test/test_usecases.sh.in | 31 +- test/testcheck_version.sh.in | 4 +- test/testflushrefresh.sh.in | 17 +- test/testhdf5.c | 2 +- test/testlibinfo.sh.in | 2 +- test/testmeta.c | 2 +- test/testswmr.pwsh.in | 519 ++ test/testswmr.sh.in | 62 +- test/testvdsswmr.pwsh.in | 190 + test/testvdsswmr.sh.in | 28 +- test/testvfdswmr.sh.in | 240 +- test/tfile.c | 936 +- test/tgenprop.c | 18 +- test/th5o.c | 119 +- test/th5s.c | 4 +- test/tid.c | 4 +- test/titerate.c | 6 +- test/tmisc.c | 104 +- test/trefer.c | 16 +- test/trefstr.c | 10 +- test/tselect.c | 109 +- test/tsohm.c | 54 +- test/ttsafe.c | 2 +- test/ttsafe_attr_vlen.c | 2 +- test/tvlstr.c | 10 +- test/tvltypes.c | 2 +- test/twriteorder.c | 8 +- test/unlink.c | 6 +- test/unregister.c | 2 +- test/use_append_chunk.c | 6 +- test/use_append_mchunks.c | 6 +- test/use_common.c | 4 +- test/use_disable_mdc_flushes.c | 2 +- test/vds.c | 33 +- test/vds_env.c | 24 +- test/vfd.c | 580 +- test/vfd_plugin.c | 439 + test/vfd_swmr.c | 44 +- test/vfd_swmr_attrdset_writer.c | 6 +- test/vfd_swmr_bigset_writer.c | 8 +- test/vfd_swmr_check_compat.c | 61 - test/vfd_swmr_dsetchks_writer.c | 13 +- test/vfd_swmr_dsetops_writer.c | 2 +- test/vfd_swmr_generator.c | 2 +- test/vfd_swmr_gfail_writer.c | 6 +- test/vfd_swmr_gperf_writer.c | 16 +- test/vfd_swmr_group_writer.c | 22 +- test/vfd_swmr_indep_rw_writer.c | 2 +- test/vfd_swmr_zoo_writer.c | 2 +- test/vol.c | 71 +- test/vol_plugin.c | 4 +- testpar/CMakeVFDTests.cmake | 18 +- testpar/t_2Gio.c | 339 +- testpar/t_bigio.c | 41 +- testpar/t_cache.c | 74 +- testpar/t_cache_image.c | 42 +- testpar/t_coll_chunk.c | 16 +- testpar/t_coll_md_read.c | 93 +- testpar/t_dset.c | 343 +- testpar/t_file.c | 20 +- testpar/t_filters_parallel.c | 9394 ++++++++++++-------- testpar/t_filters_parallel.h | 117 +- testpar/t_mdset.c | 6 +- testpar/t_mpi.c | 4 +- testpar/t_pflush1.c | 12 +- testpar/t_pflush2.c | 12 +- testpar/t_ph5basic.c | 2 +- testpar/t_pread.c | 10 +- testpar/t_prop.c | 6 +- testpar/t_shapesame.c | 64 +- testpar/t_span_tree.c | 30 +- testpar/testpflush.sh.in | 2 +- testpar/testphdf5.h | 8 +- tools/lib/h5diff.c | 2 +- tools/lib/h5diff.h | 2 +- tools/lib/h5diff_array.c | 100 +- tools/lib/h5diff_attr.c | 2 +- tools/lib/h5diff_util.c | 4 +- tools/lib/h5tools.c | 26 +- tools/lib/h5tools.h | 13 +- tools/lib/h5tools_dump.c | 69 +- tools/lib/h5tools_ref.c | 6 +- tools/lib/h5tools_str.c | 31 +- tools/lib/h5tools_str.h | 2 +- tools/lib/h5tools_utils.c | 6 +- tools/lib/h5tools_utils.h | 8 +- tools/lib/h5trav.c | 13 +- tools/lib/h5trav.h | 2 +- tools/lib/io_timer.c | 2 +- tools/libtest/h5tools_test_utils.c | 16 +- tools/src/h5copy/h5copy.c | 4 +- tools/src/h5diff/h5diff_common.c | 24 +- tools/src/h5diff/h5diff_common.h | 2 +- tools/src/h5diff/h5diff_main.c | 6 +- tools/src/h5diff/ph5diff_main.c | 7 +- tools/src/h5dump/h5dump.c | 16 +- tools/src/h5dump/h5dump_ddl.c | 16 +- tools/src/h5dump/h5dump_xml.c | 58 +- tools/src/h5format_convert/h5format_convert.c | 8 +- tools/src/h5import/h5import.c | 16 +- tools/src/h5jam/h5jam.c | 18 +- tools/src/h5jam/h5unjam.c | 6 +- tools/src/h5ls/h5ls.c | 60 +- tools/src/h5perf/perf.c | 4 +- tools/src/h5perf/pio_engine.c | 20 +- tools/src/h5perf/pio_perf.c | 16 +- tools/src/h5perf/sio_engine.c | 6 +- tools/src/h5perf/sio_perf.c | 34 +- tools/src/h5repack/h5repack.c | 92 +- tools/src/h5repack/h5repack.h | 8 +- tools/src/h5repack/h5repack_copy.c | 149 +- tools/src/h5repack/h5repack_filters.c | 4 +- tools/src/h5repack/h5repack_main.c | 50 +- tools/src/h5repack/h5repack_refs.c | 11 +- tools/src/h5repack/h5repack_verify.c | 10 +- tools/src/h5stat/h5stat.c | 8 +- tools/src/misc/h5clear.c | 6 +- tools/src/misc/h5debug.c | 8 +- tools/src/misc/h5delete.c | 2 +- tools/src/misc/h5mkgrp.c | 14 +- tools/test/h5copy/CMakeTests.cmake | 2 +- tools/test/h5copy/h5copygentest.c | 2 +- tools/test/h5copy/testh5copy.sh.in | 2 +- tools/test/h5diff/CMakeTests.cmake | 4 +- tools/test/h5diff/h5diff_plugin.sh.in | 6 +- tools/test/h5diff/h5diffgentest.c | 124 +- tools/test/h5diff/testfiles/h5diff_10.txt | 14 +- tools/test/h5diff/testfiles/h5diff_600.txt | 14 +- tools/test/h5diff/testfiles/h5diff_603.txt | 14 +- tools/test/h5diff/testfiles/h5diff_606.txt | 14 +- tools/test/h5diff/testfiles/h5diff_612.txt | 14 +- tools/test/h5diff/testfiles/h5diff_615.txt | 14 +- tools/test/h5diff/testfiles/h5diff_621.txt | 14 +- tools/test/h5diff/testfiles/h5diff_622.txt | 14 +- tools/test/h5diff/testfiles/h5diff_623.txt | 14 +- tools/test/h5diff/testfiles/h5diff_624.txt | 14 +- tools/test/h5diff/testh5diff.sh.in | 8 +- tools/test/h5dump/CMakeTests.cmake | 2 +- tools/test/h5dump/CMakeTestsPBITS.cmake | 2 +- tools/test/h5dump/CMakeVFDTests.cmake | 20 +- tools/test/h5dump/binread.c | 2 +- tools/test/h5dump/h5dumpgentest.c | 789 +- tools/test/h5dump/testh5dump.sh.in | 2 +- tools/test/h5dump/testh5dumppbits.sh.in | 2 +- tools/test/h5format_convert/h5fc_gentest.c | 2 +- tools/test/h5format_convert/testh5fc.sh.in | 2 +- tools/test/h5jam/getub.c | 8 +- tools/test/h5jam/tellub.c | 6 +- tools/test/h5jam/testh5jam.sh.in | 2 +- tools/test/h5ls/h5ls_plugin.sh.in | 2 +- tools/test/h5ls/testh5ls.sh.in | 4 +- tools/test/h5ls/testh5lsvds.sh.in | 4 +- tools/test/h5repack/CMakeTests.cmake | 2 +- tools/test/h5repack/CMakeVFDTests.cmake | 16 +- tools/test/h5repack/dynlib_rpk.c | 2 +- tools/test/h5repack/dynlib_vrpk.c | 2 +- tools/test/h5repack/h5repack.sh.in | 6 +- tools/test/h5repack/h5repackgentest.c | 6 +- tools/test/h5repack/h5repacktst.c | 230 +- tools/test/h5repack/testfiles/README | 2 +- tools/test/h5repack/testfiles/h5repack-help.txt | 15 +- .../h5repack_layout.h5-plugin_version_test.ddl | 14 +- tools/test/h5stat/testh5stat.sh.in | 2 +- tools/test/misc/h5clear_gentest.c | 32 +- tools/test/misc/h5perf_gentest.c | 6 +- tools/test/misc/testfiles/h5mkgrp_help.txt | 6 + tools/test/perform/chunk_cache.c | 22 +- tools/test/perform/direct_write_perf.c | 4 +- tools/test/perform/overhead.c | 4 +- tools/test/perform/perf_meta.c | 14 +- tools/test/perform/pio_standalone.h | 3 +- tools/test/perform/sio_standalone.h | 3 +- tools/test/perform/zip_perf.c | 12 +- tools/testfiles/h5dump-help.txt | 6 + tools/testfiles/help-1.ls | 6 + tools/testfiles/help-2.ls | 6 + tools/testfiles/help-3.ls | 6 + .../pbits/tnofilename-with-packed-bits.ddl | 6 + tools/testfiles/pbits/tpbitsIncomplete.ddl | 6 + tools/testfiles/pbits/tpbitsLengthExceeded.ddl | 6 + tools/testfiles/pbits/tpbitsLengthPositive.ddl | 6 + tools/testfiles/pbits/tpbitsMaxExceeded.ddl | 6 + tools/testfiles/pbits/tpbitsOffsetExceeded.ddl | 6 + tools/testfiles/pbits/tpbitsOffsetNegative.ddl | 6 + tools/testfiles/textlinksrc-nodangle-1.ls | 6 + tools/testfiles/tgroup-1.ls | 6 + utils/CMakeLists.txt | 19 +- utils/Makefile.am | 26 +- utils/mirror_vfd/mirror_remote.c | 2 +- utils/mirror_vfd/mirror_remote.h | 2 +- utils/mirror_vfd/mirror_server.c | 29 +- utils/mirror_vfd/mirror_server_stop.c | 4 +- utils/mirror_vfd/mirror_writer.c | 8 +- utils/test/CMakeLists.txt | 35 + utils/test/Makefile.am | 34 + utils/test/swmr_check_compat_vfd.c | 53 + utils/tools/CMakeLists.txt | 12 + utils/tools/Makefile.am | 38 + utils/tools/h5dwalk/CMakeLists.txt | 66 + utils/tools/h5dwalk/Makefile.am | 37 + utils/tools/h5dwalk/h5dwalk.1 | 42 + utils/tools/h5dwalk/h5dwalk.c | 1714 ++++ utils/tools/test/CMakeLists.txt | 8 + utils/tools/test/Makefile.am | 32 + utils/tools/test/h5dwalk/CMakeLists.txt | 15 + utils/tools/test/h5dwalk/CMakeTests.cmake | 56 + utils/tools/test/h5dwalk/Makefile.am | 43 + utils/tools/test/h5dwalk/copy_demo_files.sh.in | 86 + utils/tools/test/h5dwalk/help.h5dwalk | 13 + utils/tools/test/h5dwalk/testh5dwalk.sh.in | 249 + utils/vfd_swmr/README.md | 2 +- 781 files changed, 32987 insertions(+), 16717 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/codespell.yml delete mode 100644 .gitmodules create mode 100644 README.md delete mode 100644 README.txt create mode 100644 config/cmake/README.md.cmake.in delete mode 100644 config/cmake/README.txt.cmake.in create mode 100644 doxygen/dox/Glossary.dox create mode 100644 doxygen/img/HDF5.png create mode 100644 examples/ph5_filtered_writes.c create mode 100644 examples/ph5_filtered_writes_no_sel.c delete mode 100644 hl/tools/h5watch/swmr_check_compat_vfd.c delete mode 100644 java/lib/ext/slf4j-nop-1.7.25.jar create mode 100644 java/lib/ext/slf4j-nop-1.7.33.jar delete mode 100644 java/lib/ext/slf4j-simple-1.7.25.jar create mode 100644 java/lib/ext/slf4j-simple-1.7.33.jar delete mode 100644 java/lib/slf4j-api-1.7.25.jar create mode 100644 java/lib/slf4j-api-1.7.33.jar create mode 100644 java/test/TestH5OcopyOld.java create mode 100644 java/test/testfiles/JUnit-TestH5OcopyOld.txt create mode 100644 release_docs/HISTORY-1_13.txt create mode 100644 src/H5FDperform.c create mode 100644 test/family_v16-000000.h5 create mode 100644 test/family_v16-000001.h5 create mode 100644 test/family_v16-000002.h5 create mode 100644 test/family_v16-000003.h5 delete mode 100644 test/family_v16_00000.h5 delete mode 100644 test/family_v16_00001.h5 delete mode 100644 test/family_v16_00002.h5 delete mode 100644 test/family_v16_00003.h5 create mode 100644 test/null_vfd_plugin.c create mode 100644 test/null_vfd_plugin.h delete mode 100644 test/swmr_check_compat_vfd.c create mode 100644 test/testswmr.pwsh.in create mode 100644 test/testvdsswmr.pwsh.in create mode 100644 test/vfd_plugin.c delete mode 100644 test/vfd_swmr_check_compat.c create mode 100644 utils/test/CMakeLists.txt create mode 100644 utils/test/Makefile.am create mode 100644 utils/test/swmr_check_compat_vfd.c create mode 100644 utils/tools/CMakeLists.txt create mode 100644 utils/tools/Makefile.am create mode 100644 utils/tools/h5dwalk/CMakeLists.txt create mode 100644 utils/tools/h5dwalk/Makefile.am create mode 100644 utils/tools/h5dwalk/h5dwalk.1 create mode 100644 utils/tools/h5dwalk/h5dwalk.c create mode 100644 utils/tools/test/CMakeLists.txt create mode 100644 utils/tools/test/Makefile.am create mode 100644 utils/tools/test/h5dwalk/CMakeLists.txt create mode 100644 utils/tools/test/h5dwalk/CMakeTests.cmake create mode 100644 utils/tools/test/h5dwalk/Makefile.am create mode 100644 utils/tools/test/h5dwalk/copy_demo_files.sh.in create mode 100644 utils/tools/test/h5dwalk/help.h5dwalk create mode 100644 utils/tools/test/h5dwalk/testh5dwalk.sh.in diff --git a/.gitattributes b/.gitattributes index e01442a..2ff0dab 100644 --- a/.gitattributes +++ b/.gitattributes @@ -192,12 +192,12 @@ java/examples/testfiles/examples.intro.H5_CreateGroup.txt -text java/examples/testfiles/examples.intro.H5_CreateGroupAbsoluteRelative.txt -text java/examples/testfiles/examples.intro.H5_CreateGroupDataset.txt -text java/examples/testfiles/examples.intro.H5_ReadWrite.txt -text -java/lib/ext/slf4j-nop-1.7.25.jar -text svneol=unset#application/zip -java/lib/ext/slf4j-simple-1.7.25.jar -text svneol=unset#application/zip +java/lib/ext/slf4j-nop-1.7.33.jar -text svneol=unset#application/zip +java/lib/ext/slf4j-simple-1.7.33.jar -text svneol=unset#application/zip java/lib/hamcrest-core.jar -text svneol=unset#application/java-archive java/lib/junit.jar -text svneol=unset#application/java-archive java/lib/simplelogger.properties -text -java/lib/slf4j-api-1.7.25.jar -text svneol=unset#application/zip +java/lib/slf4j-api-1.7.33.jar -text svneol=unset#application/zip java/src/CMakeLists.txt -text java/src/Makefile.am -text java/src/hdf/CMakeLists.txt -text @@ -393,10 +393,10 @@ test/cork.c -text test/corrupt_stab_msg.h5 -text test/deflate.h5 -text test/dynlib4.c -text -test/family_v16_00000.h5 -text -test/family_v16_00001.h5 -text -test/family_v16_00002.h5 -text -test/family_v16_00003.h5 -text +test/family_v16-000000.h5 -text +test/family_v16-000001.h5 -text +test/family_v16-000002.h5 -text +test/family_v16-000003.h5 -text test/file_image_core_test.h5 -text test/filespace_1_6.h5 -text test/filespace_1_8.h5 -text diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 29e5866..99daf78 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,36 +7,36 @@ # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners # will be requested to review. -*.cmake @byrnHDF @derobins -CMakeLists.txt @byrnHDF @derobins -CMakeTests.* @byrnHDF @derobins +*.cmake @byrnHDF @ChristopherHogan @gnuoyd @derobins +CMakeLists.txt @byrnHDF @ChristopherHogan @gnuoyd @derobins +CMakeTests.* @byrnHDF @ChristopherHogan @gnuoyd @derobins -/bin/ @lrknox @derobins @qkoziol +/bin/ @lrknox @ChristopherHogan @gnuoyd @derobins @qkoziol -/c++/ @bmribler @byrnHDF @derobins +/c++/ @bmribler @byrnHDF @ChristopherHogan @gnuoyd @derobins -/config/ @lrknox @derobins @qkoziol @byrnHDF +/config/ @lrknox @byrnHDF @ChristopherHogan @gnuoyd @derobins @qkoziol -/doc/ @gnuoyd @jrmainzer +/doc/ @ChristopherHogan @gnuoyd @jrmainzer -/examples/ @lrknox @derobins @bmribler +/examples/ @lrknox @bmribler @ChristopherHogan @gnuoyd @derobins /fortran/ @brtnfld @epourmal -/hl/ @bmribler @byrnHDF @derobins +/hl/ @bmribler @byrnHDF @ChristopherHogan @gnuoyd @derobins /java/ @jhendersonHDF @byrnHDF -/m4/ @lrknox @derobins +/m4/ @lrknox @ChristopherHogan @gnuoyd @derobins /release_docs/ @lrknox @bmribler @byrnHDF -/src/ @jhendersonHDF @derobins @fortnern @qkoziol @soumagne @vchoi-hdfgroup @jrmainzer +/src/ @jhendersonHDF @fortnern @soumagne @vchoi-hdfgroup @ChristopherHogan @gnuoyd @derobins @jrmainzer @qkoziol -/test/ @jhendersonHDF @derobins @fortnern @qkoziol @soumagne @vchoi-hdfgroup @jrmainzer +/test/ @jhendersonHDF @fortnern @soumagne @vchoi-hdfgroup @ChristopherHogan @gnuoyd @derobins @jrmainzer @qkoziol -/testpar/ @jhendersonHDF @rawarren @jrmainzer @qkoziol +/testpar/ @jhendersonHDF @ChristopherHogan @gnuoyd @jrmainzer @qkoziol -/tools/ @byrnHDF @bmribler @derobins +/tools/ @byrnHDF @bmribler @ChristopherHogan @gnuoyd @derobins -/utils/ @lrknox @byrnHDF @derobins +/utils/ @lrknox @byrnHDF @ChristopherHogan @gnuoyd @derobins diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..7c4daaf --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +custom: "https://hdfgroup.org/about-us/donate-to-the-hdf-group/" diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 0000000..163353e --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,15 @@ +# GitHub Action to automate the identification of common misspellings in text files +# https://github.com/codespell-project/codespell +# https://github.com/codespell-project/actions-codespell +name: codespell +on: [push, pull_request] +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: codespell-project/actions-codespell@master + with: + skip: ./bin/trace,./hl/tools/h5watch/h5watch.c,./tools/test/h5jam/tellub.c + ignore_words_list: isnt,inout,nd,parms,parm,ba,offsetP,ser,ois,had,fiter,fo,clude,refere,minnum,offsetp,creat,ans:,eiter,lastr,ans,isn't,ifset,sur,trun,dne,tthe,hda,filname,te,htmp,minnum diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/CMakeFilters.cmake b/CMakeFilters.cmake index 51ac61c..725390b 100644 --- a/CMakeFilters.cmake +++ b/CMakeFilters.cmake @@ -19,7 +19,7 @@ include (FetchContent) set (HDF5_ALLOW_EXTERNAL_SUPPORT "NO" CACHE STRING "Allow External Library Building (NO GIT TGZ)") set_property (CACHE HDF5_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO GIT TGZ) if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ") - option (ZLIB_USE_EXTERNAL "Use External Library Building for ZLIB" 1) + option (ZLIB_USE_EXTERNAL "Use External Library Building for HDF5_ZLIB" 1) option (SZIP_USE_EXTERNAL "Use External Library Building for SZIP" 1) if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT") set (ZLIB_URL ${ZLIB_GIT_URL} CACHE STRING "Path to zlib git repository") @@ -82,7 +82,7 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT) set (H5_HAVE_ZLIB_H 1) set (H5_HAVE_LIBZ 1) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "Filter ZLIB is built") + message (VERBOSE "Filter HDF5_ZLIB is built") endif () else () message (FATAL_ERROR " ZLib is Required for ZLib support in HDF5") @@ -100,7 +100,7 @@ if (HDF5_ENABLE_Z_LIB_SUPPORT) set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${ZLIB_STATIC_LIBRARY}) INCLUDE_DIRECTORIES (${ZLIB_INCLUDE_DIRS}) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "Filter ZLIB is ON") + message (VERBOSE "Filter HDF5_ZLIB is ON") endif () endif () @@ -149,9 +149,9 @@ if (HDF5_ENABLE_SZIP_SUPPORT) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "... with library AEC") endif () - set (SZ_PACKAGE_NAME ${LIBAEC_PACKAGE_NAME}) + set (SZIP_PACKAGE_NAME ${LIBAEC_PACKAGE_NAME}) else () - set (SZ_PACKAGE_NAME ${SZIP_PACKAGE_NAME}) + set (SZIP_PACKAGE_NAME ${SZIP_PACKAGE_NAME}) endif () else () message (FATAL_ERROR "SZIP is Required for SZIP support in HDF5") diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index b506f05..b5efebf 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -182,7 +182,7 @@ if (HDF5_PACK_EXAMPLES) endif () #----------------------------------------------------------------------------- -# Configure the README.txt file for the binary package +# Configure the README.md file for the binary package #----------------------------------------------------------------------------- HDF_README_PROPERTIES(HDF5_BUILD_FORTRAN) @@ -395,13 +395,19 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) set (CPACK_PACKAGING_INSTALL_PREFIX "/${CPACK_PACKAGE_INSTALL_DIRECTORY}") set (CPACK_COMPONENTS_ALL_IN_ONE_PACKAGE ON) + list (APPEND CPACK_GENERATOR "DEB") set (CPACK_DEBIAN_PACKAGE_SECTION "Libraries") set (CPACK_DEBIAN_PACKAGE_MAINTAINER "${HDF5_PACKAGE_BUGREPORT}") -# list (APPEND CPACK_GENERATOR "RPM") + list (APPEND CPACK_GENERATOR "RPM") set (CPACK_RPM_PACKAGE_RELEASE "1") + set (CPACK_RPM_PACKAGE_RELEASE_DIST ON) set (CPACK_RPM_COMPONENT_INSTALL ON) set (CPACK_RPM_PACKAGE_RELOCATABLE ON) + set (CPACK_RPM_FILE_NAME "RPM-DEFAULT") + set (CPACK_RPM_PACKAGE_NAME "${CPACK_PACKAGE_NAME}") + set (CPACK_RPM_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION}") + set (CPACK_RPM_PACKAGE_VENDOR "${CPACK_PACKAGE_VENDOR}") set (CPACK_RPM_PACKAGE_LICENSE "BSD-style") set (CPACK_RPM_PACKAGE_GROUP "Development/Libraries") set (CPACK_RPM_PACKAGE_URL "${HDF5_PACKAGE_URL}") @@ -442,11 +448,11 @@ The HDF5 data model, file format, API, library, and tools are open and distribut if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ") if (ZLIB_FOUND AND ZLIB_USE_EXTERNAL) if (WIN32) - set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};ZLIB;ALL;/") + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};HDF5_ZLIB;ALL;/") else () - set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};ZLIB;libraries;/") - set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};ZLIB;headers;/") - set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};ZLIB;configinstall;/") + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};HDF5_ZLIB;libraries;/") + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};HDF5_ZLIB;headers;/") + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${ZLIB_INCLUDE_DIR_GEN};HDF5_ZLIB;configinstall;/") endif () endif () if (SZIP_FOUND AND SZIP_USE_EXTERNAL) diff --git a/CMakeLists.txt b/CMakeLists.txt index 11644b0..aaa068c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,9 +229,9 @@ set (HDF5_JAVA_JNI_SRC_DIR ${HDF5_SOURCE_DIR}/java/src/jni) set (HDF5_JAVA_HDF5_SRC_DIR ${HDF5_SOURCE_DIR}/java/src/hdf) set (HDF5_JAVA_TEST_SRC_DIR ${HDF5_SOURCE_DIR}/java/test) set (HDF5_JAVA_LIB_DIR ${HDF5_SOURCE_DIR}/java/lib) -set (HDF5_JAVA_LOGGING_JAR ${HDF5_SOURCE_DIR}/java/lib/slf4j-api-1.7.25.jar) -set (HDF5_JAVA_LOGGING_NOP_JAR ${HDF5_SOURCE_DIR}/java/lib/ext/slf4j-nop-1.7.25.jar) -set (HDF5_JAVA_LOGGING_SIMPLE_JAR ${HDF5_SOURCE_DIR}/java/lib/ext/slf4j-simple-1.7.25.jar) +set (HDF5_JAVA_LOGGING_JAR ${HDF5_SOURCE_DIR}/java/lib/slf4j-api-1.7.33.jar) +set (HDF5_JAVA_LOGGING_NOP_JAR ${HDF5_SOURCE_DIR}/java/lib/ext/slf4j-nop-1.7.33.jar) +set (HDF5_JAVA_LOGGING_SIMPLE_JAR ${HDF5_SOURCE_DIR}/java/lib/ext/slf4j-simple-1.7.33.jar) set (HDF5_DOXYGEN_DIR ${HDF5_SOURCE_DIR}/doxygen) #----------------------------------------------------------------------------- @@ -708,10 +708,14 @@ if (HDF5_ENABLE_PARALLEL) # Used by Parallel Compression feature set (PARALLEL_FILTERED_WRITES ON) - CHECK_SYMBOL_EXISTS (MPI_Mprobe "mpi.h" H5_HAVE_MPI_Mprobe) - CHECK_SYMBOL_EXISTS (MPI_Imrecv "mpi.h" H5_HAVE_MPI_Imrecv) - if (NOT H5_HAVE_MPI_Mprobe OR NOT H5_HAVE_MPI_Imrecv) - message (WARNING "The MPI_Mprobe and/or MPI_Imrecv functions could not be located. + CHECK_SYMBOL_EXISTS (MPI_Ibarrier "mpi.h" H5_HAVE_MPI_Ibarrier) + CHECK_SYMBOL_EXISTS (MPI_Issend "mpi.h" H5_HAVE_MPI_Issend) + CHECK_SYMBOL_EXISTS (MPI_Iprobe "mpi.h" H5_HAVE_MPI_Iprobe) + CHECK_SYMBOL_EXISTS (MPI_Irecv "mpi.h" H5_HAVE_MPI_Irecv) + if (H5_HAVE_MPI_Ibarrier AND H5_HAVE_MPI_Issend AND H5_HAVE_MPI_Iprobe AND H5_HAVE_MPI_Irecv) + set (H5_HAVE_PARALLEL_FILTERED_WRITES 1) + else () + message (WARNING "The MPI_Ibarrier/MPI_Issend/MPI_Iprobe/MPI_Irecv functions could not be located. Parallel writes of filtered data will be disabled.") set (PARALLEL_FILTERED_WRITES OFF) endif () @@ -900,10 +904,10 @@ add_subdirectory (src) if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ") if (ZLIB_FOUND AND ZLIB_USE_EXTERNAL) if (NOT ONLY_SHARED_LIBS) - add_dependencies (${HDF5_LIB_TARGET} ZLIB) + add_dependencies (${HDF5_LIB_TARGET} HDF5_ZLIB) endif () if (BUILD_SHARED_LIBS) - add_dependencies (${HDF5_LIBSH_TARGET} ZLIB) + add_dependencies (${HDF5_LIBSH_TARGET} HDF5_ZLIB) endif () endif () if (SZIP_FOUND AND SZIP_USE_EXTERNAL) @@ -951,6 +955,9 @@ if (BUILD_TESTING) if (HDF5_TEST_VFD) option (HDF5_TEST_FHEAP_VFD "Execute tests with different VFDs" ON) mark_as_advanced (HDF5_TEST_FHEAP_VFD) + + # Initialize the list of VFDs to be used for testing and create a test folder for each VFD + H5_SET_VFD_LIST() endif () option (HDF5_TEST_PASSTHROUGH_VOL "Execute tests with different passthrough VOL connectors" OFF) @@ -1009,10 +1016,32 @@ endif () # Option to build HDF5 Utilities #----------------------------------------------------------------------------- if (EXISTS "${HDF5_SOURCE_DIR}/utils" AND IS_DIRECTORY "${HDF5_SOURCE_DIR}/utils") - option (HDF5_BUILD_UTILS "Build HDF5 Utils" ON) - if (HDF5_BUILD_UTILS) - add_subdirectory (utils) + option (HDF5_BUILD_PARALLEL_TOOLS "Build Parallel HDF5 Tools" OFF) + if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL) + set (CMAKE_PREFIX_PATH "$HDF_RESOURCES_DIR") + find_package(MFU REQUIRED) + if (MFU_FOUND) + message(STATUS "LL_PATH=${LL_PATH}") + set (H5_HAVE_LIBMFU 1) + set (H5_HAVE_MFU_H 1) + set (CMAKE_REQUIRED_INCLUDES "${MFU_INCLUDE_DIR}") + set (MFU_LIBRARY_DEBUG "$MFU_LIBRARY") + set (MFU_LIBRARY_RELEASE "$MFU_LIBRARY") + endif () + find_package(CIRCLE REQUIRED) + if (CIRCLE_FOUND) + set (H5_HAVE_LIBCIRCLE 1) + set (H5_HAVE_CIRCLE_H 1) + set (CMAKE_REQUIRED_INCLUDES "${CIRCLE_INCLUDE_DIR}") + endif () + find_package(DTCMP REQUIRED) + if (DTCMP_FOUND) + set (H5_HAVE_LIBDTCMP 1) + set (H5_HAVE_DTCMP_H 1) + set (CMAKE_REQUIRED_INCLUDES "${DTCMP_INCLUDE_DIR}") + endif () endif () + add_subdirectory (utils) endif () #----------------------------------------------------------------------------- diff --git a/MANIFEST b/MANIFEST index 5fffcfc..a31e52c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -20,7 +20,6 @@ ./.gitattributes _DO_NOT_DISTRIBUTE_ ./.gitignore _DO_NOT_DISTRIBUTE_ -./.gitmodules _DO_NOT_DISTRIBUTE_ ./.autom4te.cfg _DO_NOT_DISTRIBUTE_ ./.h5chkright.ini _DO_NOT_DISTRIBUTE_ ./ACKNOWLEDGMENTS @@ -30,7 +29,7 @@ ./MANIFEST ./Makefile.dist ./Makefile.am -./README.txt +./README.md ./acsite.m4 ./autogen.sh ./configure.ac @@ -41,6 +40,8 @@ ./.github/workflows/clang-format-check.yml _DO_NOT_DISTRIBUTE_ ./.github/workflows/main.yml _DO_NOT_DISTRIBUTE_ ./.github/workflows/pr-check.yml _DO_NOT_DISTRIBUTE_ +./.github/workflows/codespell.yml _DO_NOT_DISTRIBUTE_ +./.github/FUNDING.yml _DO_NOT_DISTRIBUTE_ ./m4/aclocal_fc.m4 ./m4/aclocal_fc.f90 @@ -231,6 +232,7 @@ ./doxygen/dox/DDLBNF112.dox ./doxygen/dox/FTS.dox ./doxygen/dox/GettingStarted.dox +./doxygen/dox/Glossary.dox ./doxygen/dox/H5AC_cache_config_t.dox ./doxygen/dox/MetadataCachingInHDF5.dox ./doxygen/dox/Overview.dox @@ -297,6 +299,7 @@ ./doxygen/img/FF-IH_FileGroup.gif ./doxygen/img/FF-IH_FileObject.gif ./doxygen/img/FileFormatSpecChunkDiagram.jpg +./doxygen/img/HDF5.png ./doxygen/img/HDFG-logo.png ./doxygen/img/IOFlow.gif ./doxygen/img/IOFlow2.gif @@ -340,6 +343,8 @@ ./examples/h5_ref2reg_deprec.c ./examples/h5_shared_mesg.c ./examples/ph5example.c +./examples/ph5_filtered_writes.c +./examples/ph5_filtered_writes_no_sel.c ./examples/h5_vds.c ./examples/h5_vds-exc.c ./examples/h5_vds-exclim.c @@ -627,6 +632,7 @@ ./release_docs/HISTORY-1_8.txt ./release_docs/HISTORY-1_10.txt ./release_docs/HISTORY-1_12.txt +./release_docs/HISTORY-1_13.txt ./release_docs/INSTALL ./release_docs/INSTALL_CMake.txt ./release_docs/INSTALL_Cygwin.txt @@ -832,6 +838,7 @@ ./src/H5FDmpio.h ./src/H5FDmulti.c ./src/H5FDmulti.h +./src/H5FDperform.c ./src/H5FDros3.c ./src/H5FDros3.h ./src/H5FDpkg.h @@ -1231,10 +1238,10 @@ ./test/external_common.h ./test/external_env.c ./test/external_fname.h -./test/family_v16_00000.h5 -./test/family_v16_00001.h5 -./test/family_v16_00002.h5 -./test/family_v16_00003.h5 +./test/family_v16-000000.h5 +./test/family_v16-000001.h5 +./test/family_v16-000002.h5 +./test/family_v16-000003.h5 ./test/farray.c ./test/fheap.c ./test/file_image.c @@ -1319,6 +1326,8 @@ ./test/ntypes.c ./test/null_vol_connector.c ./test/null_vol_connector.h +./test/null_vfd_plugin.c +./test/null_vfd_plugin.h ./test/objcopy.c ./test/objcopy_ref.c ./test/ohdr.c @@ -1339,7 +1348,6 @@ ./test/supervise.subr ./test/swmr.c ./test/swmr_addrem_writer.c -./test/swmr_check_compat_vfd.c ./test/swmr_common.c ./test/swmr_common.h ./test/swmr_generator.c @@ -1376,8 +1384,10 @@ ./test/testlinks_env.sh.in ./test/testmeta.c ./test/test_mirror.sh.in +./test/testswmr.pwsh.in ./test/testswmr.sh.in ./test/testvds_env.sh.in +./test/testvdsswmr.pwsh.in ./test/testvdsswmr.sh.in ./test/testvfdswmr.sh.in ./test/tfile.c @@ -1435,7 +1445,6 @@ ./test/vfd_swmr_dsetops_writer.c ./test/vfd_swmr_dsetchks_writer.c ./test/vfd_swmr_bigset_writer.c -./test/vfd_swmr_check_compat.c ./test/vfd_swmr_common.c ./test/vfd_swmr_common.h ./test/vfd_swmr_generator.c @@ -1453,6 +1462,7 @@ ./test/vfd_swmr_writer.c ./test/vfd_swmr_zoo_writer.c ./test/vfd.c +./test/vfd_plugin.c ./test/vol.c ./test/vol_plugin.c @@ -3049,6 +3059,26 @@ ./utils/mirror_vfd/mirror_server_stop.c ./utils/mirror_vfd/mirror_writer.c +# test utilities +./utils/test/Makefile.am +./utils/test/swmr_check_compat_vfd.c + +# parallel tools (h5dwalk) and tests +./utils/tools/CMakeLists.txt +./utils/tools/Makefile.am +./utils/tools/h5dwalk/CMakeLists.txt +./utils/tools/h5dwalk/Makefile.am +./utils/tools/h5dwalk/h5dwalk.1 +./utils/tools/h5dwalk/h5dwalk.c +./utils/tools/test/CMakeLists.txt +./utils/tools/test/Makefile.am +./utils/tools/test/h5dwalk/CMakeLists.txt +./utils/tools/test/h5dwalk/CMakeTests.cmake +./utils/tools/test/h5dwalk/Makefile.am +./utils/tools/test/h5dwalk/copy_demo_files.sh.in +./utils/tools/test/h5dwalk/help.h5dwalk +./utils/tools/test/h5dwalk/testh5dwalk.sh.in + # VFD SWMR utilities ./utils/vfd_swmr/README.md ./utils/vfd_swmr/Makefile.am @@ -3168,7 +3198,6 @@ ./hl/tools/h5watch/extend_dset.c ./hl/tools/h5watch/h5watch.c ./hl/tools/h5watch/h5watchgentest.c -./hl/tools/h5watch/swmr_check_compat_vfd.c ./hl/tools/h5watch/testh5watch.sh.in # expected test output from testing h5watch @@ -3554,6 +3583,7 @@ ./java/test/testfiles/JUnit-TestH5Obasic.txt ./java/test/testfiles/JUnit-TestH5Ocreate.txt ./java/test/testfiles/JUnit-TestH5Ocopy.txt +./java/test/testfiles/JUnit-TestH5OcopyOld.txt ./java/test/testfiles/JUnit-TestH5P.txt ./java/test/testfiles/JUnit-TestH5PData.txt ./java/test/testfiles/JUnit-TestH5Pfapl.txt @@ -3597,6 +3627,7 @@ ./java/test/TestH5Obasic.java ./java/test/TestH5Ocreate.java ./java/test/TestH5Ocopy.java +./java/test/TestH5OcopyOld.java ./java/test/TestH5P.java ./java/test/TestH5PData.java ./java/test/TestH5Pfapl.java @@ -3619,9 +3650,9 @@ ./java/lib/hamcrest-core.jar ./java/lib/junit.jar ./java/lib/simplelogger.properties -./java/lib/slf4j-api-1.7.25.jar -./java/lib/ext/slf4j-nop-1.7.25.jar -./java/lib/ext/slf4j-simple-1.7.25.jar +./java/lib/slf4j-api-1.7.33.jar +./java/lib/ext/slf4j-nop-1.7.33.jar +./java/lib/ext/slf4j-simple-1.7.33.jar # CMake-specific Files ./config/toolchain/build32.cmake @@ -3657,7 +3688,7 @@ ./config/cmake/mccacheinit.cmake ./config/cmake/patch.xml ./config/cmake/PkgInfo.in -./config/cmake/README.txt.cmake.in +./config/cmake/README.md.cmake.in ./config/cmake/UseJava.cmake ./config/cmake/UseJavaClassFilelist.cmake ./config/cmake/UseJavaSymlinks.cmake @@ -3796,8 +3827,10 @@ ./tools/test/misc/vds/CMakeLists.txt ./tools/test/perform/CMakeLists.txt ./tools/test/perform/CMakeTests.cmake + ./utils/CMakeLists.txt ./utils/mirror_vfd/CMakeLists.txt +./utils/test/CMakeLists.txt ./utils/vfd_swmr/CMakeLists.txt # CMake-specific User Scripts @@ -3913,4 +3946,9 @@ ./tools/test/perform/Makefile.in ./utils/Makefile.in ./utils/mirror_vfd/Makefile.in +./utils/test/Makefile.in +./utils/tools/Makefile.in +./utils/tools/h5dwalk/Makefile.in +./utils/tools/test/Makefile.in +./utils/tools/test/h5dwalk/Makefile.in ./utils/vfd_swmr/Makefile.in diff --git a/README.md b/README.md new file mode 100644 index 0000000..7e0c281 --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ +HDF5 version 1.13.2-1 currently under development + +![HDF5 Logo](doxygen/img/HDF5.png) + +*Please refer to the release_docs/INSTALL file for installation instructions.* + +THE HDF GROUP +--------------- + +The HDF Group is the developer of HDF5®, a high-performance software library and +data format that has been adopted across multiple industries and has become a +de facto standard in scientific and research communities. + +More information about The HDF Group, the HDF5 Community and the HDF5 software +project, tools and services can be found at the Group's website. + + https://www.hdfgroup.org/ + + +DOCUMENTATION +------------- +This release is fully functional for the API described in the documentation. + + https://portal.hdfgroup.org/display/HDF5/The+HDF5+API + +Full Documentation and Programming Resources for this release can be found at + + https://portal.hdfgroup.org/display/HDF5 + +See the RELEASE.txt file in the release_docs/ directory for information specific +to the features and updates included in this release of the library. + +Several more files are located within the release_docs/ directory with specific +details for several common platforms and configurations. + + INSTALL - Start Here. General instructions for compiling and installing the library + INSTALL_CMAKE - instructions for building with CMake (Kitware.com) + INSTALL_parallel - instructions for building and configuring Parallel HDF5 + INSTALL_Windows and INSTALL_Cygwin - MS Windows installations. + + + +HELP AND SUPPORT +---------------- +Information regarding Help Desk and Support services is available at + + https://portal.hdfgroup.org/display/support/The+HDF+Help+Desk + + + +FORUM and NEWS +-------------- +The following public forums are provided for public announcements and discussions +of interest to the general HDF5 Community. + + - Homepage of the Forum + https://forum.hdfgroup.org + + - News and Announcement + https://forum.hdfgroup.org/c/news-and-announcements-from-the-hdf-group + + - HDF5 and HDF4 Topics + https://forum.hdfgroup.org/c/hdf5 + +These forums are provided as an open and public service for searching and reading. +Posting requires completing a simple registration and allows one to join in the +conversation. Please read the following instructions pertaining to the Forum's +use and configuration + https://forum.hdfgroup.org/t/quickstart-guide-welcome-to-the-new-hdf-forum + + +SNAPSHOTS, PREVIOUS RELEASES AND SOURCE CODE +-------------------------------------------- +Periodically development code snapshots are provided at the following URL: + + https://gamma.hdfgroup.org/ftp/pub/outgoing/hdf5/snapshots/ + +Source packages for current and previous releases are located at: + + https://portal.hdfgroup.org/display/support/Downloads + +Development code is available at our Github location: + + https://github.com/HDFGroup/hdf5.git + diff --git a/README.txt b/README.txt deleted file mode 100644 index b9cfe8a..0000000 --- a/README.txt +++ /dev/null @@ -1,80 +0,0 @@ -HDF5 version 1.13.1-1 currently under development - ------------------------------------------------------------------------------- -Please refer to the release_docs/INSTALL file for installation instructions. ------------------------------------------------------------------------------- - -THE HDF GROUP ---------------- - -The HDF Group is the developer of HDF5®, a high-performance software library and -data format that has been adopted across multiple industries and has become a -de facto standard in scientific and research communities. - -More information about The HDF Group, the HDF5 Community and the HDF5 software -project, tools and services can be found at the Group's website. - - https://www.hdfgroup.org/ - - -DOCUMENTATION -------------- -This release is fully functional for the API described in the documentation. - https://portal.hdfgroup.org/display/HDF5/The+HDF5+API - -Full Documentation and Programming Resources for this release can be found at - https://portal.hdfgroup.org/display/HDF5 - -See the RELEASE.txt file in the release_docs/ directory for information specific -to the features and updates included in this release of the library. - -Several more files are located within the release_docs/ directory with specific -details for several common platforms and configurations. - - INSTALL - Start Here. General instructions for compiling and installing the library - INSTALL_CMAKE - instructions for building with CMake (Kitware.com) - INSTALL_parallel - instructions for building and configuring Parallel HDF5 - INSTALL_Windows and INSTALL_Cygwin - MS Windows installations. - - - -HELP AND SUPPORT ----------------- -Information regarding Help Desk and Support services is available at - - https://portal.hdfgroup.org/display/support/The+HDF+Help+Desk - - - -FORUM and NEWS --------------- -The following public forums are provided for public announcements and discussions -of interest to the general HDF5 Community. - - Homepage of the Forum - https://forum.hdfgroup.org - - News and Announcement - https://forum.hdfgroup.org/c/news-and-announcements-from-the-hdf-group - - HDF5 and HDF4 Topics - https://forum.hdfgroup.org/c/hdf5 - -These forums are provided as an open and public service for searching and reading. -Posting requires completing a simple registration and allows one to join in the -conversation. Please read the following instructions pertaining to the Forum's -use and configuration - https://forum.hdfgroup.org/t/quickstart-guide-welcome-to-the-new-hdf-forum - - -SNAPSHOTS, PREVIOUS RELEASES AND SOURCE CODE --------------------------------------------- -Periodically development code snapshots are provided at the following URL: - https://gamma.hdfgroup.org/ftp/pub/outgoing/hdf5/snapshots/ - -Source packages for current and previous releases are located at: - https://portal.hdfgroup.org/display/support/Downloads - -Development code is available at our Github location: - https://github.com/HDFGroup/hdf5.git - diff --git a/bin/bbrelease b/bin/bbrelease index 388b9e4..a45be2e 100755 --- a/bin/bbrelease +++ b/bin/bbrelease @@ -48,7 +48,7 @@ USAGE() { cat << EOF Usage: $0 -d [--docver BRANCHNAME] [-h] [--nocheck] [--private] ... - -d DIR The name of the directory where the releas(es) should be + -d DIR The name of the directory where the release(s) should be placed. --docver BRANCHNAME This is added for 1.8 and beyond to get the correct version of documentation files from the hdf5docs @@ -103,7 +103,7 @@ EOF # Modifications # # Steps: -# 1. untar the tarball in a temporay directory; +# 1. untar the tarball in a temporary directory; # Note: do this in a temporary directory to avoid changing # the original source directory which maybe around. # 2. convert all its text files to DOS (LF-CR) style; @@ -246,7 +246,7 @@ if [ "X$methods" = "X" ]; then methods="tar" fi -# Create the temporay work directory. +# Create the temporary work directory. if mkdir $tmpdir; then echo "temporary work directory for release. "\ "Can be deleted after release completes." > $tmpdir/README @@ -320,9 +320,9 @@ ln -s `pwd` $tmpdir/$HDF5_IN_VERS || exit 1 test -f Makefile && mv Makefile $tmpdir/Makefile.x cp -p Makefile.dist Makefile -# Update README.txt and release_docs/RELEASE.txt with release information in +# Update README.md and release_docs/RELEASE.txt with release information in # line 1. -for f in README.txt release_docs/RELEASE.txt; do +for f in README.md release_docs/RELEASE.txt; do echo "HDF5 version $VERS released on $release_date" >$f.x sed -e 1d $f >>$f.x mv $f.x $f diff --git a/bin/buildhdf5 b/bin/buildhdf5 index 113a278..786c35d 100755 --- a/bin/buildhdf5 +++ b/bin/buildhdf5 @@ -178,7 +178,7 @@ LOCATE_SZLIB() esac ;; # end of case ncsa unknown) - # Unknow domain. Give a shot at the some standard places. + # Unknown domain. Give a shot at the some standard places. szlibpaths="/usr/local" ;; esac # end of case $mydomain diff --git a/bin/cmakehdf5 b/bin/cmakehdf5 index da1aef8..e59c772 100755 --- a/bin/cmakehdf5 +++ b/bin/cmakehdf5 @@ -199,7 +199,7 @@ DUMP_LOGFILE() # Show a start time stamp TIMESTAMP -# Initialize njobs if $AMKE is defined +# Initialize njobs if $MAKE is defined if [ -n "$MAKE" ]; then # assume all arguments are for --jobs njobs=`echo $MAKE | cut -s -d' ' -f2-` @@ -365,7 +365,7 @@ STEP "Test the library and tools..." "ctest . -C Release $njobs" $testlog # 7. Create an install image with this command: STEP "Create an install image..." "cpack -C Release CPackConfig.cmake" $packlog -# The implementation of installation is imcomplete (only works for linux). +# The implementation of installation is incomplete (only works for linux). # Screen it out for now till it is completed. if false; then # 8. Install with this command: diff --git a/bin/genparser b/bin/genparser index 462c2c2..9ea3152 100755 --- a/bin/genparser +++ b/bin/genparser @@ -27,7 +27,7 @@ # IMPORTANT OS X NOTE # # If you are using OS X, you will probably not have flex or bison -# installed. In addtion, even if you do have bison installed, the bison +# installed. In addition, even if you do have bison installed, the bison # version you have installed may also have a bug that makes it unable to # process our input files. # diff --git a/bin/h5cc.in b/bin/h5cc.in index 6024b4d..6835b79 100644 --- a/bin/h5cc.in +++ b/bin/h5cc.in @@ -40,7 +40,7 @@ HL="@HL@" ## $CLINKER $H5BLD_CPPFLAGS $CPPFLAGS $H5BLD_CFLAGS $CFLAGS $LDFLAGS ## ## $LIBS $clibpath $link_objs $link_args $shared_link ## ## ## -## These settings can be overriden by setting HDF5_CFLAGS, ## +## These settings can be overridden by setting HDF5_CFLAGS, ## ## HDF5_CPPFLAGS, HDF5_LDFLAGS, or HDF5_LIBS in the environment. ## ## ## ############################################################################ @@ -144,7 +144,7 @@ usage() { echo " shared libraries]" echo " " echo " You can also add or change paths and flags to the compile line using" - echo " the following environment varibles or by assigning them to their counterparts" + echo " the following environment variables or by assigning them to their counterparts" echo " in the 'Things You Can Modify to Override...'" section of $prog_name echo " " echo " Variable Current value to be replaced" @@ -318,7 +318,7 @@ fi if test "x$do_link" = "xyes"; then shared_link="" -# conditionnaly link with the hl library +# conditionally link with the hl library if test "X$HL" = "Xhl"; then libraries=" $libraries -lhdf5_hl -lhdf5 " else diff --git a/bin/h5vers b/bin/h5vers index 8f75df1..04d5d03 100755 --- a/bin/h5vers +++ b/bin/h5vers @@ -66,7 +66,7 @@ use strict; # ./H5public.h or ./src/H5public.h. # # If the version number is changed (either `-s' or `-i' was used on -# the command line) then the first line of the README.txt and RELEASE.txt files +# the command line) then the version line of the README.md and RELEASE.txt files # one directory above the H5public.h file is also modified so it looks # something like: This is hdf5-1.2.3-pre1 currently under development. # The AC_INIT macro in configure.ac will also change in this case to be @@ -156,10 +156,10 @@ while ($_ = shift) { } die "mutually exclusive options given\n" if $set && $inc; -# Determine file to use as H5public.h, README.txt, +# Determine file to use as H5public.h, README.md, # release_docs/RELEASE.txt, configure.ac, windows/src/H5pubconf.h # config/lt_vers.am and config/cmake/scripts/HDF5config.cmake. -# The README.txt, release_docs/RELEASE.txt, configure.ac, +# The README.md, release_docs/RELEASE.txt, configure.ac, # windows/src/H5pubconf.h, config/lt_vers.am and # config/cmake/scripts/HDF5config.cmake # files are always in the directory above H5public.h @@ -178,9 +178,9 @@ die "unable to read file: $LT_VERS\n" unless -r $file; my $HDF5CONFIGCMAKE = $file; $HDF5CONFIGCMAKE =~ s/[^\/]*$/..\/config\/cmake\/scripts\/HDF5config.cmake/; die "unable to read file: $HDF5CONFIGCMAKE\n" unless -r $file; -# README.txt +# README.md my $README = $file; -$README =~ s/[^\/]*$/..\/README.txt/; +$README =~ s/[^\/]*$/..\/README.md/; die "unable to read file: $README\n" unless -r $file; # release_docs/RELEASE.txt my $RELEASE = $file; @@ -303,7 +303,7 @@ if ($LT_VERS && $version_increased) { # close FILE; } -# Update the README.txt file +# Update the README.md file if ($README) { open FILE, $README or die "$README: $!\n"; my @contents = ; diff --git a/bin/newer b/bin/newer index e60ec45..5f0fdca 100755 --- a/bin/newer +++ b/bin/newer @@ -19,7 +19,7 @@ # Created Date: 2005/07/06 # Modification: # Albert Cheng 2005/8/30 -# Changed from two arguments to mulitple arguments. +# Changed from two arguments to multiple arguments. if test $# -lt 2; then exit 1 diff --git a/bin/release b/bin/release index 8774851..e71a38e 100755 --- a/bin/release +++ b/bin/release @@ -39,7 +39,7 @@ USAGE() { cat << EOF Usage: $0 -d [--docver BRANCHNAME] [-h] [--nocheck] [--private] ... - -d DIR The name of the directory where the releas(es) should be + -d DIR The name of the directory where the release(es) should be placed. --docver BRANCHNAME This is added for 1.8 and beyond to get the correct version of documentation files from the hdf5docs @@ -60,7 +60,7 @@ for compressing the resulting tar archive (if none are given then cmake-tgz -- create a tar file using the gzip default level with a build-unix.sh command file and all other CMake files needed to build HDF5 source using CMake on unix machines. - cmake-zip -- convert all text files to DOS style and create a zip file inluding cmake + cmake-zip -- convert all text files to DOS style and create a zip file including cmake scripts and .bat files to build HDF5 source using CMake on Windows. hpc-cmake-tgz -- create a tar file using the gzip default level with a build-unix.sh @@ -512,7 +512,7 @@ if [ "X$methods" = "X" ]; then methods="tar" fi -# Create the temporay work directory. +# Create the temporary work directory. if mkdir $tmpdir; then echo "temporary work directory for release. "\ "Can be deleted after release completes." > $tmpdir/README @@ -572,9 +572,9 @@ ln -s `pwd` $tmpdir/$HDF5_VERS || exit 1 test -f Makefile && mv Makefile $tmpdir/Makefile.x cp -p Makefile.dist Makefile -# Update README.txt and release_docs/RELEASE.txt with release information in +# Update README.md and release_docs/RELEASE.txt with release information in # line 1. -for f in README.txt release_docs/RELEASE.txt; do +for f in README.md release_docs/RELEASE.txt; do echo "HDF5 version $VERS released on $release_date" >$f.x sed -e 1d $f >>$f.x mv $f.x $f diff --git a/bin/runtest b/bin/runtest index 50acb76..5e05abb 100755 --- a/bin/runtest +++ b/bin/runtest @@ -630,7 +630,7 @@ while [ $# -gt 0 ]; do # setup the directory structure for snapshot test. CMD=setup ;; - -*) # Unknow option + -*) # Unknown option PRINT "Unknown option ($1)" USAGE exit 1 @@ -830,7 +830,7 @@ if [ -z "$NOCVS" ]; then else # make sure the cvs update, if done by another host, has completed. # First wait for the presence of $CVSLOG which signals some host - # has started the cvs update. Then wait for the absense of $CVSLOG_LOCK + # has started the cvs update. Then wait for the absence of $CVSLOG_LOCK # which signals the host has completed the cvs update. WAITFOR $CVSLOG 90 if [ $WAIT_STATUS -ne 0 ]; then @@ -911,7 +911,7 @@ if [ -n "$TESTHOST" -a $HOSTNAME != "$TESTHOST" ]; then PRINT $h is not reachable "(`date`)" ;; *) - PRINT "CHECK_RSH for $h returned unknow result ($RSH)" + PRINT "CHECK_RSH for $h returned unknown result ($RSH)" ;; esac >> $TMP_OUTPUT 2>&1 done diff --git a/bin/snapshot b/bin/snapshot index 20c73cd..7fcf3ab 100755 --- a/bin/snapshot +++ b/bin/snapshot @@ -156,7 +156,7 @@ Usage: $PROGNAME [all] [checkout] [ftp [diff] [test] [srcdir] [release] [h "snapshot srcdir" is equivalent to "snapshot srcdir all" "snapshot srcdir checkout" is equivalent to "snapshot checkout" srcdirname : - Use as the srcdir testing directory if srcdir is choosen. + Use as the srcdir testing directory if srcdir is chosen. If starts with '-', it is append to the default name E.g., "snapshot srcdir srcdirname -xx" uses hostname-xx [Default is hostname] @@ -422,7 +422,7 @@ while [ $# -gt 0 ] ; do fi cmd="" EXEC_CMD_ARG="$@" - # exit the parsing while loop since all arguments have been consummed. + # exit the parsing while loop since all arguments have been consumed. break ;; check-vfd) @@ -455,7 +455,7 @@ while [ $# -gt 0 ] ; do OP_CONFIGURE="$OP_CONFIGURE $1" ;; *) - echo "Unkown option $1" + echo "Unknown option $1" errcode=1 cmd="help" break @@ -623,7 +623,7 @@ if [ "$cmd" = "all" -o -n "$cmdtest" -o -n "$cmddiff" ]; then fi fi - # if diff is choosen, exit 0 if no significant differences are found. + # if diff is chosen, exit 0 if no significant differences are found. # otherwise, exit 1. This includes cases of other failures. if [ -n "$cmddiff" ]; then if [ $snapshot = no ]; then @@ -799,7 +799,7 @@ if [ "$cmd" = "all" -o -n "$cmdrel" ]; then # Replace the previous version with the current version. # Should check if the errcode of the release process but there # are other failures after release was done (e.g. h5vers or git failures) - # that should allow the replacement to occure. + # that should allow the replacement to occur. rm -rf ${PREVIOUS} mv ${CURRENT} ${PREVIOUS} fi #Release snapshot diff --git a/bin/trace b/bin/trace index 7afc3e7..169719c 100755 --- a/bin/trace +++ b/bin/trace @@ -381,9 +381,9 @@ sub rewrite_func ($$$$$) { # Wrap lines that will be longer than the limit if ($len_if_added > $max_trace_macro_line_len) { - # Wrap line, with indention + # Wrap line, with indentation $trace .= ",\n "; - $len = 13; # Set to 13, for indention + $len = 13; # Set to 13, for indentation # Indent an extra space to account for extra digit in 'H5TRACE' macro if (scalar(@arg_str) >= 10) { diff --git a/bin/warnhist b/bin/warnhist index 742a6b5..0150138 100755 --- a/bin/warnhist +++ b/bin/warnhist @@ -78,7 +78,7 @@ sub do_help { print "\t-S \tDisplay warnings for files which contain a string, \n"; print "\t\t is a comma separated list, with no spaces\n"; print "\t\tFor example: 'H5Fint' or 'H5Fint,H5Gnode'\n"; - print "\t-l\tDisplay line nunbers for file/warning\n"; + print "\t-l\tDisplay line numbers for file/warning\n"; print "\t-u\tLeave 'unique' types in warnings, instead of genericizing them\n"; print "\t-i \tIgnore named files, \n"; print "\t\t is a comma separated list, with no spaces\n"; diff --git a/c++/src/H5AbstractDs.cpp b/c++/src/H5AbstractDs.cpp index eeb0155..70d8531 100644 --- a/c++/src/H5AbstractDs.cpp +++ b/c++/src/H5AbstractDs.cpp @@ -302,7 +302,7 @@ AbstractDs::getStrType() const //-------------------------------------------------------------------------- // Function: AbstractDs::getVarLenType -///\brief Returns the floating-point datatype of this abstract dataset, +///\brief Returns the variable length datatype of this abstract dataset, /// which can be a dataset or an attribute. ///\return VarLenType instance ///\exception H5::DataTypeIException diff --git a/c++/src/H5Attribute.cpp b/c++/src/H5Attribute.cpp index 954222c..520a4f6 100644 --- a/c++/src/H5Attribute.cpp +++ b/c++/src/H5Attribute.cpp @@ -562,7 +562,7 @@ Attribute::p_read_variable_len(const DataType &mem_type, H5std_string &strg) con ///\exception H5::IdComponentException when the attempt to close the HDF5 /// object fails // Description: -// The underlaying reference counting in the C library ensures +// The underlying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. // Programmer Binh-Minh Ribler - 2000 diff --git a/c++/src/H5DataSet.cpp b/c++/src/H5DataSet.cpp index ce56e30..627d81b 100644 --- a/c++/src/H5DataSet.cpp +++ b/c++/src/H5DataSet.cpp @@ -769,7 +769,7 @@ DataSet::p_read_variable_len(const hid_t mem_type_id, const hid_t mem_space_id, ///\exception H5::IdComponentException when the attempt to close the HDF5 /// object fails // Description: -// The underlaying reference counting in the C library ensures +// The underlying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. //-------------------------------------------------------------------------- diff --git a/c++/src/H5DataSpace.cpp b/c++/src/H5DataSpace.cpp index 342e9fa..e4aef83 100644 --- a/c++/src/H5DataSpace.cpp +++ b/c++/src/H5DataSpace.cpp @@ -629,7 +629,7 @@ DataSpace::getId() const ///\exception H5::IdComponentException when the attempt to close the HDF5 /// object fails // Description: -// The underlaying reference counting in the C library ensures +// The underlying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. // Programmer Binh-Minh Ribler - 2000 diff --git a/c++/src/H5DataType.cpp b/c++/src/H5DataType.cpp index af58a90..ff8f6dc 100644 --- a/c++/src/H5DataType.cpp +++ b/c++/src/H5DataType.cpp @@ -927,7 +927,7 @@ DataType::p_opentype(const H5Location &loc, const char *dtype_name) const ///\exception H5::IdComponentException when the attempt to close the HDF5 /// object fails // Description: -// The underlaying reference counting in the C library ensures +// The underlying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. // Programmer Binh-Minh Ribler - 2000 @@ -983,7 +983,7 @@ DataType::close() // - Replaced decRefCount with close() to let the C library // handle the reference counting - BMR, Jun 1, 2006 // - Added the use of H5CPP_EXITED to terminate the HDF5 library -// and elimiate previous memory leaks. See comments in the +// and eliminate previous memory leaks. See comments in the // header file "H5PredType.h" for details. - BMR, Mar 30, 2012 // - Major re-implementation of the global constants was done // to avoid relying on the order of the creation and deletion diff --git a/c++/src/H5DcreatProp.cpp b/c++/src/H5DcreatProp.cpp index a9270fd..da19d8d 100644 --- a/c++/src/H5DcreatProp.cpp +++ b/c++/src/H5DcreatProp.cpp @@ -552,7 +552,7 @@ DSetCreatPropList::setShuffle() const ///\exception H5::PropListIException ///\par Description /// The values of space allocation time can be one of the -/// followings: +/// following: /// \li \c H5D_ALLOC_TIME_DEFAULT /// \li \c H5D_ALLOC_TIME_EARLY /// \li \c H5D_ALLOC_TIME_LATE diff --git a/c++/src/H5File.cpp b/c++/src/H5File.cpp index f92171b..92a8d59 100644 --- a/c++/src/H5File.cpp +++ b/c++/src/H5File.cpp @@ -665,7 +665,7 @@ H5File::getLocId() const ///\exception H5::IdComponentException when the attempt to close the HDF5 /// object fails // Description -// The underlaying reference counting in the C library ensures +// The underlying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. // December 2000 diff --git a/c++/src/H5Group.cpp b/c++/src/H5Group.cpp index fe791da..b017f4f 100644 --- a/c++/src/H5Group.cpp +++ b/c++/src/H5Group.cpp @@ -208,7 +208,7 @@ Group::getId() const ///\exception H5::IdComponentException when the attempt to close the HDF5 /// object fails // Description: -// The underlaying reference counting in the C library ensures +// The underlying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. // Programmer Binh-Minh Ribler - 2000 diff --git a/c++/src/H5Location.cpp b/c++/src/H5Location.cpp index eb8dd4c..13a89aa 100644 --- a/c++/src/H5Location.cpp +++ b/c++/src/H5Location.cpp @@ -1745,7 +1745,7 @@ H5Location::getObjinfo(const H5std_string &name, hbool_t follow_link, H5G_stat_t // Function: H5Location::getObjinfo ///\brief This is an overloaded member function, provided for convenience. /// It differs from the above functions in that it doesn't have -/// the paramemter \a follow_link. +/// the parameter \a follow_link. // Nov, 2005 //-------------------------------------------------------------------------- void diff --git a/c++/src/H5Location.h b/c++/src/H5Location.h index 740d0ce..0aec4d2 100644 --- a/c++/src/H5Location.h +++ b/c++/src/H5Location.h @@ -15,7 +15,7 @@ #ifndef H5Location_H #define H5Location_H -#include "H5Classes.h" // constains forward class declarations +#include "H5Classes.h" // contains forward class declarations namespace H5 { @@ -334,7 +334,7 @@ class H5_DLLCPP H5Location : public IdComponent { #endif // DOXYGEN_SHOULD_SKIP_THIS // Noop destructor. - virtual ~H5Location(); + virtual ~H5Location() override; }; // end of H5Location } // namespace H5 diff --git a/c++/src/H5PredType.cpp b/c++/src/H5PredType.cpp index c407c33..9458fb7 100644 --- a/c++/src/H5PredType.cpp +++ b/c++/src/H5PredType.cpp @@ -790,7 +790,7 @@ September 2015: classes, such as PropList, PredType, DataSpace, etc... Previously, these global constants were declared statically and the C++ library used a constant, called PredType::AtExit, to detect when all the global - contants are destroyed then close the C library (H5close). This method + constants are destroyed then close the C library (H5close). This method relied on the order of the constants being created and destroyed and that PredType constants be the last to be destroyed. In September 2015, it was recognized that the order in which the global constants were diff --git a/c++/src/H5PropList.cpp b/c++/src/H5PropList.cpp index 7ee8395..e7a83af 100644 --- a/c++/src/H5PropList.cpp +++ b/c++/src/H5PropList.cpp @@ -301,7 +301,7 @@ PropList::getId() const ///\exception H5::IdComponentException when the attempt to close the HDF5 /// object fails // Description: -// The underlaying reference counting in the C library ensures +// The underlying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. // Programmer Binh-Minh Ribler - 2000 diff --git a/c++/src/cpp_doc_config b/c++/src/cpp_doc_config index 3eb7645..1e26187 100644 --- a/c++/src/cpp_doc_config +++ b/c++/src/cpp_doc_config @@ -38,7 +38,7 @@ PROJECT_NAME = # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "1.13.1-1, currently under development" +PROJECT_NUMBER = "1.13.2-1, currently under development" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/c++/test/CMakeVFDTests.cmake b/c++/test/CMakeVFDTests.cmake index 360d14e..ced9b0c 100644 --- a/c++/test/CMakeVFDTests.cmake +++ b/c++/test/CMakeVFDTests.cmake @@ -15,20 +15,7 @@ ### T E S T I N G ### ############################################################################## ############################################################################## - -set (VFD_LIST - sec2 - stdio - core - core_paged - split - multi - family -) - -if (H5_HAVE_DIRECT) - set (VFD_LIST ${VFD_LIST} direct) -endif () +H5_CREATE_VFD_DIR() ############################################################################## ############################################################################## @@ -38,7 +25,6 @@ endif () macro (ADD_VFD_TEST vfdname resultcode) if (NOT HDF5_ENABLE_USING_MEMCHECKER) - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdname}") add_test ( NAME CPP_VFD-${vfdname}-cpp_testhdf5-clear-objects COMMAND ${CMAKE_COMMAND} -E remove diff --git a/c++/test/tattr.cpp b/c++/test/tattr.cpp index 26699d2..dc968f9 100644 --- a/c++/test/tattr.cpp +++ b/c++/test/tattr.cpp @@ -95,7 +95,7 @@ struct attr4_struct { const H5std_string ATTR5_NAME("Attr5"); const int ATTR5_RANK = 0; -float attr_data5 = -5.123f; // Test data for 5th attribute +float attr_data5 = -5.123F; // Test data for 5th attribute /* Info for another attribute */ const H5std_string ATTR1A_NAME("Attr1_a"); @@ -1782,7 +1782,7 @@ test_attr_dense_create(FileCreatPropList &fcpl, FileAccPropList &fapl) unsigned attr_num; for (attr_num = 0; attr_num < max_compact; attr_num++) { // Create attribute - sprintf(attr_name, "attr %02u", attr_num); + snprintf(attr_name, sizeof(attr_name), "attr %02u", attr_num); Attribute attr = dataset.createAttribute(attr_name, PredType::NATIVE_UINT, ds_space); // Write data to the attribute @@ -1794,7 +1794,7 @@ test_attr_dense_create(FileCreatPropList &fcpl, FileAccPropList &fapl) { // Add one more attribute, to push into "dense" storage // Create another attribute - sprintf(attr_name, "attr %02u", attr_num); + snprintf(attr_name, sizeof(attr_name), "attr %02u", attr_num); Attribute attr = dataset.createAttribute(attr_name, PredType::NATIVE_UINT, ds_space); // Write data to the attribute @@ -1804,7 +1804,7 @@ test_attr_dense_create(FileCreatPropList &fcpl, FileAccPropList &fapl) // Attempt to add attribute again, which should fail try { // Create another attribute - sprintf(attr_name, "attr %02u", attr_num); + snprintf(attr_name, sizeof(attr_name), "attr %02u", attr_num); Attribute attr = dataset.createAttribute(attr_name, PredType::NATIVE_UINT, ds_space); // continuation here, that means no exception has been thrown diff --git a/c++/test/testhdf5.cpp b/c++/test/testhdf5.cpp index 71428ce..af50646 100644 --- a/c++/test/testhdf5.cpp +++ b/c++/test/testhdf5.cpp @@ -114,7 +114,7 @@ main(int argc, char *argv[]) TestSummary(); /* Clean up test files, if allowed */ - if (GetTestCleanup() && !getenv("HDF5_NOCLEANUP")) + if (GetTestCleanup() && !getenv(HDF5_NOCLEANUP)) TestCleanup(); /* Release test infrastructure */ diff --git a/c++/test/th5s.cpp b/c++/test/th5s.cpp index 5808136..461d258 100644 --- a/c++/test/th5s.cpp +++ b/c++/test/th5s.cpp @@ -73,7 +73,7 @@ struct space4_struct { unsigned u; float f; char c2; -} space4_data = {'v', 987123, -3.14f, 'g'}; /* Test data for 4th dataspace */ +} space4_data = {'v', 987123, -3.14F, 'g'}; /* Test data for 4th dataspace */ /* Null dataspace */ int space5_data = 7; diff --git a/c++/test/titerate.cpp b/c++/test/titerate.cpp index b6a9436..57761ea 100644 --- a/c++/test/titerate.cpp +++ b/c++/test/titerate.cpp @@ -160,7 +160,7 @@ test_iter_group(FileAccPropList &fapl) DataSpace filespace; for (int i = 0; i < NDATASETS; i++) { - sprintf(name, "Dataset %d", i); + snprintf(name, sizeof(name), "Dataset %d", i); // Create a dataset in the file DataSet dataset = file.createDataSet(name, datatype, filespace); @@ -351,7 +351,7 @@ test_iter_group(FileAccPropList &fapl) * Purpose Open an attribute and verify that it has a the correct name *------------------------------------------------------------------------- */ -const H5std_string FILE_NAME("titerate.h5"); +const H5std_string FILE_NAME("test_member_access.h5"); const H5std_string GRP_NAME("/Group_A"); const H5std_string FDATASET_NAME("file dset"); const H5std_string GDATASET_NAME("group dset"); @@ -396,6 +396,9 @@ test_HDFFV_9920() int attr_data[2] = {100, 200}; hsize_t dims[1] = {DIM1}; + /* Output message about test being performed */ + SUBTEST("Member access"); + try { // Create a new file and a group in it H5File file(FILE_NAME, H5F_ACC_TRUNC); @@ -425,6 +428,7 @@ test_HDFFV_9920() printelems(file, FDATASET_NAME, FATTR_NAME); printelems(gr1, GDATASET_NAME, GATTR_NAME); + PASSED(); } // end of try block // Catch all failures for handling in the same way @@ -473,4 +477,5 @@ extern "C" void cleanup_iterate() { HDremove(FILE_ITERATE.c_str()); + HDremove(FILE_NAME.c_str()); } // cleanup_iterate diff --git a/c++/test/tobject.cpp b/c++/test/tobject.cpp index 2b694da..f040007 100644 --- a/c++/test/tobject.cpp +++ b/c++/test/tobject.cpp @@ -740,7 +740,7 @@ test_object() MESSAGE(5, ("Testing Object Functions\n")); test_get_objname(); // Test get object name from groups/datasets - test_existance(); // Test check for object existance + test_existance(); // Test check for object existence test_get_objname_ontypes(); // Test get object name from types test_get_objtype(); // Test get object type test_open_object_header(); // Test object header functions (H5O) diff --git a/c++/test/ttypes.cpp b/c++/test/ttypes.cpp index 69c93d1..8afb4bb 100644 --- a/c++/test/ttypes.cpp +++ b/c++/test/ttypes.cpp @@ -28,7 +28,7 @@ using namespace H5; #include "h5cpputil.h" // C++ utilility header file /* - * Offset from alinged memory returned by malloc(). This can be used to test + * Offset from aligned memory returned by malloc(). This can be used to test * that type conversions handle non-aligned buffers correctly. */ #define ALIGNMENT 1 diff --git a/c++/test/tvlstr.cpp b/c++/test/tvlstr.cpp index c91b566..405ca07 100644 --- a/c++/test/tvlstr.cpp +++ b/c++/test/tvlstr.cpp @@ -917,32 +917,32 @@ test_vl_rewrite() int i; char name[256]; // Buffer for names & data for (i = 0; i < REWRITE_NDATASETS; i++) { - sprintf(name, "/set_%d", i); + snprintf(name, sizeof(name), "/set_%d", i); write_scalar_dset(file1, type, space, name, name); } // Effectively copy data from file 1 to 2. for (i = 0; i < REWRITE_NDATASETS; i++) { - sprintf(name, "/set_%d", i); + snprintf(name, sizeof(name), "/set_%d", i); read_scalar_dset(file1, type, space, name, name); write_scalar_dset(file2, type, space, name, name); } // Read back from file 2. for (i = 0; i < REWRITE_NDATASETS; i++) { - sprintf(name, "/set_%d", i); + snprintf(name, sizeof(name), "/set_%d", i); read_scalar_dset(file2, type, space, name, name); } // Remove from file 2. for (i = 0; i < REWRITE_NDATASETS; i++) { - sprintf(name, "/set_%d", i); + snprintf(name, sizeof(name), "/set_%d", i); file2.unlink(name); } // Effectively copy from file 1 to file 2. for (i = 0; i < REWRITE_NDATASETS; i++) { - sprintf(name, "/set_%d", i); + snprintf(name, sizeof(name), "/set_%d", i); read_scalar_dset(file1, type, space, name, name); write_scalar_dset(file2, type, space, name, name); } diff --git a/config/BlankForm b/config/BlankForm index ce4bf53..14131b2 100644 --- a/config/BlankForm +++ b/config/BlankForm @@ -61,7 +61,7 @@ fi # # PROD_CFLAGS Flags to pass to the compiler to create a # PROD_CPPFLAGS production version of the library. These -# usualy exclude symbolic debugging switches +# usually exclude symbolic debugging switches # (like `-g') and include optimization switches # (like `-O'). # @@ -84,7 +84,7 @@ fi # compiler or a descendent of gcc such as EGCS or PGCC. # # AM_CFLAGS Flags added directly into this variable will -# be propogated to the compiler wrapper scripts (h5cc, +# be propagated to the compiler wrapper scripts (h5cc, # h5c++, et cetera) in addition to being used to compile # the library. # diff --git a/config/cmake/CTestCustom.cmake b/config/cmake/CTestCustom.cmake index f958804..9517e4b 100644 --- a/config/cmake/CTestCustom.cmake +++ b/config/cmake/CTestCustom.cmake @@ -19,6 +19,8 @@ set (CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 50000) set (CTEST_CUSTOM_WARNING_EXCEPTION ${CTEST_CUSTOM_WARNING_EXCEPTION} "note.*expected.*void.*but argument is of type.*volatile" + "plugin-build.*:[ \t]*warning" + "CMake Warning*stamp" "src.ZLIB.*:[ \t]*warning" "warning LNK4197:.*ZLIB-prefix" "src.SZIP.*:[ \t]*warning" diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 483e551..43aa243 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -30,7 +30,7 @@ MARK_AS_ADVANCED (HDF5_STRICT_FORMAT_CHECKS) # ---------------------------------------------------------------------- # Decide whether the data accuracy has higher priority during data -# conversions. If not, some hard conversions will still be prefered even +# conversions. If not, some hard conversions will still be preferred even # though the data may be wrong (for example, some compilers don't # support denormalized floating values) to maximize speed. #----------------------------------------------------------------------------- diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index ae5a8c6..1eaeabe 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -265,6 +265,9 @@ /* Define if we have parallel support */ #cmakedefine H5_HAVE_PARALLEL @H5_HAVE_PARALLEL@ +/* Define if we have support for writing to filtered datasets in parallel */ +#cmakedefine H5_HAVE_PARALLEL_FILTERED_WRITES @H5_HAVE_PARALLEL_FILTERED_WRITES@ + /* Define if both pread and pwrite exist. */ #cmakedefine H5_HAVE_PREADWRITE @H5_HAVE_PREADWRITE@ @@ -669,7 +672,7 @@ /* Version number of package */ #define H5_VERSION "@HDF5_PACKAGE_VERSION_STRING@" -/* Data accuracy is prefered to speed during data conversions */ +/* Data accuracy is preferred to speed during data conversions */ #cmakedefine H5_WANT_DATA_ACCURACY @H5_WANT_DATA_ACCURACY@ /* Check exception handling functions during data conversions */ diff --git a/config/cmake/HDF5Macros.cmake b/config/cmake/HDF5Macros.cmake index 090524a..8b8b334 100644 --- a/config/cmake/HDF5Macros.cmake +++ b/config/cmake/HDF5Macros.cmake @@ -58,5 +58,46 @@ macro (H5_SET_LIB_OPTIONS libtarget libname libtype libpackage) endif () endif () endif () +endmacro () + +# Initialize the list of VFDs to be used for testing and create a test folder for each VFD +macro (H5_SET_VFD_LIST) + set (VFD_LIST + sec2 + stdio + core + core_paged + split + multi + family + splitter + #log - log VFD currently has file space allocation bugs + ) + + if (H5_HAVE_DIRECT) + set (VFD_LIST ${VFD_LIST} direct) + endif () + if (H5_HAVE_PARALLEL) + # MPI I/O VFD is currently incompatible with too many tests in the VFD test set + # set (VFD_LIST ${VFD_LIST} mpio) + endif () + if (H5_HAVE_MIRROR_VFD) + set (VFD_LIST ${VFD_LIST} mirror) + endif () + if (H5_HAVE_ROS3_VFD) + set (VFD_LIST ${VFD_LIST} ros3) + endif () + if (H5_HAVE_LIBHDFS) + set (VFD_LIST ${VFD_LIST} hdfs) + endif () + if (H5_HAVE_WINDOWS) + set (VFD_LIST ${VFD_LIST} windows) + endif () +endmacro () +# Initialize the list of VFDs to be used for testing and create a test folder for each VFD +macro (H5_CREATE_VFD_DIR) + foreach (vfdtest ${VFD_LIST}) + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdtest}") + endforeach () endmacro () diff --git a/config/cmake/HDF5PluginMacros.cmake b/config/cmake/HDF5PluginMacros.cmake index 4e05399..dbed15c 100644 --- a/config/cmake/HDF5PluginMacros.cmake +++ b/config/cmake/HDF5PluginMacros.cmake @@ -57,8 +57,8 @@ macro (EXTERNAL_PLUGIN_LIBRARY compress_type) add_dependencies (h5ex_d_mafisc ${HDF5_LIBSH_TARGET}) target_include_directories (h5ex_d_mafisc PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR}") endif () - if (ENABLE_SZF) - add_dependencies (h5szf ${HDF5_LIBSH_TARGET}) + if (ENABLE_SZ) + add_dependencies (h5sz ${HDF5_LIBSH_TARGET}) add_dependencies (h5ex_d_sz ${HDF5_LIBSH_TARGET}) target_include_directories (h5ex_d_sz PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR}") endif () diff --git a/config/cmake/HDF5UseFortran.cmake b/config/cmake/HDF5UseFortran.cmake index 0d05e20..43e006c 100644 --- a/config/cmake/HDF5UseFortran.cmake +++ b/config/cmake/HDF5UseFortran.cmake @@ -134,8 +134,8 @@ FORTRAN_RUN ("REAL and INTEGER KINDs" PROG_OUTPUT ) # dnl The output from the above program will be: -# dnl -- LINE 1 -- valid integer kinds (comma seperated list) -# dnl -- LINE 2 -- valid real kinds (comma seperated list) +# dnl -- LINE 1 -- valid integer kinds (comma separated list) +# dnl -- LINE 2 -- valid real kinds (comma separated list) # dnl -- LINE 3 -- max decimal precision for reals # dnl -- LINE 4 -- number of valid integer kinds # dnl -- LINE 5 -- number of valid real kinds diff --git a/config/cmake/HDF5_Examples.cmake.in b/config/cmake/HDF5_Examples.cmake.in index db638fd..9ff9b6f 100644 --- a/config/cmake/HDF5_Examples.cmake.in +++ b/config/cmake/HDF5_Examples.cmake.in @@ -83,12 +83,12 @@ set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDN ############################################################################################################### if(WIN32) set(SITE_OS_NAME "Windows") - set(ENV{HDF5_DIR} "${INSTALLDIR}/share/cmake") + set(ENV{HDF5_ROOT} "${INSTALLDIR}") set(CTEST_BINARY_NAME ${CTEST_SOURCE_NAME}\\build) set(CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_SOURCE_NAME}") set(CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_BINARY_NAME}") else() - set(ENV{HDF5_DIR} "${INSTALLDIR}/share/cmake") + set(ENV{HDF5_ROOT} "${INSTALLDIR}") set(ENV{LD_LIBRARY_PATH} "${INSTALLDIR}/lib") set(CTEST_BINARY_NAME ${CTEST_SOURCE_NAME}/build) set(CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_SOURCE_NAME}") diff --git a/config/cmake/HDFCXXCompilerFlags.cmake b/config/cmake/HDFCXXCompilerFlags.cmake index a5af43c..e20ed6b 100644 --- a/config/cmake/HDFCXXCompilerFlags.cmake +++ b/config/cmake/HDFCXXCompilerFlags.cmake @@ -240,7 +240,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # Append more extra warning flags that only gcc 9.3+ knows about if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.3) - # do not use C warnings, gnu-warnings 9.3, no cxx warniings + # do not use C warnings, gnu-warnings 9.3, no cxx warnings # ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/9.3") endif () diff --git a/config/cmake/README.md.cmake.in b/config/cmake/README.md.cmake.in new file mode 100644 index 0000000..b60e729 --- /dev/null +++ b/config/cmake/README.md.cmake.in @@ -0,0 +1,80 @@ +@HDF5_PACKAGE_NAME@ version @HDF5_PACKAGE_VERSION_STRING@ +------------------------------------------------------------------------------ + +This directory contains the binary (release) distribution of +@HDF5_PACKAGE_NAME@ @HDF5_PACKAGE_VERSION_MAJOR@ that was compiled on; + @BINARY_PLATFORM@. + +It was built with the following options: + -- @LIB_TYPE@ C/C++/Fortran libraries + -- SZIP (encoder enabled) and ZLIB + -- @LIB_TYPE@ HDF5 tools + -- Java @Java_VERSION@ + +The contents of this directory are: + + COPYING - Copyright notice + README.md - This file + @HDF5_PACKAGE_NAME@-@HDF5_PACKAGE_VERSION@-@BINARY_SYSTEM_NAME@.@BINARY_INSTALL_ENDING@ - HDF5 Install Package + +This binary was built with the ZLIB and SZIP/Libaec external libraries and are +included for convenience. Libaec is an unrestricted open-source replacement for SZIP +(version 1.0.4, Encoder ENABLED). + +The official ZLIB and SZIP/Libaec pages are at: + + ZLIB: https://git.savannah.gnu.org/cgit/gzip.git/ + https://git.savannah.gnu.org/cgit/gzip.git/tree/COPYING + SZIP/Libaec: https://gitlab.dkrz.de/k202009/libaec + https://gitlab.dkrz.de/k202009/libaec/-/blob/master/Copyright.txt + + +Installation +=========================================================================== +1. Execute @HDF5_PACKAGE_NAME@-@HDF5_PACKAGE_VERSION@-@BINARY_SYSTEM_NAME@.@BINARY_INSTALL_ENDING@ +2. Follow prompts +=========================================================================== + +After Installation +=========================================================================== +The examples folder, HDF5Examples, located in the +HDF5 install folder, can be built and tested with CMake and the supplied +HDF5_Examples.cmake file. The HDF5_Examples.cmake expects HDF5 to have +been installed in the default location with above compilers (see the +libhdf5.settings file in the lib install folder). Also, the CMake +utility should be installed. + +To test the installation with the examples; + Create a directory to run the examples. + Copy HDF5Examples folder to this directory. + Copy CTestScript.cmake to this directory. + Copy HDF5_Examples.cmake to this directory. + Copy HDF5_Examples_options.cmake to this directory. + The default source folder is defined as "HDF5Examples". It can be changed + with the CTEST_SOURCE_NAME script option. + The default installation folder is defined as "@CMAKE_INSTALL_PREFIX@". + It can be changed with the INSTALLDIR script option. + The default ctest configuration is defined as "Release". It can be changed + with the CTEST_CONFIGURATION_TYPE script option. Note that this must + be the same as the value used with the -C command line option. + The default build configuration is defined to build and use static libraries. + Shared libraries can be used with the STATICONLYLIBRARIES script option set to "NO". + Other options can be changed by editing the HDF5_Examples_options.cmake file. + + If the defaults are okay, execute from this directory: + ctest -S HDF5_Examples.cmake -C Release -V -O test.log + If the defaults need change, execute from this directory: + ctest -S HDF5_Examples.cmake,CTEST_SOURCE_NAME=MyExamples,INSTALLDIR=MyLocation -C Release -V -O test.log + +When executed, the ctest script will save the results to the log file, test.log, as +indicated by the ctest command. If you wish the to see more build and test information, +add "-VV" to the ctest command. The output should show; + 100% tests passed, 0 tests failed out of 156. + +For more information see USING_CMake_Examples.txt in the install folder. +=========================================================================== + +Documentation for this release can be found at the following URL: + https://portal.hdfgroup.org/display/HDF5/HDF5 + +Bugs should be reported to help@hdfgroup.org. diff --git a/config/cmake/README.txt.cmake.in b/config/cmake/README.txt.cmake.in deleted file mode 100644 index 9289870..0000000 --- a/config/cmake/README.txt.cmake.in +++ /dev/null @@ -1,80 +0,0 @@ -@HDF5_PACKAGE_NAME@ version @HDF5_PACKAGE_VERSION_STRING@ ------------------------------------------------------------------------------- - -This directory contains the binary (release) distribution of -@HDF5_PACKAGE_NAME@ @HDF5_PACKAGE_VERSION_MAJOR@ that was compiled on; - @BINARY_PLATFORM@. - -It was built with the following options: - -- @LIB_TYPE@ C/C++/Fortran libraries - -- SZIP (encoder enabled) and ZLIB - -- @LIB_TYPE@ HDF5 tools - -- Java @Java_VERSION@ - -The contents of this directory are: - - COPYING - Copyright notice - README.txt - This file - @HDF5_PACKAGE_NAME@-@HDF5_PACKAGE_VERSION@-@BINARY_SYSTEM_NAME@.@BINARY_INSTALL_ENDING@ - HDF5 Install Package - -This binary was built with the ZLIB and SZIP/Libaec external libraries and are -included for convenience. Libaec is an unrestricted open-source replacement for SZIP -(version 1.0.4, Encoder ENABLED). - -The official ZLIB and SZIP/Libaec pages are at: - - ZLIB: https://git.savannah.gnu.org/cgit/gzip.git/ - https://git.savannah.gnu.org/cgit/gzip.git/tree/COPYING - SZIP/Libaec: https://gitlab.dkrz.de/k202009/libaec - https://gitlab.dkrz.de/k202009/libaec/-/blob/master/Copyright.txt - - -Installation -=========================================================================== -1. Execute @HDF5_PACKAGE_NAME@-@HDF5_PACKAGE_VERSION@-@BINARY_SYSTEM_NAME@.@BINARY_INSTALL_ENDING@ -2. Follow prompts -=========================================================================== - -After Installation -=========================================================================== -The examples folder, HDF5Examples, located in the -HDF5 install folder, can be built and tested with CMake and the supplied -HDF5_Examples.cmake file. The HDF5_Examples.cmake expects HDF5 to have -been installed in the default location with above compilers (see the -libhdf5.settings file in the lib install folder). Also, the CMake -utility should be installed. - -To test the installation with the examples; - Create a directory to run the examples. - Copy HDF5Examples folder to this directory. - Copy CTestScript.cmake to this directory. - Copy HDF5_Examples.cmake to this directory. - Copy HDF5_Examples_options.cmake to this directory. - The default source folder is defined as "HDF5Examples". It can be changed - with the CTEST_SOURCE_NAME script option. - The default installation folder is defined as "@CMAKE_INSTALL_PREFIX@". - It can be changed with the INSTALLDIR script option. - The default ctest configuration is defined as "Release". It can be changed - with the CTEST_CONFIGURATION_TYPE script option. Note that this must - be the same as the value used with the -C command line option. - The default build configuration is defined to build and use static libraries. - Shared libraries can be used with the STATICONLYLIBRARIES script option set to "NO". - Other options can be changed by editing the HDF5_Examples_options.cmake file. - - If the defaults are okay, execute from this directory: - ctest -S HDF5_Examples.cmake -C Release -V -O test.log - If the defaults need change, execute from this directory: - ctest -S HDF5_Examples.cmake,CTEST_SOURCE_NAME=MyExamples,INSTALLDIR=MyLocation -C Release -V -O test.log - -When executed, the ctest script will save the results to the log file, test.log, as -indicated by the ctest command. If you wish the to see more build and test information, -add "-VV" to the ctest command. The output should show; - 100% tests passed, 0 tests failed out of 156. - -For more information see USING_CMake_Examples.txt in the install folder. -=========================================================================== - -Documentation for this release can be found at the following URL: - https://portal.hdfgroup.org/display/HDF5/HDF5 - -Bugs should be reported to help@hdfgroup.org. diff --git a/config/cmake/cacheinit.cmake b/config/cmake/cacheinit.cmake index e423deb..221c0c9 100644 --- a/config/cmake/cacheinit.cmake +++ b/config/cmake/cacheinit.cmake @@ -47,12 +47,12 @@ set (HDF5_MINGW_STATIC_GCC_LIBS ON CACHE BOOL "Statically link libgcc/libstdc++" set (HDF5_ALLOW_EXTERNAL_SUPPORT "NO" CACHE STRING "Allow External Library Building (NO GIT TGZ)" FORCE) set_property (CACHE HDF5_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO GIT TGZ) -set (ZLIB_TGZ_NAME "ZLib.tar.gz" CACHE STRING "Use ZLib from compressed file" FORCE) +set (ZLIB_TGZ_NAME "ZLib.tar.gz" CACHE STRING "Use HDF5_ZLib from compressed file" FORCE) set (SZIP_TGZ_NAME "SZip.tar.gz" CACHE STRING "Use SZip from compressed file" FORCE) set (SZAEC_TGZ_NAME "LIBAEC.tar.gz" CACHE STRING "Use SZip AEC from compressed file" FORCE) set (USE_LIBAEC ON CACHE BOOL "Use libaec szip replacement" FORCE) -set (ZLIB_PACKAGE_NAME "zlib" CACHE STRING "Name of ZLIB package" FORCE) +set (ZLIB_PACKAGE_NAME "zlib" CACHE STRING "Name of HDF5_ZLIB package" FORCE) set (LIBAEC_PACKAGE_NAME "libaec" CACHE STRING "Name of AEC SZIP package" FORCE) set (SZIP_PACKAGE_NAME "szip" CACHE STRING "Name of SZIP package" FORCE) @@ -85,12 +85,12 @@ set (BLOSC_TGZ_NAME "c-blosc.tar.gz" CACHE STRING "Use BLOSC from compressed fil set (BLOSC_PACKAGE_NAME "blosc" CACHE STRING "Name of BLOSC package" FORCE) -set (ZLIB_GIT_URL "https://git@bitbucket.hdfgroup.org/scm/test/zlib.git" CACHE STRING "Use ZLIB from HDF repo" FORCE) -set (ZLIB_GIT_BRANCH "master" CACHE STRING "" FORCE) +set (BLOSC_ZLIB_GIT_URL "https://git@bitbucket.hdfgroup.org/scm/test/zlib.git" CACHE STRING "Use BLOSC_ZLIB from HDF repo" FORCE) +set (BLOSC_ZLIB_GIT_BRANCH "master" CACHE STRING "" FORCE) -set (ZLIB_TGZ_NAME "ZLib.tar.gz" CACHE STRING "Use ZLib from compressed file" FORCE) +set (BLOSC_ZLIB_TGZ_NAME "ZLib.tar.gz" CACHE STRING "Use BLOSC_ZLib from compressed file" FORCE) -set (ZLIB_PACKAGE_NAME "zlib" CACHE STRING "Name of ZLIB package" FORCE) +set (BLOSC_ZLIB_PACKAGE_NAME "zlib" CACHE STRING "Name of BLOSC_ZLIB package" FORCE) ####### # bzip2 @@ -159,15 +159,15 @@ set (LZF_PACKAGE_NAME "lzf" CACHE STRING "Name of LZF package" FORCE) #set (MAFISC_PACKAGE_NAME "mafisc" CACHE STRING "Name of MAFISC package" FORCE) ###### -# szf +# sz ###### -set (SZF_GIT_URL "https://github.com/disheng222/SZ" CACHE STRING "Use SZ from github repository" FORCE) -set (SZF_GIT_BRANCH "master" CACHE STRING "" FORCE) +set (SZ_GIT_URL "https://github.com/disheng222/SZ" CACHE STRING "Use SZ filter from github repository" FORCE) +set (SZ_GIT_BRANCH "master" CACHE STRING "" FORCE) -set (SZF_TGZ_NAME "szf.tar.gz" CACHE STRING "Use SZ from compressed file" FORCE) +set (SZ_TGZ_NAME "szf.tar.gz" CACHE STRING "Use SZ filter from compressed file" FORCE) -set (SZF_PACKAGE_NAME "szf" CACHE STRING "Name of SZ package" FORCE) +set (SZ_PACKAGE_NAME "SZ" CACHE STRING "Name of SZ filter package" FORCE) ###### # zfp diff --git a/config/cmake/hdf5-config.cmake.in b/config/cmake/hdf5-config.cmake.in index eab09ba..35cee4f 100644 --- a/config/cmake/hdf5-config.cmake.in +++ b/config/cmake/hdf5-config.cmake.in @@ -68,8 +68,8 @@ endif () if (${HDF5_PACKAGE_NAME}_BUILD_JAVA) set (${HDF5_PACKAGE_NAME}_JAVA_INCLUDE_DIRS @PACKAGE_CURRENT_BUILD_DIR@/lib/jarhdf5-@HDF5_VERSION_STRING@.jar - @PACKAGE_CURRENT_BUILD_DIR@/lib/slf4j-api-1.7.25.jar - @PACKAGE_CURRENT_BUILD_DIR@/lib/slf4j-nop-1.7.25.jar + @PACKAGE_CURRENT_BUILD_DIR@/lib/slf4j-api-1.7.33.jar + @PACKAGE_CURRENT_BUILD_DIR@/lib/slf4j-nop-1.7.33.jar ) set (${HDF5_PACKAGE_NAME}_JAVA_LIBRARY "@PACKAGE_CURRENT_BUILD_DIR@/lib") set (${HDF5_PACKAGE_NAME}_JAVA_LIBRARIES "${${HDF5_PACKAGE_NAME}_JAVA_LIBRARY}") @@ -116,7 +116,7 @@ if (NOT TARGET "@HDF5_PACKAGE@") include (@PACKAGE_SHARE_INSTALL_DIR@/@ZLIB_PACKAGE_NAME@@HDF_PACKAGE_EXT@-targets.cmake) endif () if (${HDF5_PACKAGE_NAME}_ENABLE_SZIP_SUPPORT AND ${HDF5_PACKAGE_NAME}_PACKAGE_EXTLIBS) - include (@PACKAGE_SHARE_INSTALL_DIR@/@SZ_PACKAGE_NAME@@HDF_PACKAGE_EXT@-targets.cmake) + include (@PACKAGE_SHARE_INSTALL_DIR@/@SZIP_PACKAGE_NAME@@HDF_PACKAGE_EXT@-targets.cmake) endif () include (@PACKAGE_SHARE_INSTALL_DIR@/@HDF5_PACKAGE@@HDF_PACKAGE_EXT@-targets.cmake) endif () diff --git a/config/cmake/libh5cc.in b/config/cmake/libh5cc.in index ecdd13e..f5d8d4c 100644 --- a/config/cmake/libh5cc.in +++ b/config/cmake/libh5cc.in @@ -27,6 +27,27 @@ prg=$dir/$(basename -- "$prg") || exit printf '%s\n' "$prg" printf 'dir is %s\n' "$dir" + +# Show the configuration summary of the library recorded in the +# libhdf5.settings file reside in the lib directory. +showconfigure() +{ + cat $dir/lib/libhdf5.settings + status=$? +} + export PKG_CONFIG_PATH=$dir/lib/pkgconfig -@_PKG_CONFIG_COMPILER@ $@ `pkg-config --define-variable=prefix=$dir --cflags --libs @_PKG_CONFIG_LIBNAME@` +for arg in $@ ; do + case "$arg" in + -showconfig) + showconfigure + exit $status + ;; + *) + @_PKG_CONFIG_COMPILER@ $@ `pkg-config --define-variable=prefix=$dir --cflags --libs @_PKG_CONFIG_LIBNAME@` + status=$? + exit $status + ;; + esac +done diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 2f5af77..9a3ead2 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -37,7 +37,7 @@ cmake_minimum_required (VERSION 3.12) # CTEST_SOURCE_NAME - source folder ############################################################################## -set (CTEST_SOURCE_VERSION "1.13.1") +set (CTEST_SOURCE_VERSION "1.13.2") set (CTEST_SOURCE_VERSEXT "-1") ############################################################################## @@ -68,7 +68,7 @@ endif () # build generator must be defined if (NOT DEFINED BUILD_GENERATOR) - message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2019, VS201964, VS2017, or VS201764, VS2015, VS201564") + message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2019, VS201964, VS2017, VS201764, VS2015, VS201564") endif () ################################################################### diff --git a/config/cmake_ext_mod/ConfigureChecks.cmake b/config/cmake_ext_mod/ConfigureChecks.cmake index 53cc2d1..07b2904 100644 --- a/config/cmake_ext_mod/ConfigureChecks.cmake +++ b/config/cmake_ext_mod/ConfigureChecks.cmake @@ -129,7 +129,7 @@ CHECK_INCLUDE_FILE_CONCAT ("netinet/in.h" ${HDF_PREFIX}_HAVE_NETINET_IN_H) CHECK_INCLUDE_FILE_CONCAT ("netdb.h" ${HDF_PREFIX}_HAVE_NETDB_H) CHECK_INCLUDE_FILE_CONCAT ("arpa/inet.h" ${HDF_PREFIX}_HAVE_ARPA_INET_H) -## Check for non-standard extenstion quadmath.h +## Check for non-standard extension quadmath.h CHECK_INCLUDE_FILES(quadmath.h C_HAVE_QUADMATH) if (${C_HAVE_QUADMATH}) @@ -223,65 +223,63 @@ set (LINUX_LFS 0) set (HDF_EXTRA_C_FLAGS) set (HDF_EXTRA_FLAGS) if (MINGW OR NOT WINDOWS) - # Might want to check explicitly for Linux and possibly Cygwin - # instead of checking for not Solaris or Darwin. - if (NOT ${HDF_PREFIX}_HAVE_SOLARIS AND NOT ${HDF_PREFIX}_HAVE_DARWIN) - # Linux Specific flags - # This was originally defined as _POSIX_SOURCE which was updated to - # _POSIX_C_SOURCE=199506L to expose a greater amount of POSIX - # functionality so clock_gettime and CLOCK_MONOTONIC are defined - # correctly. This was later updated to 200112L so that - # posix_memalign() is visible for the direct VFD code on Linux - # systems. - # POSIX feature information can be found in the gcc manual at: - # http://www.gnu.org/s/libc/manual/html_node/Feature-Test-Macros.html - set (HDF_EXTRA_C_FLAGS -D_POSIX_C_SOURCE=200809L) - - # Need to add this so that O_DIRECT is visible for the direct - # VFD on Linux systems. - set (HDF_EXTRA_C_FLAGS ${HDF_EXTRA_C_FLAGS} -D_GNU_SOURCE) - - option (HDF_ENABLE_LARGE_FILE "Enable support for large (64-bit) files on Linux." ON) - if (HDF_ENABLE_LARGE_FILE AND NOT DEFINED TEST_LFS_WORKS_RUN) - set (msg "Performing TEST_LFS_WORKS") - try_run (TEST_LFS_WORKS_RUN TEST_LFS_WORKS_COMPILE - ${CMAKE_BINARY_DIR} - ${HDF_RESOURCES_EXT_DIR}/HDFTests.c - COMPILE_DEFINITIONS "-DTEST_LFS_WORKS" - ) + if (CMAKE_SYSTEM_NAME MATCHES "Linux") + # Linux Specific flags + # This was originally defined as _POSIX_SOURCE which was updated to + # _POSIX_C_SOURCE=199506L to expose a greater amount of POSIX + # functionality so clock_gettime and CLOCK_MONOTONIC are defined + # correctly. This was later updated to 200112L so that + # posix_memalign() is visible for the direct VFD code on Linux + # systems. + # POSIX feature information can be found in the gcc manual at: + # http://www.gnu.org/s/libc/manual/html_node/Feature-Test-Macros.html + set (HDF_EXTRA_C_FLAGS -D_POSIX_C_SOURCE=200809L) + + # Need to add this so that O_DIRECT is visible for the direct + # VFD on Linux systems. + set (HDF_EXTRA_C_FLAGS ${HDF_EXTRA_C_FLAGS} -D_GNU_SOURCE) + + option (HDF_ENABLE_LARGE_FILE "Enable support for large (64-bit) files on Linux." ON) + if (HDF_ENABLE_LARGE_FILE AND NOT DEFINED TEST_LFS_WORKS_RUN) + set (msg "Performing TEST_LFS_WORKS") + try_run (TEST_LFS_WORKS_RUN TEST_LFS_WORKS_COMPILE + ${CMAKE_BINARY_DIR} + ${HDF_RESOURCES_EXT_DIR}/HDFTests.c + COMPILE_DEFINITIONS "-DTEST_LFS_WORKS" + ) - # The LARGEFILE definitions were from the transition period - # and are probably no longer needed. The FILE_OFFSET_BITS - # check should be generalized for all POSIX systems as it - # is in the Autotools. - if (TEST_LFS_WORKS_COMPILE) - if (TEST_LFS_WORKS_RUN MATCHES 0) - set (TEST_LFS_WORKS 1 CACHE INTERNAL ${msg}) - set (LARGEFILE 1) - set (HDF_EXTRA_FLAGS ${HDF_EXTRA_FLAGS} -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_LARGEFILE_SOURCE) - if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "${msg}... yes") + # The LARGEFILE definitions were from the transition period + # and are probably no longer needed. The FILE_OFFSET_BITS + # check should be generalized for all POSIX systems as it + # is in the Autotools. + if (TEST_LFS_WORKS_COMPILE) + if (TEST_LFS_WORKS_RUN MATCHES 0) + set (TEST_LFS_WORKS 1 CACHE INTERNAL ${msg}) + set (LARGEFILE 1) + set (HDF_EXTRA_FLAGS ${HDF_EXTRA_FLAGS} -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_LARGEFILE_SOURCE) + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (VERBOSE "${msg}... yes") + endif () + else () + set (TEST_LFS_WORKS "" CACHE INTERNAL ${msg}) + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (VERBOSE "${msg}... no") + endif () + file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Test TEST_LFS_WORKS Run failed with the following exit code:\n ${TEST_LFS_WORKS_RUN}\n" + ) endif () else () set (TEST_LFS_WORKS "" CACHE INTERNAL ${msg}) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "${msg}... no") + message (VERBOSE "${msg}... no") endif () file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log - "Test TEST_LFS_WORKS Run failed with the following exit code:\n ${TEST_LFS_WORKS_RUN}\n" + "Test TEST_LFS_WORKS Compile failed\n" ) endif () - else () - set (TEST_LFS_WORKS "" CACHE INTERNAL ${msg}) - if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "${msg}... no") - endif () - file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log - "Test TEST_LFS_WORKS Compile failed\n" - ) endif () - endif () - set (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} ${HDF_EXTRA_FLAGS}) + set (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} ${HDF_EXTRA_FLAGS}) endif () endif () @@ -549,7 +547,7 @@ if (WINDOWS) message (VERBOSE "Performing Test InitOnceExecuteOnce - Success") endif () file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log - "Performing C SOURCE FILE Test InitOnceExecuteOnce succeded with the following output:\n" + "Performing C SOURCE FILE Test InitOnceExecuteOnce succeeded with the following output:\n" "${OUTPUT}\n" "Return value: ${HAVE_IOEO}\n") else () diff --git a/config/cmake_ext_mod/HDFLibMacros.cmake b/config/cmake_ext_mod/HDFLibMacros.cmake index 2af3229..40d06e7 100644 --- a/config/cmake_ext_mod/HDFLibMacros.cmake +++ b/config/cmake_ext_mod/HDFLibMacros.cmake @@ -139,11 +139,11 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) # add_subdirectory(${szip_SOURCE_DIR} ${szip_BINARY_DIR}) # endif() # -##include (${BINARY_DIR}/${SZ_PACKAGE_NAME}${HDF_PACKAGE_EXT}-targets.cmake) +##include (${BINARY_DIR}/${SZIP_PACKAGE_NAME}${HDF_PACKAGE_EXT}-targets.cmake) # Create imported target szip-static if (USE_LIBAEC) add_library(${HDF_PACKAGE_NAMESPACE}sz-static STATIC IMPORTED) - HDF_IMPORT_SET_LIB_OPTIONS (${HDF_PACKAGE_NAMESPACE}sz-static "sz" STATIC "") + HDF_IMPORT_SET_LIB_OPTIONS (${HDF_PACKAGE_NAMESPACE}sz-static "szaec" STATIC "") add_dependencies (${HDF_PACKAGE_NAMESPACE}sz-static SZIP) add_library(${HDF_PACKAGE_NAMESPACE}aec-static STATIC IMPORTED) HDF_IMPORT_SET_LIB_OPTIONS (${HDF_PACKAGE_NAMESPACE}aec-static "aec" STATIC "") @@ -188,7 +188,7 @@ endmacro () #------------------------------------------------------------------------------- macro (EXTERNAL_ZLIB_LIBRARY compress_type) if (${compress_type} MATCHES "GIT") - EXTERNALPROJECT_ADD (ZLIB + EXTERNALPROJECT_ADD (HDF5_ZLIB GIT_REPOSITORY ${ZLIB_URL} GIT_TAG ${ZLIB_BRANCH} INSTALL_COMMAND "" @@ -207,7 +207,7 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type) -DPACKAGE_NAMESPACE=${HDF_PACKAGE_NAMESPACE} ) elseif (${compress_type} MATCHES "TGZ") - EXTERNALPROJECT_ADD (ZLIB + EXTERNALPROJECT_ADD (HDF5_ZLIB URL ${ZLIB_URL} URL_MD5 "" INSTALL_COMMAND "" @@ -226,19 +226,15 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type) -DPACKAGE_NAMESPACE=${HDF_PACKAGE_NAMESPACE} ) endif () - externalproject_get_property (ZLIB BINARY_DIR SOURCE_DIR) + externalproject_get_property (HDF5_ZLIB BINARY_DIR SOURCE_DIR) - if (WIN32) - set (ZLIB_LIB_NAME "zlib") - else () - set (ZLIB_LIB_NAME "z") - endif () + set (ZLIB_LIB_NAME "z") ##include (${BINARY_DIR}/${ZLIB_PACKAGE_NAME}${HDF_PACKAGE_EXT}-targets.cmake) # Create imported target zlib-static add_library(${HDF_PACKAGE_NAMESPACE}zlib-static STATIC IMPORTED) # add_library(${HDF_PACKAGE_NAMESPACE}zlib-static ALIAS zlib-static) HDF_IMPORT_SET_LIB_OPTIONS (${HDF_PACKAGE_NAMESPACE}zlib-static ${ZLIB_LIB_NAME} STATIC "") - add_dependencies (${HDF_PACKAGE_NAMESPACE}zlib-static ZLIB) + add_dependencies (${HDF_PACKAGE_NAMESPACE}zlib-static HDF5_ZLIB) set (ZLIB_STATIC_LIBRARY "${HDF_PACKAGE_NAMESPACE}zlib-static") set (ZLIB_LIBRARIES ${ZLIB_STATIC_LIBRARY}) @@ -256,6 +252,6 @@ macro (PACKAGE_ZLIB_LIBRARY compress_type) ) set (EXTERNAL_HEADER_LIST ${EXTERNAL_HEADER_LIST} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/zconf.h) if (${compress_type} MATCHES "GIT" OR ${compress_type} MATCHES "TGZ") - add_dependencies (ZLIB-GenHeader-Copy ZLIB) + add_dependencies (ZLIB-GenHeader-Copy HDF5_ZLIB) endif () endmacro () diff --git a/config/cmake_ext_mod/HDFMacros.cmake b/config/cmake_ext_mod/HDFMacros.cmake index 6da2b74..0f9b367 100644 --- a/config/cmake_ext_mod/HDFMacros.cmake +++ b/config/cmake_ext_mod/HDFMacros.cmake @@ -223,7 +223,7 @@ macro (TARGET_C_PROPERTIES wintarget libtype) endmacro () #----------------------------------------------------------------------------- -# Configure the README.txt file for the binary package +# Configure the README.md file for the binary package #----------------------------------------------------------------------------- macro (HDF_README_PROPERTIES target_fortran) set (BINARY_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) @@ -301,8 +301,8 @@ macro (HDF_README_PROPERTIES target_fortran) endif () configure_file ( - ${HDF_RESOURCES_DIR}/README.txt.cmake.in - ${CMAKE_BINARY_DIR}/README.txt @ONLY + ${HDF_RESOURCES_DIR}/README.md.cmake.in + ${CMAKE_BINARY_DIR}/README.md @ONLY ) endmacro () @@ -356,7 +356,7 @@ macro (HDF_DIR_PATHS package_prefix) endif () endif () if (NOT ${package_prefix}_INSTALL_CMAKE_DIR) - set (${package_prefix}_INSTALL_CMAKE_DIR share/cmake) + set (${package_prefix}_INSTALL_CMAKE_DIR cmake) endif () # Always use full RPATH, i.e. don't skip the full RPATH for the build tree @@ -427,7 +427,7 @@ macro (HDF_DIR_PATHS package_prefix) endif () endif () - if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + if (NOT ${package_prefix}_EXTERNALLY_CONFIGURED AND CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) if (CMAKE_HOST_UNIX) set (CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/HDF_Group/${HDF5_PACKAGE_NAME}/${HDF5_PACKAGE_VERSION}" CACHE PATH "Install path prefix, prepended onto install directories." FORCE) diff --git a/config/cmake_ext_mod/HDFTests.c b/config/cmake_ext_mod/HDFTests.c index c434be5..2b2a202 100644 --- a/config/cmake_ext_mod/HDFTests.c +++ b/config/cmake_ext_mod/HDFTests.c @@ -115,8 +115,8 @@ int main(void) for (currentArg = llwidthArgs; *currentArg != NULL; currentArg++) { char formatString[64]; - sprintf(formatString, "%%%sd", *currentArg); - sprintf(s, formatString, x); + snprintf(formatString, sizeof(formatString), "%%%sd", *currentArg); + snprintf(s, 128, formatString, x); if (strcmp(s, "1099511627776") == 0) { printf("PRINTF_LL_WIDTH=[%s]\n", *currentArg); diff --git a/config/cmake_ext_mod/HDFUseFortran.cmake b/config/cmake_ext_mod/HDFUseFortran.cmake index 9cbcd76..c96298f 100644 --- a/config/cmake_ext_mod/HDFUseFortran.cmake +++ b/config/cmake_ext_mod/HDFUseFortran.cmake @@ -22,7 +22,7 @@ if (NOT CMAKE_VERSION VERSION_LESS "3.14.0") endif () #------------------------------------------------------------------------------- -# Fix Fortran flags if we are compiling staticly on Windows using +# Fix Fortran flags if we are compiling statically on Windows using # Windows_MT.cmake from config/cmake/UserMacros #------------------------------------------------------------------------------- if (BUILD_STATIC_CRT_LIBS) diff --git a/config/cmake_ext_mod/NSIS.template.in b/config/cmake_ext_mod/NSIS.template.in index ecced05..db79466 100644 --- a/config/cmake_ext_mod/NSIS.template.in +++ b/config/cmake_ext_mod/NSIS.template.in @@ -849,7 +849,7 @@ Section "Uninstall" @CPACK_NSIS_DELETE_ICONS@ @CPACK_NSIS_DELETE_ICONS_EXTRA@ - ;Delete empty start menu parent diretories + ;Delete empty start menu parent directories StrCpy $MUI_TEMP "$SMPROGRAMS\$MUI_TEMP" startMenuDeleteLoop: @@ -868,7 +868,7 @@ Section "Uninstall" Delete "$SMPROGRAMS\$MUI_TEMP\Uninstall.lnk" @CPACK_NSIS_DELETE_ICONS_EXTRA@ - ;Delete empty start menu parent diretories + ;Delete empty start menu parent directories StrCpy $MUI_TEMP "$SMPROGRAMS\$MUI_TEMP" secondStartMenuDeleteLoop: diff --git a/config/cmake_ext_mod/runTest.cmake b/config/cmake_ext_mod/runTest.cmake index f552dcd..05f4473 100644 --- a/config/cmake_ext_mod/runTest.cmake +++ b/config/cmake_ext_mod/runTest.cmake @@ -243,7 +243,14 @@ if (NOT TEST_SKIP_COMPARE) list (LENGTH test_act len_act) file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref) list (LENGTH test_ref len_ref) + if (NOT len_act EQUAL len_ref) + set (TEST_COMPARE_RESULT 1) + endif () if (len_act GREATER 0 AND len_ref GREATER 0) + if (TEST_SORT_COMPARE) + list (SORT test_act) + list (SORT test_ref) + endif () math (EXPR _FP_LEN "${len_ref} - 1") foreach (line RANGE 0 ${_FP_LEN}) list (GET test_act ${line} str_act) @@ -263,9 +270,6 @@ if (NOT TEST_SKIP_COMPARE) message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty") endif () endif () - if (NOT len_act EQUAL len_ref) - set (TEST_COMPARE_RESULT 1) - endif () endif () endif () diff --git a/config/conclude.am b/config/conclude.am index 7d19082..e798d8d 100644 --- a/config/conclude.am +++ b/config/conclude.am @@ -270,10 +270,27 @@ build-check-p: $(LIB) $(PROGS) $(chk_TESTS) echo "===Parallel tests in `echo ${PWD} | sed -e s:.*/::` ended `date`===";\ fi -VFD_LIST = sec2 stdio core core_paged split multi family +VFD_LIST = sec2 stdio core core_paged split multi family splitter + +# log VFD currently has file space allocation bugs +# VFD_LIST += log + if DIRECT_VFD_CONDITIONAL VFD_LIST += direct endif +if BUILD_PARALLEL_CONDITIONAL + # MPI I/O VFD is currently incompatible with too many tests in the VFD test set + # VFD_LIST += mpio +endif +if MIRROR_VFD_CONDITIONAL + VFD_LIST += mirror +endif +if ROS3_VFD_CONDITIONAL + VFD_LIST += ros3 +endif +if HDFS_VFD_CONDITIONAL + VFD_LIST += hdfs +endif # Run test with different Virtual File Driver check-vfd: $(LIB) $(PROGS) $(chk_TESTS) diff --git a/config/gnu-cxxflags b/config/gnu-cxxflags index 56e89bc..b544813 100644 --- a/config/gnu-cxxflags +++ b/config/gnu-cxxflags @@ -58,7 +58,7 @@ load_gnu_arguments() if test "X-" = "X-$cxx_flags_set"; then # PathScale compiler spits out gcc version string too. Need to # filter it out. - # icc beginning with version 12 includes a "gcc version compatiblilty" + # icc beginning with version 12 includes a "gcc version compatibility" # string, causing the gcc H5_CXXFLAGS to be erroneously added. The line # "grep -v 'icc version'" causes the discarding of any output # containing 'icc version'. The cxx_version for icc is correctly determined diff --git a/config/gnu-flags b/config/gnu-flags index 89239df..fe6626d 100644 --- a/config/gnu-flags +++ b/config/gnu-flags @@ -58,7 +58,7 @@ load_gnu_arguments() if test "X-" = "X-$cc_flags_set"; then # PathScale compiler spits out gcc version string too. Need to # filter it out. - # icc beginning with version 12 includes a "gcc version compatiblilty" + # icc beginning with version 12 includes a "gcc version compatibility" # string, causing the gcc H5_CFLAGS to be erroneously added. The line # "grep -v 'icc version'" causes the discarding of any output # containing 'icc version'. The cc_version for icc is correctly determined diff --git a/config/sanitizer/code-coverage.cmake b/config/sanitizer/code-coverage.cmake index c79aeac..e71bfd7 100644 --- a/config/sanitizer/code-coverage.cmake +++ b/config/sanitizer/code-coverage.cmake @@ -80,10 +80,17 @@ option( OFF) # Programs -find_program(LLVM_COV_PATH llvm-cov) -find_program(LLVM_PROFDATA_PATH llvm-profdata) -find_program(LCOV_PATH lcov) -find_program(GENHTML_PATH genhtml) +if(WIN32) + find_program(LLVM_COV_PATH llvm-cov PATHS ENV VS2019INSTALLDIR PATH_SUFFIXES "VC/Tools/Llvm/x64/bin") + find_program(LLVM_PROFDATA_PATH llvm-profdata PATHS ENV VS2019INSTALLDIR PATH_SUFFIXES "VC/Tools/Llvm/x64/bin") + find_program(LCOV_PATH lcov PATHS ENV VS2019INSTALLDIR PATH_SUFFIXES "VC/Tools/Llvm/x64/bin") + find_program(GENHTML_PATH genhtml PATHS ENV VS2019INSTALLDIR PATH_SUFFIXES "VC/Tools/Llvm/x64/bin") +else() + find_program(LLVM_COV_PATH llvm-cov) + find_program(LLVM_PROFDATA_PATH llvm-profdata) + find_program(LCOV_PATH lcov) + find_program(GENHTML_PATH genhtml) +endif() # Variables set(CMAKE_COVERAGE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/ccov) diff --git a/config/sanitizer/sanitizers.cmake b/config/sanitizer/sanitizers.cmake index 8bf1d5b..b06992f 100644 --- a/config/sanitizer/sanitizers.cmake +++ b/config/sanitizer/sanitizers.cmake @@ -83,12 +83,21 @@ if(USE_SANITIZER) elseif(MSVC) if(USE_SANITIZER MATCHES "([Aa]ddress)") message(STATUS "Building with Address sanitizer") - append("/fsanitize=address" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) + append("-fsanitize=address" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) else() message(FATAL_ERROR "This sanitizer not yet supported in the MSVC environment: ${USE_SANITIZER}") endif() else() message(FATAL_ERROR "USE_SANITIZER is not supported on this platform.") endif() + elseif(MSVC) + if(USE_SANITIZER MATCHES "([Aa]ddress)") + message(STATUS "Building with Address sanitizer") + append("/fsanitize=address" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) + else() + message(FATAL_ERROR "This sanitizer not yet supported in the MSVC environment: ${USE_SANITIZER}") + endif() + else() + message(FATAL_ERROR "USE_SANITIZER is not supported on this platform.") endif() endif() diff --git a/config/solaris b/config/solaris index b7c7963..6b6b74d 100644 --- a/config/solaris +++ b/config/solaris @@ -39,7 +39,7 @@ if test "X-" = "X-$cc_flags_set"; then # Debug # NDEBUG is handled explicitly by the configure script - # -g is hanled by the symbols flags + # -g is handled by the symbols flags DEBUG_CFLAGS= # Symbols @@ -101,7 +101,7 @@ if test "X-" = "X-$f9x_flags_set"; then # Debug # NDEBUG is handled explicitly by the configure script - # -g is hanled by the symbols flags + # -g is handled by the symbols flags DEBUG_FCFLAGS= # Symbols @@ -157,7 +157,7 @@ if test -z "$cxx_flags_set"; then # Debug # NDEBUG is handled explicitly by the configure script - # -g is hanled by the symbols flags + # -g is handled by the symbols flags DEBUG_CXXFLAGS= # Symbols diff --git a/configure.ac b/configure.ac index f8a1e4d..53d181b 100644 --- a/configure.ac +++ b/configure.ac @@ -23,7 +23,7 @@ AC_PREREQ([2.69]) ## NOTE: Do not forget to change the version number here when we do a ## release!!! ## -AC_INIT([HDF5], [1.13.1-1], [help@hdfgroup.org]) +AC_INIT([HDF5], [1.13.2-1], [help@hdfgroup.org]) AC_CONFIG_SRCDIR([src/H5.c]) AC_CONFIG_HEADERS([src/H5config.h]) @@ -390,7 +390,7 @@ if test "X$CC_BASENAME" = "Xclang"; then # 'address;undefined'. Which and which combinations of these are # supported varies by compiler version, but unsupported options # or combinations will result in configure errors reported in config.log. - # Comma separated lists of sanitize options wil be entered intact in + # Comma separated lists of sanitize options will be entered intact in # one -fsanitize= flag. Space separated lists will be entered in # separate -fsanitize= flags. # NOTE: No sanity checking done here! @@ -599,7 +599,7 @@ if test "X$HDF_FORTRAN" = "Xyes"; then AC_SUBST([OBJECT_NAMELEN_DEFAULT_F]) ## -------------------------------------------------------------------- - ## Fortran source extention + ## Fortran source extension ## AC_FC_SRCEXT([f90]) @@ -1122,6 +1122,12 @@ else AC_MSG_RESULT([no]) fi +## These need to be exposed for some tests. +AC_SUBST([H5_UTILS_TEST_BUILDDIR]) +H5_UTILS_TEST_BUILDDIR='utils/test' +AC_SUBST([H5_TEST_BUILDDIR]) +H5_TEST_BUILDDIR='test' + ## ---------------------------------------------------------------------- ## Check if they would like to disable building tools ## @@ -1214,7 +1220,7 @@ if test "X$HDF5_DOXYGEN" = "Xyes"; then DOXYGEN_HTML_HEADER='$(SRCDIR)/doxygen/hdf5_header.html' DOXYGEN_HTML_FOOTER='$(SRCDIR)/doxygen/hdf5_footer.html' DOXYGEN_HTML_EXTRA_STYLESHEET='$(SRCDIR)/doxygen/hdf5doxy.css' - DOXYGEN_HTML_EXTRA_FILES='$(SRCDIR)/doxygen/hdf5_navtree_hacks.js $(SRCDIR)/doxygen/img/ftv2node.png $(SRCDIR)/doxygen/img/ftv2pnode.png' + DOXYGEN_HTML_EXTRA_FILES='$(SRCDIR)/doxygen/hdf5_navtree_hacks.js $(SRCDIR)/doxygen/img/FF-IH_FileGroup.gif $(SRCDIR)/doxygen/img/FF-IH_FileObject.gif $(SRCDIR)/doxygen/img/FileFormatSpecChunkDiagram.jpg $(SRCDIR)/doxygen/img/ftv2node.png $(SRCDIR)/doxygen/img/ftv2pnode.png $(SRCDIR)/doxygen/img/HDFG-logo.png $(SRCDIR)/doxygen/img/IOFlow2.gif $(SRCDIR)/doxygen/img/IOFlow3.gif $(SRCDIR)/doxygen/img/IOFlow.gif $(SRCDIR)/doxygen/img/PaletteExample1.gif $(SRCDIR)/doxygen/img/Palettes.fm.anc.gif' DOXYGEN_TAG_FILE=hdf5.tag DOXYGEN_SERVER_BASED_SEARCH=NO DOXYGEN_EXTERNAL_SEARCH=NO @@ -1686,6 +1692,7 @@ fi ## command-line switch. The value is an include path and/or a library path. ## If the library path is specified then it must be preceded by a comma. ## +AC_SUBST([LL_PATH]) AC_SUBST([USE_FILTER_SZIP]) USE_FILTER_SZIP="no" AC_ARG_WITH([szlib], [AS_HELP_STRING([--with-szlib=DIR], @@ -1770,7 +1777,7 @@ if test "x$HAVE_SZLIB" = "xyes" -a "x$HAVE_SZLIB_H" = "xyes"; then export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$szlib_lib" fi - AC_SUBST([LL_PATH]) LL_PATH="$LD_LIBRARY_PATH" + LL_PATH="$LD_LIBRARY_PATH" AC_CACHE_VAL([hdf5_cv_szlib_can_encode], [AC_RUN_IFELSE( @@ -2144,7 +2151,7 @@ for hdf5_cv_printf_ll in ll l L q unknown; do ],[[ char *s = malloc(128); long long x = (long long)1048576 * (long long)1048576; - sprintf(s,"%${hdf5_cv_printf_ll}d",x); + snprintf(s,128,"%${hdf5_cv_printf_ll}d",x); exit(strcmp(s,"1099511627776")); ]])] , [break],,[continue]) @@ -2977,11 +2984,11 @@ if test -n "$PARALLEL"; then fi ## ---------------------------------------------------------------------- - ## Check for the MPI-3 functions necessary for the Parallel Compression + ## Check for the MPI functions necessary for the Parallel Compression ## feature. If these are not present, issue a warning that Parallel ## Compression will be disabled. ## - AC_MSG_CHECKING([for MPI_Mprobe and MPI_Imrecv functions]) + AC_MSG_CHECKING([for MPI_Ibarrier/MPI_Issend/MPI_Iprobe/MPI_Irecv functions]) AC_LINK_IFELSE( [AC_LANG_PROGRAM( @@ -2989,16 +2996,19 @@ if test -n "$PARALLEL"; then #include ]], [[ - MPI_Message message; + int flag; MPI_Init(0, (void *) 0); - MPI_Mprobe(0, 0, 0, &message, (void *) 0); - MPI_Imrecv((void *) 0, 0, 0, (void *) 0, (void *) 0); + MPI_Ibarrier(0, (void *) 0); + MPI_Issend((void *) 0, 0, 0, 0, 0, 0, (void *) 0); + MPI_Iprobe(0, 0, 0, &flag, (void *) 0); + MPI_Irecv((void *) 0, 0, 0, 0, 0, 0, (void *) 0); ]] )], [AC_MSG_RESULT([yes]) - PARALLEL_FILTERED_WRITES=yes], + PARALLEL_FILTERED_WRITES=yes + AC_DEFINE([HAVE_PARALLEL_FILTERED_WRITES], [1], [Define if we have support for writing to filtered datasets in parallel])], [AC_MSG_RESULT([no]) - AC_MSG_WARN([A simple MPI program using the MPI_Mprobe and MPI_Imrecv functions could not be compiled and linked. + AC_MSG_WARN([A simple MPI program using the MPI_Ibarrier, MPI_Issend, MPI_Iprobe and MPI_Irecv functions could not be compiled and linked. Parallel writes of filtered data will be disabled.]) PARALLEL_FILTERED_WRITES=no] ) @@ -3033,6 +3043,138 @@ if test -n "$PARALLEL"; then fi ## ---------------------------------------------------------------------- +## Build parallel tools if parallel tools, parallel, and build tools options +## are all enabled. +## +AC_SUBST([PARALLEL_TOOLS]) + +## Default is no parallel tools +PARALLEL_TOOLS=no + +AC_MSG_CHECKING([parallel tools]) +AC_ARG_ENABLE([parallel-tools], + [AS_HELP_STRING([--enable-parallel-tools], + [Enable building parallel tools. + [default=no]])], + [PARALLEL_TOOLS=$enableval]) + +if test "X${PARALLEL_TOOLS}" = "Xyes"; then + if test "X${HDF5_TOOLS}" != "Xyes"; then + AC_MSG_ERROR([--enable-tools is required for --enable-parallel-tools]) + fi + if test "X${PARALLEL}" != "Xyes"; then + AC_MSG_ERROR([--enable-parallel is required for --enable-parallel-tools]) + fi +fi + +case "X-$PARALLEL_TOOLS" in + X-|X-no) + AC_MSG_RESULT([no]) + ;; + X-yes) + AC_MSG_RESULT([yes]) + ;; + *) + ;; +esac + +if test "X${PARALLEL_TOOLS}" = "Xyes"; then + ## Is the mpiFileUtils library (libmfu) required and available? + ## + AC_SUBST([H5DWALK_LDFLAGS]) + AC_SUBST([H5DWALK_LIBS]) + AC_SUBST([H5DWALK_CPPFLAGS]) + + H5DWALK_LDFLAGS="${H5DWALK_LDFLAGS}" + + ## Default is not present + HAVE_LIBMFU=no + + AC_ARG_WITH([libmfu], + [AS_HELP_STRING([--with-libmfu=DIR], + [Use the libmfu library [default=no]])],, + [withval=no]) + + case "X-$withval" in + X-yes) + HAVE_LIBMFU="yes" + AC_CHECK_HEADERS([mfu.h],, [unset HAVE_LIBMFU]) + if test "x$HAVE_LIBMFU" = "xyes"; then + AC_CHECK_LIB([mfu], [mfu_init], [H5DWALK_LIBS="-lmfu"], [unset HAVE_LIBMFU]) + fi + if test -z "$HAVE_LIBMFU" -a -n "$HDF5_CONFIG_ABORT"; then + AC_MSG_ERROR([couldn't find libmfu library]) + fi + ;; + X-|X-no|X-none) + HAVE_LIBMFU="no" + AC_MSG_CHECKING([for libmfu library]) + AC_MSG_RESULT([suppressed]) + ;; + *) + HAVE_LIBMFU="yes" + case "$withval" in + *,*) + libmfu_inc="`echo $withval |cut -f1 -d,`" + libmfu_lib="`echo $withval |cut -f2 -d, -s`" + ;; + *) + if test -n "$withval"; then + libmfu_inc="$withval/include" + libmfu_lib="$withval/lib64" + libcircle_lib="$withval/lib" + fi + ;; + esac + + saved_CPPFLAGS="$CPPFLAGS" + saved_AM_CPPFLAGS="$AM_CPPFLAGS" + saved_LDFLAGS="$LDFLAGS" + saved_AM_LDFLAGS="$AM_LDFLAGS" + + ## For these checks we need the libmfu locations added to CPPFLAGS, + ## AM_CPPFLAGS, LDFLAGS, and AM_LDFLAGS. The third param should set them + ## back to these saved values. If the checks pass, then normally these four + ## flag variables would be updated, but in this case we put the changes in + ## variables specific to H5DWALK since they aren't used elsewhere. + if test -n "$libmfu_inc"; then + CPPFLAGS="$CPPFLAGS -I$libmfu_inc" + AM_CPPFLAGS="$AM_CPPFLAGS -I$libmfu_inc" + fi + + if test -n "$libmfu_lib"; then + LDFLAGS="$LDFLAGS -L$libmfu_lib -L$libcircle_lib" + AM_LDFLAGS="$AM_LDFLAGS -L$libmfu_lib -L$libcircle_lib" + fi + + if test "x$HAVE_LIBMFU" = "xyes"; then + AC_CHECK_LIB([mfu], [mfu_init],[H5DWALK_LIBS="-lmfu"], [CPPFLAGS="$saved_CPPFLAGS"; AM_CPPFLAGS="$saved_AM_CPPFLAGS"; LDFLAGS="$saved_LDFLAGS"; AM_LDFLAGS="$saved_AM_LDFLAGS"; unset HAVE_LIBMFU]) + if test -n "$HAVE_LIBMFU"; then + AC_CHECK_HEADERS([mfu.h],[H5DWALK_CPPFLAGS="-I$libmfu_inc"],[CPPFLAGS="$saved_CPPFLAGS"; AM_CPPFLAGS="$saved_AM_CPPFLAGS"; LDFLAGS="$saved_LDFLAGS"; AM_LDFLAGS="$saved_AM_LDFLAGS"; unset HAVE_LIBMFU]) + fi + fi + + if test -z "$HAVE_LIBMFU" -a -n "$HDF5_CONFIG_ABORT"; then + AC_MSG_ERROR([couldn't find libmfu library]) + else + H5DWALK_LDFLAGS="-L$libmfu_lib -L$libcircle_lib" + fi + + if test -z "$LD_LIBRARY_PATH"; then + export LD_LIBRARY_PATH="$libmfu_lib:$libcircle_lib" + else + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$libmfu_lib:$libcircle_lib" + fi + + LL_PATH="$LD_LIBRARY_PATH" + ;; + esac +fi + +## +AM_CONDITIONAL([PARALLEL_TOOLS_CONDITIONAL], [test "X$HAVE_LIBMFU" = "Xyes"]) + +## ---------------------------------------------------------------------- ## Check if the map API is enabled by --enable-map-api ## AC_SUBST([MAP_API]) @@ -3182,7 +3324,7 @@ AC_MSG_CHECKING([if the auxiliary process is enabled]) AC_ARG_ENABLE([aux-process], [AS_HELP_STRING([--enable-aux-process], - [Build the auxiliary process for the VFD SWMR. + [Build the auxiliary process for VFD SWMR. [default=no]])], [AUX_PROCESS=$enableval], [AUX_PROCESS=no]) @@ -3194,6 +3336,9 @@ else AC_MSG_RESULT([no]) fi +## Aux process files built only if able. +AM_CONDITIONAL([AUX_PROCESS_CONDITIONAL], [test "X$AUX_PROCESS" = "Xyes"]) + ## ---------------------------------------------------------------------- ## Check if Read-Only S3 virtual file driver is enabled by --enable-ros3-vfd ## @@ -3392,7 +3537,7 @@ fi ## ---------------------------------------------------------------------- ## Decide whether the data accuracy has higher priority during data -## conversions. If not, some hard conversions will still be prefered even +## conversions. If not, some hard conversions will still be preferred even ## though the data may be wrong (for example, some compilers don't ## support denormalized floating values) to maximize speed. ## @@ -3406,7 +3551,7 @@ AC_ARG_ENABLE([dconv-accuracy], if test "$DATA_ACCURACY" = "yes"; then AC_MSG_RESULT([yes]) AC_DEFINE([WANT_DATA_ACCURACY], [1], - [Data accuracy is prefered to speed during data conversions]) + [Data accuracy is preferred to speed during data conversions]) else AC_MSG_RESULT([no]) fi @@ -3582,8 +3727,8 @@ fi ## and installed with the libraries (used to generate libhdf5.settings). ## -## HDF5 version from the first line of the README.txt file. -H5_VERSION="`cut -d' ' -f3 $srcdir/README.txt | head -1`" +## HDF5 version from the first line of the README.md file. +H5_VERSION="`cut -d' ' -f3 $srcdir/README.md | head -1`" AC_SUBST([H5_VERSION]) ## Configuration date @@ -4087,6 +4232,13 @@ AC_CONFIG_FILES([src/libhdf5.settings testpar/testpflush.sh utils/Makefile utils/mirror_vfd/Makefile + utils/test/Makefile + utils/tools/Makefile + utils/tools/h5dwalk/Makefile + utils/tools/test/Makefile + utils/tools/test/h5dwalk/Makefile + utils/tools/test/h5dwalk/copy_demo_files.sh + utils/tools/test/h5dwalk/testh5dwalk.sh utils/vfd_swmr/Makefile tools/Makefile tools/lib/Makefile diff --git a/doxygen/CMakeLists.txt b/doxygen/CMakeLists.txt index 3462d50..920fafa 100644 --- a/doxygen/CMakeLists.txt +++ b/doxygen/CMakeLists.txt @@ -21,7 +21,7 @@ if (DOXYGEN_FOUND) set (DOXYGEN_HTML_HEADER ${HDF5_DOXYGEN_DIR}/hdf5_header.html) set (DOXYGEN_HTML_FOOTER ${HDF5_DOXYGEN_DIR}/hdf5_footer.html) set (DOXYGEN_HTML_EXTRA_STYLESHEET ${HDF5_DOXYGEN_DIR}/hdf5doxy.css) - set (DOXYGEN_HTML_EXTRA_FILES "${HDF5_DOXYGEN_DIR}/hdf5_navtree_hacks.js ${HDF5_DOXYGEN_DIR}/img/ftv2node.png ${HDF5_DOXYGEN_DIR}/img/ftv2pnode.png") + set (DOXYGEN_HTML_EXTRA_FILES "${HDF5_DOXYGEN_DIR}/hdf5_navtree_hacks.js ${HDF5_DOXYGEN_DIR}/img/FF-IH_FileGroup.gif ${HDF5_DOXYGEN_DIR}/img/FF-IH_FileObject.gif ${HDF5_DOXYGEN_DIR}/img/FileFormatSpecChunkDiagram.jpg ${HDF5_DOXYGEN_DIR}/img/ftv2node.png ${HDF5_DOXYGEN_DIR}/img/ftv2pnode.png ${HDF5_DOXYGEN_DIR}/img/HDFG-logo.png ${HDF5_DOXYGEN_DIR}/img/IOFlow2.gif ${HDF5_DOXYGEN_DIR}/img/IOFlow3.gif ${HDF5_DOXYGEN_DIR}/img/IOFlow.gif ${HDF5_DOXYGEN_DIR}/img/PaletteExample1.gif ${HDF5_DOXYGEN_DIR}/img/Palettes.fm.anc.gif") set (DOXYGEN_TAG_FILE ${HDF5_BINARY_DIR}/hdf5.tag) set (DOXYGEN_SERVER_BASED_SEARCH NO) set (DOXYGEN_EXTERNAL_SEARCH NO) diff --git a/doxygen/aliases b/doxygen/aliases index f83a875..11fa691 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -299,7 +299,7 @@ ALIASES += ref_rfc20130630="HDF5 Dynamically Loaded Filters" ALIASES += ref_rfc20121114="Direct Chunk Write" ALIASES += ref_rfc20121024="HDF5 File Space Management" -ALIASES += ref_rfc20120828="New HDF5 API Routines for HPC Applications" +ALIASES += ref_rfc20120828="New HDF5 API Routines for HPC Applications - Read/Write Multiple Datasets in an HDF5 file" ALIASES += ref_rfc20120523="HDF5 File Space Management: Paged Aggregation" ALIASES += ref_rfc20120501="HDF5 File Image Operations" ALIASES += ref_rfc20120305="Enabling a Strict Consistency Semantics Model in Parallel HDF5" diff --git a/doxygen/dox/About.dox b/doxygen/dox/About.dox index 9000a78..b03ebf6 100644 --- a/doxygen/dox/About.dox +++ b/doxygen/dox/About.dox @@ -124,6 +124,4 @@ version. Talk to your friendly IT-team if you need write access, or you need someone to push an updated version for you! -\todo Make the publication a GitHub action! - */ diff --git a/doxygen/dox/Glossary.dox b/doxygen/dox/Glossary.dox new file mode 100644 index 0000000..9ccd27d --- /dev/null +++ b/doxygen/dox/Glossary.dox @@ -0,0 +1,565 @@ +/** \page GLS Glossary + +\section GLS_A A + +
+
Array datatype
+
A family of HDF5 datatypes whose elements are arrays of a fixed rank (≤ + 32) and fixed finite extent. All array elements must be of the same HDF5 + datatype.
+
+ +
+
Array variable
+

A variable that can store (logically) dense, rectilinear, multidimensional + arrays of elements of a given HDF5 datatype.

+

The combination of array rank (dimensionality) and extent is called an + array variable's shape. This includes the degenerate array shapes of a + singleton (scalar) and the empty array (null).

+

The array element datatype is sometimes referred to as the array + variable's type, which is not entirely accurate because the array variable's + type is 'array of element type' rather than 'element type'.

+

In HDF5, there are two kinds of array variables, attributes and datasets, + and the distinction is functional (i.e., how they can be used) rather than + conceptual. Attributes are commonly used for descriptive "light-weight" + HDF5 object metadata while datasets are HDF5 objects used to store + "heavy-weight" problem-sized data.

+
+
+ +
+
Attribute
+

A named array variable that is associated with an HDF5 object, its + owner or attributee, and used to represent application domain-specific + metadata of the object. Intuitively, the set of an object's attributes can + be thought of as its key-value pair collection. Attribute names (keys) can + be arbitrary Unicode strings, but must be unique per object, i.e., an + object can have at most one attribute with a given name.

+

A scalar attribute is an attribute backed by a singleton array + variable. A null attribute is attribute backed by an empty array + variable.

+
+
+ +\section GLS_B B + +
+
Bitfield datatype
+
A family of HDF5 datatypes whose elements are fixed-width bit fields.
+
+ +\section GLS_C C + +
+
Chunked layout
+
+

A dataset storage layout where the dataset elements are partitioned into + fixed-size multidimensional chunks or tiles. Chunked layout is mandatory + for datasets with one or more dimensions of indefinite (infinite) extent + or where compression or other filters are applied to the dataset elements.

+

Chunked layout may improve I/O performance for certain access patterns.

+
+
+ +
+
Committed datatype
+
An immutable kind of HDF5 object that is used to store an HDF5 datatype + definition, which can be referenced by multiple array variables. When linked + to an HDF5 group, a committed datatype can be located by an HDF5 path name, + and is sometimes called a named datatype.
+
+ +
+
Compact layout
+
+
+ +
+
Compound datatype
+
+

A family of HDF5 datatypes whose elements are records with named fields + of other HDF5 datatypes. Currently, on ASCII field names are supported.

+

Similar to a struct in C or a COMMON block in + Fortran.

+
+
+ +
+
Contiguous layout
+
A dataset storage layout where the dataset elements are physically stored + in an HDF5 file as a contiguous block of bytes.
+
+ +\section GLS_D D + +
+
Dataset
+
+

A kind of HDF5 object, a linked array variable. which can be located in + an HDF5 file through a path name. Datasets are commonly used to store + "heavy-weight" problem-sized data.

+

The HDF5 library offers a lot of features aimed at optimized dataset + access and storage, including compression and partial I/O.

+
+
+ +
+
Dataspace
+
The shape of an array variable. With the exception of degenerate cases + (empty set, singleton), this is a rectilinear lattice or grid of a certain + rank (dimensionality) and extent.
+
+ +
+
Datatype
+
+

An HDF5 datatype consists of an abstract data type (a set of elements) + and a bit-level representation of these elements in storage such as an HDF5 + file or memory.

+

The HDF5 library comes with a large set of predefined datatypes and + offers mechanisms for creating user-defined datatypes.

+

The ten major families or classes of HDF5 datatypes are:

+
    +
  • Integer datatypes
  • +
  • Floating-point number datatypes
  • +
  • String datatypes
  • +
  • Bitfield datatypes
  • +
  • Opaque datatypes
  • +
  • Compound datatypes
  • +
  • Reference datatypes
  • +
  • Enumerated datatypes
  • +
  • Variable-length sequence datatypes
  • +
  • Array datatypes
  • +
+
+
+ +\section GLS_E E + +
+
Enumeration datatype
+
A family of HDF5 datatypes whose elements represent named integer values + called members or enumerators. Currently, only ASCII names are supported.
+
+ +
+
External layout
+
A form of contiguous layout where a dataset's elements are physically + stored in unformatted binary files outside the HDF5 file.
+
+ +
+
External link
+
An HDF5 link whose destination is specified as a pair of an HDF5 file name +and an HDF5 path name in that file.
+
+ +\section GLS_F F + +
+
Field
+
See compound datatype.
+
+ +
+
File
+
+
    +
  1. A byte stream (in a storage context such as a file system or in + memory) formatted according to the HDF5 File Format Specification.
  2. +
  3. A (logical) container for HDF5 objects.
  4. +
+
+
+ +
+
File format
+
+
+ +
+
Fill value
+
+
+ +
+
Filter
+
+
+ +\section GLS_G G + +
+
Group
+
+

A kind of HDF5 object that stores a collection of HDF5 links. Each HDF5 + file contains at least one group, it's root group.

+

Among the destinations of an HDF5 group's links may be other HDF5 groups + (including the group itself!). This ability is sometimes referred to as the + closure property of groups. It is the basis for creating hierarchical or + more general graph-like structures.

+
+
+ +\section GLS_H H + +
+
Hard link
+
An HDF5 link whose destination is specified (internally) as the address of + an HDF5 object in the same HDF5 file.
+
+ +
+
Hierarchy
+
See group.
+
+ +
+
Hyperslab
+
+

A regular multidimensional pattern described by four vectors whose length + equals the rank of the pattern.

+
    +
  1. start - the offset where the first block of the hyperslab begins
  2. +
  3. stride - the offset between pattern blocks
  4. +
  5. count - the number of blocks
  6. +
  7. block - the extent of an individual pattern block
  8. +
+

For example, the black squares on a (two-dimensional) chessboard with + origin at (0,0) can be represented as the union of two + hyperslabs representing the even (0,2,4,6) and + odd (1,3,5,7) rows, respectively.

+ +

The hyperslab parameters for the even rows are: start (0,0), + stride (2,2), count (4,4), block + (1,1). Likewise the parameters for the odd rows are: start + (1,1), stride (2,2), count + (4,4), block (1,1).

+
+
+ +\section GLS_I I + +
+
Identifier
+
An opaque, transient handle used by the HDF5 library to manipulate + in-memory representations of HDF5 items.
+
+ +\section GLS_L L + +
+
Library
+
+
+ +
+
Link
+
+

A named, uni-directional association between a source and a + destination. In HDF5, the source is always the HDF5 group that hosts the + link in its link collection.

+

There are several ways to specify a link's destination:

+
    +
  • The address of an HDF5 object in the same HDF5 file; so-called hard + link.
  • +
  • A path name in the same or a different file; so-called soft or + external link.
  • +
  • User-defined
  • +
+

A link name can be any Unicode string that does not contain slashes + ("/") or consists of a single dot character + ("."). A link name must be unique in a group's link + collection.

+
+
+ +\section GLS_M M + +
+
Metadata
+
Data that in a given context has a descriptive or documentation function + for other data. Typically, the metadata is small compared to the data it + describes.
+
+ +
+
Member
+
+

A link destination is sometimes referred to as a member of the link's + source (group). This way of speaking invites confusion: A destination (e.g., + object) can be the destination of multiple links in the same (!) or + different groups. It would then be a "member" of a given group with + multiplicity greater than one and be a member of multiple groups.

+

It is the link that is a member of the group's link collection and not + the link destination.

+
+
+ +\section GLS_N N + +
+
Name
+
+

A Unicode string that depending on the item it names might be subject to + certain character restrictions, such as ASCII-encoded only. In HDF5, the + user might encounter the following names:

+
    +
  • A link name
  • +
  • A path name
  • +
  • An attribute name
  • +
  • A field name (compound datatypes)
  • +
  • A constant name (enumeration datatypes)
  • +
  • A tag name (opaque datatypes)
  • +
  • A file name
  • +
+
+
+ + +
+
Named datatype
+
See committed datatype.
+
+ +
+
Null dataspace
+
A shape which represents the empty set. Array variables with this shape + cannot store any values.
+
+ +\section GLS_O O + +
+
Object
+
An HDF5 group, dataset or named datatype; an HDF5 item that can be linked + to zero or more groups and decorated with zero or more HDF5 attributes.
+
+ +
+
Object reference
+
+
    +
  1. A datatype for representing references to objects in a file.
  2. +
  3. A value of the object reference datatype.
  4. +
+
+
+ +
+
Opaque datatype
+
A family of HDF5 datatypes whose elements are byte sequences of a given + fixed length. An opaque datatype can be tagged with a sequence of up to 256 + ASCII characters, e.g., MIME code.
+
+ +\section GLS_P P + +
+
Path name
+
A Unicode string that is the concatenation of link names separated by + slashes ('/'). In HDF5, path names are used to locate and refer + to HDF5 objects.
+
+ +
+
Plugin
+
An HDF5 library feature or capability that can be added dynamically at + application run time rather than library compilation time. Plugins are + usually implemented as shared libraries, and their discovery and loading + behavior can be controlled programmatically or through environment + variables. +
+
+ +
+
Point selection
+
A dataspace selection that consists of a set of points (coordinates) in + the same dataspace.
+
+ +
+
Property list
+
+

An HDF5 API construct, a means of customizing the behavior of the HDF5 + library when creating, accessing or modifying HDF5 items.

+

While the default property settings are sufficient in many cases, certain + HDF5 features, such as compression, can be reasonably controlled only by the + user who has to provide the desired settings via property lists.

+
+
+ +\section GLS_R R + +
+
Rank
+
The number of dimensions of a non-null dataspace.
+
+ +
+
Reference
+
+
    +
  1. An HDF5 object reference
  2. +
  3. An HDF5 dataset region reference
  4. +
+
+
+ +
+
Reference datatype
+
+
    +
  1. An HDF5 datatype whose elements represent references to HDF5 + objects.
  2. +
  3. An HDF5 datatype whose elements represent references to regions of an + HDF5 dataset.
  4. +
+
+
+ +
+
Region reference
+
See dataset region reference.
+
+ +
+
Root group
+
+

An HDF5 group that is present in all HDF5 files and that acts as the + entry or base point for all other data stored in an HDF5 file.

+

The root group is "the mother of all objects" in an HDF5 file in the + sense that all objects (and their attributes) can be discovered, + beginning at the root group, by combinations of the following + operations:

+
    +
  • Link traversal
  • +
  • De-referencing of object references
  • +
+

This discovery is portable and robust with respect to file-internal + storage reorganization.

+
+
+ +\section GLS_S S + +
+
Scalar dataspace
+
A kind of HDF5 dataspace that has the shape of a singleton, i.e., a set + containing a single element. Array variables with this shape store exactly one + element.
+
+ +
+
Selection
+
+
    +
  1. A subset of points of an HDF5 dataspace. The subset might be a point + selection or a combination (union, intersection, etc.) of hyperslabs.
  2. +
  3. A subset of dataset elements associated with a dataspace selection as + described under 1.
  4. +
+
+
+ +
+
Serialization
+
+
    +
  1. The flattening of an N-dimensional array into a 1-dimensional + array.
  2. +
  3. The encoding of a complex data item as a linear byte stream.
  4. +
+
+
+ +
+
Soft link
+
A kind of HDF5 link in which the link destination is specified as an HDF5 + path name. The path name may or may not refer to an actual object.
+
+ +
+
Storage layout
+
The storage arrangement for dataset elements, links in a group's link + collection, or attributes in an object's attribute collection.
+
+ +
+
String datatype
+
+
+ +
+
Super block
+
An HDF5 file format primitive; a block of data which contains information + required to access HDF5 files in a portable manner on multiple platforms. The + super block contains information such as version numbers, the size of offsets + and lengths, and the location of the root group.
+
+ +
+
SWMR
+
Single Writer Multiple Reader, a file access mode in which a single + process is permitted to write data to an HDF5 file while other processes are + permitted to read data from the same file without the need of inter-process + communication or synchronization.
+
+ +
+
Symbolic link
+
An external link or a soft link.
+
+ +\section GLS_U U + +
+
User block
+
An HDF5 file format primitive that allows one to set aside a fixed-size + (at least 512 bytes or any power of 2 thereafter) contiguous range of bytes at + the beginning of an HDF5 file for application purposes which will be + skipped/ignored by the HDF5 library.
+
+ +
+
UTF-8
+
+

A variable-length (1-4 bytes per code point) encoding of the Unicode set + of code points. This is the encoding supported by HDF5 to represent Unicode + strings.

+

The ASCII encoding is a proper subset of UTF-8.

+
+
+ +\section GLS_V V + +
+
Variable-length (sequence) datatype
+
A family of HDF5 datatypes whose elements are variable-length sequences of + a given datatype.
+
+ +
+
Virtual Dataset (VDS)
+
An HDF5 dataset with virtual storage layout. A dataset whose elements are + partially or entirely stored physically in other datasets.
+
+ +
+
Virtual File Driver (VFD)
+
+
+ + +
+
Virtual layout
+
+
+ + +
+
Virtual Object Layer (VOL)
+
+
+ +*/ diff --git a/doxygen/dox/H5AC_cache_config_t.dox b/doxygen/dox/H5AC_cache_config_t.dox index 9b9862b..2584aac 100644 --- a/doxygen/dox/H5AC_cache_config_t.dox +++ b/doxygen/dox/H5AC_cache_config_t.dox @@ -26,7 +26,7 @@ * * \Emph{*** DEPRECATED ***} Use \Code{H5Fstart/stop} logging functions instead * - * The trace file is a debuging feature that allow the capture of + * The trace file is a debugging feature that allow the capture of * top level metadata cache requests for purposes of debugging and/or * optimization. This field should normally be set to \c FALSE, as * trace file collection imposes considerable overhead. @@ -82,7 +82,7 @@ * H5C_incr__off ) && ( decr_mode == H5C_decr__off )}). There * is no logical reason why this should be so, but it simplifies * implementation and testing, and I can't think of any reason - * why it would be desireable. If you can think of one, I'll + * why it would be desirable. If you can think of one, I'll * revisit the issue. (JM) * \endparblock * @@ -383,7 +383,7 @@ * to disk.\n * When the sync point is reached (or when there is a user generated * flush), process zero flushes sufficient entries to bring it into - * complience with its min clean size (or flushes all dirty entries in + * compliance with its min clean size (or flushes all dirty entries in * the case of a user generated flush), broad casts the list of * entries just cleaned to all the other processes, and then exits * the sync point.\n @@ -412,4 +412,4 @@ * received from process zero.\n * To avoid possible messages from the past/future, all caches must * wait until all caches are done before leaving the sync point. - */ \ No newline at end of file + */ diff --git a/doxygen/dox/Overview.dox b/doxygen/dox/Overview.dox index 754722e..040769c 100644 --- a/doxygen/dox/Overview.dox +++ b/doxygen/dox/Overview.dox @@ -1,14 +1,12 @@ /** \mainpage notitle -This is the documentation set for HDF5. You can -download it as a tgz archive for offline reading. - -This is the documention set for HDF5 in terms of specifications and software -developed and maintained by The HDF -Group. It is impractical to document the entire HDF5 ecosystem in one place, -and you should also consult the documentation sets of the many outstanding -community projects. +This is the documentation set for HDF5. +It includes specifications and documentation +of software and tools developed and maintained by +The HDF Group. It is impractical to document +the entire HDF5 ecosystem in one place, and you should also consult the documentation +sets of the many outstanding community projects. For a first contact with HDF5, the best place is to have a look at the \link GettingStarted getting started \endlink page that shows you how to write and @@ -19,12 +17,28 @@ technical documentation consists to varying degrees of information related to tasks, concepts, or reference material. As its title suggests, the \link RM Reference Manual \endlink is 100% reference material, while the \link Cookbook \endlink is focused on tasks. The different guide-type -documents cover a mix of tasks, concepts, and reference, to help a certain +documents cover a mix of tasks, concepts, and reference, to help a specific audience succeed. -Finally, do not miss the search engine (top right-hand corner)! If you are -looking for a specific function, it'll take you there directly. If unsure, it'll -give you an idea of what's on offer and a few promising leads. +\par Versions + Version-specific documentation (see the version in the title area) can be found + here: + - HDF5 develop branch (this site) + - HDF5 1.12.x + - HDF5 1.10.x + - HDF5 1.8.x + +\par Search + If you are looking for a specific function, constant, type, etc., use the + search box in the top right-hand corner!\n Otherwise, check out the + \link FTS full-text search\endlink. + +\par Offline reading + You can download it as a tgz archive for offline reading. + +\par History + A snapshot (~April 2017) of the pre-Doxygen HDF5 documentation can be found + here. \par ToDo List There is plenty of unfinished business. diff --git a/doxygen/examples/DebuggingHDF5Applications.html b/doxygen/examples/DebuggingHDF5Applications.html index c6aaf74..3390887 100644 --- a/doxygen/examples/DebuggingHDF5Applications.html +++ b/doxygen/examples/DebuggingHDF5Applications.html @@ -173,7 +173,7 @@ IOT Trap, core dumped. mm Yes - Library memory managment + Library memory management o diff --git a/doxygen/examples/Filters.html b/doxygen/examples/Filters.html index 2d5bc5e..7054a3b 100644 --- a/doxygen/examples/Filters.html +++ b/doxygen/examples/Filters.html @@ -54,7 +54,7 @@

Two types of filters can be applied to raw data I/O: permanent filters and transient filters. The permanent filter pipeline is - defned when the dataset is created while the transient pipeline + defined when the dataset is created while the transient pipeline is defined for each I/O operation. During an H5Dwrite() the transient filters are applied first in the order defined and then the permanent filters are applied @@ -211,7 +211,7 @@ should be turned on so such cases can be handled gracefully by storing the original data instead of the compressed data. The cd_nvalues should be one with cd_value[0] - being a compression agression level between zero and nine, + being a compression aggression level between zero and nine, inclusive (zero is the fastest compression while nine results in the best compression ratio). @@ -252,7 +252,7 @@

The function that acts as the filter always returns zero (failure) if the md5() function was not detected at - configuration time (left as an excercise for the reader). + configuration time (left as an exercise for the reader). Otherwise the function is broken down to an input and output half. The output half calculates a checksum, increases the size of the output buffer if necessary, and appends the checksum to @@ -374,7 +374,7 @@ Method This is the name of the method as defined with - H5Zregister() with the charaters + H5Zregister() with the characters "< or ">" prepended to indicate input or output. diff --git a/doxygen/examples/H5.format.1.0.html b/doxygen/examples/H5.format.1.0.html index 2d3ffbe..d2b6610 100644 --- a/doxygen/examples/H5.format.1.0.html +++ b/doxygen/examples/H5.format.1.0.html @@ -10,12 +10,12 @@ @@ -2414,7 +2414,7 @@ Elena> "Free-space object" @@ -2916,7 +2916,7 @@ Elena> "Free-space object"

The fill value message stores a single data point value which is returned to the application when an uninitialized data point - is read from the dataset. The fill value is interpretted with + is read from the dataset. The fill value is interpreted with the same datatype as the dataset. If no fill value message is present then a fill value of all zero is assumed. @@ -3327,7 +3327,7 @@ Elena> "Free-space object"

-
-
    +
    1. Introduction
    2. Disk Format Level 0 - File Signature and Super Block
    3. Disk Format Level 1 - File Infrastructure -
        +
        1. Disk Format Level 1A - B-link Trees and B-tree Nodes
        2. Disk Format Level 1B - Group
        3. Disk Format Level 1C - Group Entry @@ -26,9 +26,9 @@
        4. Disk Format Level 2 - Data Objects -
            +
            1. Disk Format Level 2a - Data Object Headers -
                +
                1. Name: NIL
                2. Name: Simple Dataspace
                3. Name: Data Storage - External Data Files
                4. Name: Data Storage - Layout @@ -495,7 +495,7 @@ Elena> "Free-space object"
End of File Address This is the relative file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if the free list is not used.
Normalization. The value can be 0 if there is no normalization, 1 if the most significant bit of the mantissa is always set (except for 0.0), and 2 if the most - signficant bit of the mantissa is not stored but is + significant bit of the mantissa is not stored but is implied to be set. The value 3 is reserved and will not appear in this field.
+
@@ -3386,7 +3386,7 @@ Elena> "Free-space object"

-
Filter Pipeline Message
+
diff --git a/doxygen/examples/H5.format.1.1.html b/doxygen/examples/H5.format.1.1.html index ebbbe8e..b91ac90 100644 --- a/doxygen/examples/H5.format.1.1.html +++ b/doxygen/examples/H5.format.1.1.html @@ -36,18 +36,18 @@ TABLE.list TD { border:none; }
Filter Pipeline Message
diff --git a/doxygen/examples/H5.format.2.0.html b/doxygen/examples/H5.format.2.0.html index 3653489..4a5fe37 100644 --- a/doxygen/examples/H5.format.2.0.html +++ b/doxygen/examples/H5.format.2.0.html @@ -821,7 +821,7 @@ II.A. Disk Format: Level 0A - Format Signature and Superblock @@ -7884,7 +7884,7 @@ IV.A.2. Disk Format: Level 2A2 - Data Object Header Messages - diff --git a/doxygen/examples/H5.format.html b/doxygen/examples/H5.format.html index e16805f..cbcb387 100644 --- a/doxygen/examples/H5.format.html +++ b/doxygen/examples/H5.format.html @@ -936,7 +936,7 @@ - @@ -20202,7 +20202,7 @@ disk address for the chunk.

- +
-
    +
    1. Introduction
    2. Disk Format Level 0 - File Metadata -
        +
        1. Disk Format Level 0A - File Signature and Super Block
        2. Disk Format Level 0B - File Driver Info
      1. Disk Format Level 1 - File Infrastructure -
          +
          1. Disk Format Level 1A - B-link Trees and B-tree Nodes
          2. Disk Format Level 1B - Group
          3. Disk Format Level 1C - Group Entry @@ -58,9 +58,9 @@ TABLE.list TD { border:none; }
          4. Disk Format Level 2 - Data Objects -
              +
              1. Disk Format Level 2a - Data Object Headers -
                  +
                  1. Name: NIL
                  2. Name: Simple Dataspace @@ -73,13 +73,13 @@ TABLE.list TD { border:none; }
   -
    +
    1. Disk Format Level 2 - Data Objects (Continued)
      1. Disk Format Level 2a - Data Object Headers(Continued) -
          +
          1. Name: Reserved - not assigned yet
          2. Name: Data Storage - External Data Files @@ -616,7 +616,7 @@ TABLE.list TD { border:none; }

This is the absolute file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if space from the free list is not used.

@@ -3184,7 +3184,7 @@ TABLE.list TD { border:none; }
Normalization. The value can be 0 if there is no normalization, 1 if the most significant bit of the mantissa is always set (except for 0.0), and 2 if the most - signficant bit of the mantissa is not stored but is + significant bit of the mantissa is not stored but is implied to be set. The value 3 is reserved and will not appear in this field.

This is the absolute file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if space from the free list is not used.

@@ -4910,7 +4910,7 @@ III.F. Disk Format: Level 1F - Fractal Heap enough to store objects greater than 16 bytes or not. If the heap ID length is 18 bytes or smaller, the ‘normal’ tiny heap ID form is used. If the heap ID length is greater than 18 bytes in - length, the “extented” form is used. See format description below + length, the “extended” form is used. See format description below for both sub-types.

3Message stored is not shared, but is sharable. + Message stored is not shared, but is shareable.

This is the absolute file address of the first byte past the end of all HDF5 data. It is used to determine whether a - file has been accidently truncated and as an address where + file has been accidentally truncated and as an address where file data allocation can occur if space from the free list is not used.

@@ -8691,7 +8691,7 @@ three rows are needed.
3Message stored is not shared, but is sharable. + Message stored is not shared, but is shareable.

Length fo External File Name

Length of External File Name

This is the length for the external file name.

This field exists if bit 0 of flags is set.

diff --git a/doxygen/examples/ThreadSafeLibrary.html b/doxygen/examples/ThreadSafeLibrary.html index 2ea42b4..97f7742 100644 --- a/doxygen/examples/ThreadSafeLibrary.html +++ b/doxygen/examples/ThreadSafeLibrary.html @@ -165,7 +165,7 @@ The structure is defined in H5private.h as:
-    /* cancelability structure */
+    /* cancellability structure */
     typedef struct H5_cancel_struct {
       int previous_state;
       unsigned int cancel_count;
diff --git a/doxygen/examples/VFL.html b/doxygen/examples/VFL.html
index 9776f96..624d942 100644
--- a/doxygen/examples/VFL.html
+++ b/doxygen/examples/VFL.html
@@ -306,7 +306,7 @@ H5Dread(dataset, type, mspace, fspace, buffer, dxpl);
 

-The transfer propery list can be queried in a manner similar to the file +The transfer property list can be queried in a manner similar to the file access property list: the driver provides a function (or functions) to return various information about the transfer property list: @@ -1210,7 +1210,7 @@ Flush all data for file file to storage.

Example: The sec2 driver doesn't cache any data but it also doesn't -extend the Unix file as agressively as it should. Therefore, when finalizing a +extend the Unix file as aggressively as it should. Therefore, when finalizing a file it should write a zero to the last byte of the allocated region so that when reopening the file later the EOF marker will be at least as large as the EOA marker saved in the superblock (otherwise HDF5 will refuse to open the diff --git a/doxygen/hdf5_navtree_hacks.js b/doxygen/hdf5_navtree_hacks.js index 942970c..dda8984 100644 --- a/doxygen/hdf5_navtree_hacks.js +++ b/doxygen/hdf5_navtree_hacks.js @@ -223,7 +223,7 @@ $(document).ready(function() { (function (){ // wait until the first "selected" element has been created try { - // this line will triger an exception if there is no #selected element, i.e., before the tree structure is complete. + // this line will trigger an exception if there is no #selected element, i.e., before the tree structure is complete. document.getElementById("selected").className = "item selected"; // ok, the default tree has been created, we can keep going... diff --git a/doxygen/hdf5doxy_layout.xml b/doxygen/hdf5doxy_layout.xml index 6efa690..24642b5 100644 --- a/doxygen/hdf5doxy_layout.xml +++ b/doxygen/hdf5doxy_layout.xml @@ -7,7 +7,7 @@ - + diff --git a/doxygen/img/HDF5.png b/doxygen/img/HDF5.png new file mode 100644 index 0000000..0458fa8 Binary files /dev/null and b/doxygen/img/HDF5.png differ diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9ab870f..3f329c1 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -42,6 +42,14 @@ set (examples h5_vds-percival-unlim-maxmin ) +if (H5_HAVE_PARALLEL) + set (parallel_examples + ph5example + ph5_filtered_writes + ph5_filtered_writes_no_sel + ) +endif () + foreach (example ${examples}) add_executable (${example} ${HDF5_EXAMPLES_SOURCE_DIR}/${example}.c) target_include_directories (${example} PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") @@ -63,23 +71,25 @@ foreach (example ${examples}) endforeach () if (H5_HAVE_PARALLEL) - add_executable (ph5example ${HDF5_EXAMPLES_SOURCE_DIR}/ph5example.c) - target_include_directories (ph5example PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") - if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (ph5example STATIC) - target_link_libraries (ph5example PRIVATE ${HDF5_LIB_TARGET} ${MPI_C_LIBRARIES}) - else () - TARGET_C_PROPERTIES (ph5example SHARED) - target_link_libraries (ph5example PRIVATE ${HDF5_LIBSH_TARGET} ${MPI_C_LIBRARIES}) - endif () - set_target_properties (ph5example PROPERTIES FOLDER examples) + foreach (parallel_example ${parallel_examples}) + add_executable (${parallel_example} ${HDF5_EXAMPLES_SOURCE_DIR}/${parallel_example}.c) + target_include_directories (${parallel_example} PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (${parallel_example} STATIC) + target_link_libraries (${parallel_example} PRIVATE ${HDF5_LIB_TARGET} ${MPI_C_LIBRARIES}) + else () + TARGET_C_PROPERTIES (${parallel_example} SHARED) + target_link_libraries (${parallel_example} PRIVATE ${HDF5_LIBSH_TARGET} ${MPI_C_LIBRARIES}) + endif () + set_target_properties (${parallel_example} PROPERTIES FOLDER examples) - #----------------------------------------------------------------------------- - # Add Target to clang-format - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_EXAMPLES_ph5example_FORMAT ph5example) - endif () + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_EXAMPLES_${parallel_example}_FORMAT ${parallel_example}) + endif () + endforeach () endif () if (BUILD_TESTING AND HDF5_TEST_EXAMPLES) diff --git a/examples/CMakeTests.cmake b/examples/CMakeTests.cmake index 70142c8..3e24ba0 100644 --- a/examples/CMakeTests.cmake +++ b/examples/CMakeTests.cmake @@ -101,22 +101,26 @@ if (H5_HAVE_PARALLEL AND HDF5_TEST_PARALLEL AND NOT WIN32) # Ensure that 24 is a multiple of the number of processes. # The number 24 corresponds to SPACE1_DIM1 and SPACE1_DIM2 defined in ph5example.c math(EXPR NUMPROCS "24 / ((24 + ${MPIEXEC_MAX_NUMPROCS} - 1) / ${MPIEXEC_MAX_NUMPROCS})") - if (HDF5_ENABLE_USING_MEMCHECKER) - add_test (NAME MPI_TEST_EXAMPLES-ph5example COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) - else () - add_test (NAME MPI_TEST_EXAMPLES-ph5example COMMAND "${CMAKE_COMMAND}" - -D "TEST_PROGRAM=${MPIEXEC_EXECUTABLE};${MPIEXEC_NUMPROC_FLAG};${NUMPROCS};${MPIEXEC_PREFLAGS};$;${MPIEXEC_POSTFLAGS}" - -D "TEST_ARGS:STRING=" - -D "TEST_EXPECT=0" - -D "TEST_OUTPUT=ph5example.out" - -D "TEST_REFERENCE:STRING=PHDF5 tests finished with no errors" - -D "TEST_FILTER:STRING=PHDF5 tests finished with no errors" - -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" - -P "${HDF_RESOURCES_EXT_DIR}/grepTest.cmake" - ) - endif () - if (last_test) - set_tests_properties (MPI_TEST_EXAMPLES-ph5example PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "MPI_TEST_EXAMPLES-ph5example") + + foreach (parallel_example ${parallel_examples}) + if (HDF5_ENABLE_USING_MEMCHECKER) + add_test (NAME MPI_TEST_EXAMPLES-${parallel_example} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) + else () + add_test (NAME MPI_TEST_EXAMPLES-${parallel_example} COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${MPIEXEC_EXECUTABLE};${MPIEXEC_NUMPROC_FLAG};${NUMPROCS};${MPIEXEC_PREFLAGS};$;${MPIEXEC_POSTFLAGS}" + -D "TEST_ARGS:STRING=" + -D "TEST_EXPECT=0" + -D "TEST_SKIP_COMPARE=TRUE" + -D "TEST_OUTPUT=${parallel_example}.out" + -D "TEST_REFERENCE:STRING=PHDF5 example finished with no errors" + #-D "TEST_FILTER:STRING=PHDF5 tests finished with no errors" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -P "${HDF_RESOURCES_EXT_DIR}/grepTest.cmake" + ) + endif () + if (last_test) + set_tests_properties (MPI_TEST_EXAMPLES-${parallel_example} PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "MPI_TEST_EXAMPLES-${parallel_example}") + endforeach () endif () diff --git a/examples/Makefile.am b/examples/Makefile.am index 7b5aa63..161f789 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -20,7 +20,7 @@ include $(top_srcdir)/config/commence.am if BUILD_PARALLEL_CONDITIONAL - EXAMPLE_PROG_PARA = ph5example + EXAMPLE_PROG_PARA = ph5example ph5_filtered_writes ph5_filtered_writes_no_sel endif INSTALL_SCRIPT_FILES = run-c-ex.sh @@ -50,7 +50,7 @@ INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c h5_compou h5_group.c h5_select.c h5_attribute.c h5_mount.c h5_drivers.c \ h5_reference_deprec.c h5_ref_extern.c h5_ref_compat.c h5_ref2reg_deprec.c \ h5_extlink.c h5_elink_unix2win.c h5_shared_mesg.c h5_debug_trace.c \ - ph5example.c \ + ph5example.c ph5_filtered_writes.c ph5_filtered_writes_no_sel.c \ h5_vds.c h5_vds-exc.c h5_vds-exclim.c h5_vds-eiger.c h5_vds-simpleIO.c \ h5_vds-percival.c h5_vds-percival-unlim.c h5_vds-percival-unlim-maxmin.c @@ -119,6 +119,8 @@ h5_reference_deprec: $(srcdir)/h5_reference_deprec.c h5_ref2reg_deprec: $(srcdir)/h5_ref2reg_deprec.c h5_drivers: $(srcdir)/h5_drivers.c ph5example: $(srcdir)/ph5example.c +ph5_filtered_writes: $(srcdir)/ph5_filtered_writes.c +ph5_filtered_writes_no_sel: $(srcdir)/ph5_filtered_writes_no_sel.c h5_dtransform: $(srcdir)/h5_dtransform.c h5_extlink: $(srcdir)/h5_extlink.c $(EXTLINK_DIRS) h5_elink_unix2win: $(srcdir)/h5_elink_unix2win.c $(EXTLINK_DIRS) diff --git a/examples/gaussians.c b/examples/gaussians.c index 9eed872..f67ea2f 100644 --- a/examples/gaussians.c +++ b/examples/gaussians.c @@ -86,7 +86,7 @@ usage(const char *progname) fprintf(stderr, "usage: %s [-u milliseconds]\n" "\n" - "-c: increase the frame number continously (reader mode)\n" + "-c: increase the frame number continuously (reader mode)\n" "-f: add \"fuzz\" (linear noise) to the data (writer mode)\n" "-u ms: milliseconds interval between updates to %s.h5\n" "\n", diff --git a/examples/h5_drivers.c b/examples/h5_drivers.c index bd57004..d9321ed 100644 --- a/examples/h5_drivers.c +++ b/examples/h5_drivers.c @@ -35,7 +35,7 @@ void cleanup(const char *filename) { if (cleanup_g == -1) - cleanup_g = getenv("HDF5_NOCLEANUP") ? 0 : 1; + cleanup_g = getenv(HDF5_NOCLEANUP) ? 0 : 1; if (cleanup_g) remove(filename); } diff --git a/examples/h5_extlink.c b/examples/h5_extlink.c index f9d4046..61fd589 100644 --- a/examples/h5_extlink.c +++ b/examples/h5_extlink.c @@ -414,14 +414,14 @@ UD_hard_create(const char *link_name, hid_t loc_group, const void *udata, size_t token = *((H5O_token_t *)udata); - //! [H5Open_by_token_snip] + //! [H5Oopen_by_token_snip] /* Open the object this link points to so that we can increment * its reference count. This also ensures that the token passed * in points to a real object (although this check is not perfect!) */ target_obj = H5Oopen_by_token(loc_group, token); - //! [H5Open_by_token_snip] + //! [H5Oopen_by_token_snip] if (target_obj < 0) { ret_value = -1; diff --git a/examples/h5_mount.c b/examples/h5_mount.c index e984d5b..3f087f7 100644 --- a/examples/h5_mount.c +++ b/examples/h5_mount.c @@ -109,7 +109,7 @@ main(void) } /* - * Close all identifers + * Close all identifiers */ H5Tclose(tid); H5Dclose(did); diff --git a/examples/h5_shared_mesg.c b/examples/h5_shared_mesg.c index 0f623b9..f26b360 100644 --- a/examples/h5_shared_mesg.c +++ b/examples/h5_shared_mesg.c @@ -109,7 +109,7 @@ main(void) * This keeps the indexes simple when only a few messages are shared, * but allows them to scale for many messages. If many messages are * deleted from the B-tree, it scales back down into a list. - * A "resonable" setting for maximum list size and minimum btree size + * A "reasonable" setting for maximum list size and minimum btree size * depends on what kinds of messages will be stored in the file. * These numbers are the same for all indexes in a file. * We'll guess at some numbers, though we could just as easily have kept @@ -227,7 +227,7 @@ error: /*------------------------------------------------------------------------- * Function: create_standard_file * - * Purpose: A helper functon for the example. Creates an HDF5 file + * Purpose: A helper function for the example. Creates an HDF5 file * with many repeated messages using the file creation * property list FCPL. * diff --git a/examples/h5_vds-percival-unlim-maxmin.c b/examples/h5_vds-percival-unlim-maxmin.c index 7e029a3..f459c20 100644 --- a/examples/h5_vds-percival-unlim-maxmin.c +++ b/examples/h5_vds-percival-unlim-maxmin.c @@ -152,7 +152,7 @@ main(void) wdata[j] = 10 * (i + 1); /* - * Open the source files and datasets. Appen data to each dataset and + * Open the source files and datasets. Append data to each dataset and * close all resources. */ diff --git a/examples/ph5_filtered_writes.c b/examples/ph5_filtered_writes.c new file mode 100644 index 0000000..0f399dd --- /dev/null +++ b/examples/ph5_filtered_writes.c @@ -0,0 +1,490 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Example of using the parallel HDF5 library to write to datasets + * with filters applied to them. + * + * If the HDF5_NOCLEANUP environment variable is set, the file that + * this example creates will not be removed as the example finishes. + * + * The need of requirement of parallel file prefix is that in general + * the current working directory in which compiling is done, is not suitable + * for parallel I/O and there is no standard pathname for parallel file + * systems. In some cases, the parallel file name may even need some + * parallel file type prefix such as: "pfs:/GF/...". Therefore, this + * example parses the HDF5_PARAPREFIX environment variable for a prefix, + * if one is needed. + */ + +#include + +#include "hdf5.h" + +#if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_PARALLEL_FILTERED_WRITES) + +#define EXAMPLE_FILE "ph5_filtered_writes.h5" +#define EXAMPLE_DSET1_NAME "DSET1" +#define EXAMPLE_DSET2_NAME "DSET2" + +#define EXAMPLE_DSET_DIMS 2 +#define EXAMPLE_DSET_CHUNK_DIM_SIZE 10 + +/* Dataset datatype */ +#define HDF5_DATATYPE H5T_NATIVE_INT +typedef int C_DATATYPE; + +#ifndef PATH_MAX +#define PATH_MAX 512 +#endif + +/* Global variables */ +int mpi_rank, mpi_size; + +/* + * Routine to set an HDF5 filter on the given DCPL + */ +static void +set_filter(hid_t dcpl_id) +{ + htri_t filter_avail; + + /* + * Check if 'deflate' filter is available + */ + filter_avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); + if (filter_avail < 0) + return; + else if (filter_avail) { + /* + * Set 'deflate' filter with reasonable + * compression level on DCPL + */ + H5Pset_deflate(dcpl_id, 6); + } + else { + /* + * Set Fletcher32 checksum filter on DCPL + * since it is always available in HDF5 + */ + H5Pset_fletcher32(dcpl_id); + } +} + +/* + * Routine to fill a data buffer with data. Assumes + * dimension rank is 2 and data is stored contiguous. + */ +void +fill_databuf(hsize_t start[], hsize_t count[], hsize_t stride[], C_DATATYPE *data) +{ + C_DATATYPE *dataptr = data; + hsize_t i, j; + + /* Use MPI rank value for data */ + for (i = 0; i < count[0]; i++) { + for (j = 0; j < count[1]; j++) { + *dataptr++ = mpi_rank; + } + } +} + +/* Cleanup created file */ +static void +cleanup(char *filename) +{ + hbool_t do_cleanup = getenv(HDF5_NOCLEANUP) ? 0 : 1; + + if (do_cleanup) + MPI_File_delete(filename, MPI_INFO_NULL); +} + +/* + * Routine to write to a dataset in a fashion + * where no chunks in the dataset are written + * to by more than 1 MPI rank. This will + * generally give the best performance as the + * MPI ranks will need the least amount of + * inter-process communication. + */ +static void +write_dataset_no_overlap(hid_t file_id, hid_t dxpl_id) +{ + C_DATATYPE data[EXAMPLE_DSET_CHUNK_DIM_SIZE][4 * EXAMPLE_DSET_CHUNK_DIM_SIZE]; + hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; + hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t stride[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + size_t i, j; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t file_dataspace = H5I_INVALID_HID; + + /* + * ------------------------------------ + * Setup Dataset Creation Property List + * ------------------------------------ + */ + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + + /* + * REQUIRED: Dataset chunking must be enabled to + * apply a data filter to the dataset. + * Chunks in the dataset are of size + * EXAMPLE_DSET_CHUNK_DIM_SIZE x EXAMPLE_DSET_CHUNK_DIM_SIZE. + */ + chunk_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + chunk_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + H5Pset_chunk(dcpl_id, EXAMPLE_DSET_DIMS, chunk_dims); + + /* Set filter to be applied to created datasets */ + set_filter(dcpl_id); + + /* + * ------------------------------------ + * Define the dimensions of the dataset + * and create it + * ------------------------------------ + */ + + /* + * Create a dataset composed of 4 chunks + * per MPI rank. The first dataset dimension + * scales according to the number of MPI ranks. + * The second dataset dimension stays fixed + * according to the chunk size. + */ + dataset_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE * mpi_size; + dataset_dims[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + file_dataspace = H5Screate_simple(EXAMPLE_DSET_DIMS, dataset_dims, NULL); + + /* Create the dataset */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET1_NAME, HDF5_DATATYPE, file_dataspace, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT); + + /* + * ------------------------------------ + * Setup selection in the dataset for + * each MPI rank + * ------------------------------------ + */ + + /* + * Each MPI rank's selection covers a + * single chunk in the first dataset + * dimension. Each MPI rank's selection + * covers 4 chunks in the second dataset + * dimension. This leads to each MPI rank + * writing to 4 chunks of the dataset. + */ + start[0] = mpi_rank * EXAMPLE_DSET_CHUNK_DIM_SIZE; + start[1] = 0; + stride[0] = 1; + stride[1] = 1; + count[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + count[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL); + + /* + * -------------------------------------- + * Fill data buffer with MPI rank's rank + * value to make it easy to see which + * part of the dataset each rank wrote to + * -------------------------------------- + */ + + fill_databuf(start, count, stride, &data[0][0]); + + /* + * --------------------------------- + * Write to the dataset collectively + * --------------------------------- + */ + + H5Dwrite(dset_id, HDF5_DATATYPE, H5S_BLOCK, file_dataspace, dxpl_id, data); + + /* + * -------------- + * Close HDF5 IDs + * -------------- + */ + + H5Sclose(file_dataspace); + H5Pclose(dcpl_id); + H5Dclose(dset_id); +} + +/* + * Routine to write to a dataset in a fashion + * where every chunk in the dataset is written + * to by every MPI rank. This will generally + * give the worst performance as the MPI ranks + * will need the most amount of inter-process + * communication. + */ +static void +write_dataset_overlap(hid_t file_id, hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; + hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t stride[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + size_t i, j; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t file_dataspace = H5I_INVALID_HID; + + /* + * ------------------------------------ + * Setup Dataset Creation Property List + * ------------------------------------ + */ + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + + /* + * REQUIRED: Dataset chunking must be enabled to + * apply a data filter to the dataset. + * Chunks in the dataset are of size + * mpi_size x EXAMPLE_DSET_CHUNK_DIM_SIZE. + */ + chunk_dims[0] = mpi_size; + chunk_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + H5Pset_chunk(dcpl_id, EXAMPLE_DSET_DIMS, chunk_dims); + + /* Set filter to be applied to created datasets */ + set_filter(dcpl_id); + + /* + * ------------------------------------ + * Define the dimensions of the dataset + * and create it + * ------------------------------------ + */ + + /* + * Create a dataset composed of N chunks, + * where N is the number of MPI ranks. The + * first dataset dimension scales according + * to the number of MPI ranks. The second + * dataset dimension stays fixed according + * to the chunk size. + */ + dataset_dims[0] = mpi_size * chunk_dims[0]; + dataset_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + + file_dataspace = H5Screate_simple(EXAMPLE_DSET_DIMS, dataset_dims, NULL); + + /* Create the dataset */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET2_NAME, HDF5_DATATYPE, file_dataspace, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT); + + /* + * ------------------------------------ + * Setup selection in the dataset for + * each MPI rank + * ------------------------------------ + */ + + /* + * Each MPI rank's selection covers + * part of every chunk in the first + * dimension. Each MPI rank's selection + * covers all of every chunk in the + * second dimension. This leads to + * each MPI rank writing an equal + * amount of data to every chunk + * in the dataset. + */ + start[0] = mpi_rank; + start[1] = 0; + stride[0] = chunk_dims[0]; + stride[1] = 1; + count[0] = mpi_size; + count[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + + H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL); + + /* + * -------------------------------------- + * Fill data buffer with MPI rank's rank + * value to make it easy to see which + * part of the dataset each rank wrote to + * -------------------------------------- + */ + + data = malloc(mpi_size * EXAMPLE_DSET_CHUNK_DIM_SIZE * sizeof(C_DATATYPE)); + + fill_databuf(start, count, stride, data); + + /* + * --------------------------------- + * Write to the dataset collectively + * --------------------------------- + */ + + H5Dwrite(dset_id, HDF5_DATATYPE, H5S_BLOCK, file_dataspace, dxpl_id, data); + + free(data); + + /* + * -------------- + * Close HDF5 IDs + * -------------- + */ + + H5Sclose(file_dataspace); + H5Pclose(dcpl_id); + H5Dclose(dset_id); +} + +int +main(int argc, char **argv) +{ + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + char * par_prefix = NULL; + char filename[PATH_MAX]; + + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + /* + * ---------------------------------- + * Start parallel access to HDF5 file + * ---------------------------------- + */ + + /* Setup File Access Property List with parallel I/O access */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl_id, comm, info); + + /* + * OPTIONAL: Set collective metadata reads on FAPL to allow + * parallel writes to filtered datasets to perform + * better at scale. While not strictly necessary, + * this is generally recommended. + */ + H5Pset_all_coll_metadata_ops(fapl_id, true); + + /* + * OPTIONAL: Set the latest file format version for HDF5 in + * order to gain access to different dataset chunk + * index types and better data encoding methods. + * While not strictly necessary, this is generally + * recommended. + */ + H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, PATH_MAX, "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", EXAMPLE_FILE); + + /* Create HDF5 file */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + + /* + * -------------------------------------- + * Setup Dataset Transfer Property List + * with collective I/O + * -------------------------------------- + */ + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + + /* + * REQUIRED: Setup collective I/O for the dataset + * write operations. Parallel writes to + * filtered datasets MUST be collective, + * even if some ranks have no data to + * contribute to the write operation. + * + * Refer to the 'ph5_filtered_writes_no_sel' + * example to see how to setup a dataset + * write when one or more MPI ranks have + * no data to contribute to the write + * operation. + */ + H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); + + /* + * -------------------------------- + * Create and write to each dataset + * -------------------------------- + */ + + /* + * Write to a dataset in a fashion where no + * chunks in the dataset are written to by + * more than 1 MPI rank. This will generally + * give the best performance as the MPI ranks + * will need the least amount of inter-process + * communication. + */ + write_dataset_no_overlap(file_id, dxpl_id); + + /* + * Write to a dataset in a fashion where + * every chunk in the dataset is written + * to by every MPI rank. This will generally + * give the worst performance as the MPI ranks + * will need the most amount of inter-process + * communication. + */ + write_dataset_overlap(file_id, dxpl_id); + + /* + * ------------------ + * Close all HDF5 IDs + * ------------------ + */ + + H5Pclose(dxpl_id); + H5Pclose(fapl_id); + H5Fclose(file_id); + + printf("PHDF5 example finished with no errors\n"); + + /* + * ------------------------------------ + * Cleanup created HDF5 file and finish + * ------------------------------------ + */ + + cleanup(filename); + + MPI_Finalize(); + + return 0; +} + +#else + +int +main(void) +{ + printf("HDF5 not configured with parallel support or parallel filtered writes are disabled!\n"); + return 0; +} + +#endif diff --git a/examples/ph5_filtered_writes_no_sel.c b/examples/ph5_filtered_writes_no_sel.c new file mode 100644 index 0000000..2e29838 --- /dev/null +++ b/examples/ph5_filtered_writes_no_sel.c @@ -0,0 +1,370 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Example of using the parallel HDF5 library to collectively write to + * datasets with filters applied to them when one or MPI ranks do not + * have data to contribute to the dataset. + * + * If the HDF5_NOCLEANUP environment variable is set, the file that + * this example creates will not be removed as the example finishes. + * + * The need of requirement of parallel file prefix is that in general + * the current working directory in which compiling is done, is not suitable + * for parallel I/O and there is no standard pathname for parallel file + * systems. In some cases, the parallel file name may even need some + * parallel file type prefix such as: "pfs:/GF/...". Therefore, this + * example parses the HDF5_PARAPREFIX environment variable for a prefix, + * if one is needed. + */ + +#include + +#include "hdf5.h" + +#if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_PARALLEL_FILTERED_WRITES) + +#define EXAMPLE_FILE "ph5_filtered_writes_no_sel.h5" +#define EXAMPLE_DSET_NAME "DSET" + +#define EXAMPLE_DSET_DIMS 2 +#define EXAMPLE_DSET_CHUNK_DIM_SIZE 10 + +/* Dataset datatype */ +#define HDF5_DATATYPE H5T_NATIVE_INT +typedef int C_DATATYPE; + +#ifndef PATH_MAX +#define PATH_MAX 512 +#endif + +/* Global variables */ +int mpi_rank, mpi_size; + +/* + * Routine to set an HDF5 filter on the given DCPL + */ +static void +set_filter(hid_t dcpl_id) +{ + htri_t filter_avail; + + /* + * Check if 'deflate' filter is available + */ + filter_avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); + if (filter_avail < 0) + return; + else if (filter_avail) { + /* + * Set 'deflate' filter with reasonable + * compression level on DCPL + */ + H5Pset_deflate(dcpl_id, 6); + } + else { + /* + * Set Fletcher32 checksum filter on DCPL + * since it is always available in HDF5 + */ + H5Pset_fletcher32(dcpl_id); + } +} + +/* + * Routine to fill a data buffer with data. Assumes + * dimension rank is 2 and data is stored contiguous. + */ +void +fill_databuf(hsize_t start[], hsize_t count[], hsize_t stride[], C_DATATYPE *data) +{ + C_DATATYPE *dataptr = data; + hsize_t i, j; + + /* Use MPI rank value for data */ + for (i = 0; i < count[0]; i++) { + for (j = 0; j < count[1]; j++) { + *dataptr++ = mpi_rank; + } + } +} + +/* Cleanup created file */ +static void +cleanup(char *filename) +{ + hbool_t do_cleanup = getenv(HDF5_NOCLEANUP) ? 0 : 1; + + if (do_cleanup) + MPI_File_delete(filename, MPI_INFO_NULL); +} + +/* + * Routine to write to a dataset in a fashion + * where no chunks in the dataset are written + * to by more than 1 MPI rank. This will + * generally give the best performance as the + * MPI ranks will need the least amount of + * inter-process communication. + */ +static void +write_dataset_some_no_sel(hid_t file_id, hid_t dxpl_id) +{ + C_DATATYPE data[EXAMPLE_DSET_CHUNK_DIM_SIZE][4 * EXAMPLE_DSET_CHUNK_DIM_SIZE]; + hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; + hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t stride[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + hbool_t no_selection; + size_t i, j; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t file_dataspace = H5I_INVALID_HID; + + /* + * ------------------------------------ + * Setup Dataset Creation Property List + * ------------------------------------ + */ + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + + /* + * REQUIRED: Dataset chunking must be enabled to + * apply a data filter to the dataset. + * Chunks in the dataset are of size + * EXAMPLE_DSET_CHUNK_DIM_SIZE x EXAMPLE_DSET_CHUNK_DIM_SIZE. + */ + chunk_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + chunk_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + H5Pset_chunk(dcpl_id, EXAMPLE_DSET_DIMS, chunk_dims); + + /* Set filter to be applied to created datasets */ + set_filter(dcpl_id); + + /* + * ------------------------------------ + * Define the dimensions of the dataset + * and create it + * ------------------------------------ + */ + + /* + * Create a dataset composed of 4 chunks + * per MPI rank. The first dataset dimension + * scales according to the number of MPI ranks. + * The second dataset dimension stays fixed + * according to the chunk size. + */ + dataset_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE * mpi_size; + dataset_dims[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + file_dataspace = H5Screate_simple(EXAMPLE_DSET_DIMS, dataset_dims, NULL); + + /* Create the dataset */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, HDF5_DATATYPE, file_dataspace, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT); + + /* + * ------------------------------------ + * Setup selection in the dataset for + * each MPI rank + * ------------------------------------ + */ + + /* + * Odd rank value MPI ranks do not + * contribute any data to the dataset. + */ + no_selection = (mpi_rank % 2) == 1; + + if (no_selection) { + /* + * MPI ranks not contributing data to + * the dataset should call H5Sselect_none + * on the file dataspace that will be + * passed to H5Dwrite. + */ + H5Sselect_none(file_dataspace); + } + else { + /* + * Even MPI ranks contribute data to + * the dataset. Each MPI rank's selection + * covers a single chunk in the first dataset + * dimension. Each MPI rank's selection + * covers 4 chunks in the second dataset + * dimension. This leads to each contributing + * MPI rank writing to 4 chunks of the dataset. + */ + start[0] = mpi_rank * EXAMPLE_DSET_CHUNK_DIM_SIZE; + start[1] = 0; + stride[0] = 1; + stride[1] = 1; + count[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + count[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL); + + /* + * -------------------------------------- + * Fill data buffer with MPI rank's rank + * value to make it easy to see which + * part of the dataset each rank wrote to + * -------------------------------------- + */ + + fill_databuf(start, count, stride, &data[0][0]); + } + + /* + * --------------------------------- + * Write to the dataset collectively + * --------------------------------- + */ + + H5Dwrite(dset_id, HDF5_DATATYPE, no_selection ? H5S_ALL : H5S_BLOCK, file_dataspace, dxpl_id, data); + + /* + * -------------- + * Close HDF5 IDs + * -------------- + */ + + H5Sclose(file_dataspace); + H5Pclose(dcpl_id); + H5Dclose(dset_id); +} + +int +main(int argc, char **argv) +{ + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + char * par_prefix = NULL; + char filename[PATH_MAX]; + + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + /* + * ---------------------------------- + * Start parallel access to HDF5 file + * ---------------------------------- + */ + + /* Setup File Access Property List with parallel I/O access */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl_id, comm, info); + + /* + * OPTIONAL: Set collective metadata reads on FAPL to allow + * parallel writes to filtered datasets to perform + * better at scale. While not strictly necessary, + * this is generally recommended. + */ + H5Pset_all_coll_metadata_ops(fapl_id, true); + + /* + * OPTIONAL: Set the latest file format version for HDF5 in + * order to gain access to different dataset chunk + * index types and better data encoding methods. + * While not strictly necessary, this is generally + * recommended. + */ + H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, PATH_MAX, "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", EXAMPLE_FILE); + + /* Create HDF5 file */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + + /* + * -------------------------------------- + * Setup Dataset Transfer Property List + * with collective I/O + * -------------------------------------- + */ + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + + /* + * REQUIRED: Setup collective I/O for the dataset + * write operations. Parallel writes to + * filtered datasets MUST be collective, + * even if some ranks have no data to + * contribute to the write operation. + */ + H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); + + /* + * -------------------------------- + * Create and write to the dataset + * -------------------------------- + */ + + /* + * Write to a dataset in a fashion where no + * chunks in the dataset are written to by + * more than 1 MPI rank and some MPI ranks + * have nothing to contribute to the dataset. + * In this case, the MPI ranks that have no + * data to contribute must still participate + * in the collective H5Dwrite call, but should + * call H5Sselect_none on the file dataspace + * passed to the H5Dwrite call. + */ + write_dataset_some_no_sel(file_id, dxpl_id); + + /* + * ------------------ + * Close all HDF5 IDs + * ------------------ + */ + + H5Pclose(dxpl_id); + H5Pclose(fapl_id); + H5Fclose(file_id); + + printf("PHDF5 example finished with no errors\n"); + + /* + * ------------------------------------ + * Cleanup created HDF5 file and finish + * ------------------------------------ + */ + + cleanup(filename); + + MPI_Finalize(); + + return 0; +} + +#else + +int +main(void) +{ + printf("HDF5 not configured with parallel support or parallel filtered writes are disabled!\n"); + return 0; +} + +#endif diff --git a/examples/ph5example.c b/examples/ph5example.c index a5f3c75..36fbfd5 100644 --- a/examples/ph5example.c +++ b/examples/ph5example.c @@ -30,7 +30,7 @@ * for parallel I/O and there is no standard pathname for parallel file * systems. In some cases, the parallel file name may even needs some * parallel file type prefix such as: "pfs:/GF/...". Therefore, this - * example requires an explicite parallel file prefix. See the usage + * example requires an explicit parallel file prefix. See the usage * for more detail. */ @@ -898,7 +898,7 @@ usage(void) { printf("Usage: testphdf5 [-f ] [-r] [-w] [-v]\n"); printf("\t-f\tfile prefix for parallel test files.\n"); - printf("\t \te.g. pfs:/PFS/myname\n"); + printf("\t \t e.g. pfs:/PFS/myname\n"); printf("\t \tcan be set via $" PARAPREFIX ".\n"); printf("\t \tDefault is current directory.\n"); printf("\t-c\tno cleanup\n"); @@ -934,7 +934,7 @@ mkfilenames(char *prefix) "Need to adjust the code to accommodate the large size.\n"); } for (i = 0; i < n; i++) { - sprintf(testfiles[i], "%s/ParaEg%d.h5", prefix, i); + snprintf(testfiles[i], PATH_MAX, "%s/ParaEg%d.h5", prefix, i); } return (0); } @@ -1073,11 +1073,11 @@ main(int argc, char **argv) finish: if (mpi_rank == 0) { /* only process 0 reports */ if (nerrors) - printf("***PHDF5 tests detected %d errors***\n", nerrors); + printf("***PHDF5 example detected %d errors***\n", nerrors); else { - printf("===================================\n"); - printf("PHDF5 tests finished with no errors\n"); - printf("===================================\n"); + printf("=====================================\n"); + printf("PHDF5 example finished with no errors\n"); + printf("=====================================\n"); } } if (docleanup) diff --git a/examples/testh5cc.sh.in b/examples/testh5cc.sh.in index ba80f2d..a8974bf 100644 --- a/examples/testh5cc.sh.in +++ b/examples/testh5cc.sh.in @@ -80,7 +80,7 @@ applib=libapp${H5TOOL}.a # short hands # Caution: if some *.h5 files must be cleaned here, list them by names. # Don't use the wildcard form of *.h5 as it will wipe out even *.h5 generated -# by otehr test programs. This will cause a racing condition error when +# by other test programs. This will cause a racing condition error when # parallel make (e.g., gmake -j 4) is used. temp_SRC="$hdf5main $v16main $v18main $v110main $v112main $appmain $prog1 $prog2" temp_OBJ=`echo $temp_SRC | sed -e "s/\.${suffix}/.o/g"` diff --git a/fortran/examples/compound.f90 b/fortran/examples/compound.f90 index d9f8235..57624c6 100644 --- a/fortran/examples/compound.f90 +++ b/fortran/examples/compound.f90 @@ -41,7 +41,7 @@ INTEGER(HID_T) :: dt3_id ! Memory datatype identifier (for double precision field) INTEGER(HID_T) :: dt4_id ! Memory datatype identifier (for real field) INTEGER(HID_T) :: dt5_id ! Memory datatype identifier - INTEGER(HID_T) :: plist_id ! Dataset trasfer property + INTEGER(HID_T) :: plist_id ! Dataset transfer property INTEGER(SIZE_T) :: typesize @@ -111,21 +111,21 @@ type_size = type_sizec + type_sizei + type_sized + type_sizer CALL h5tcreate_f(H5T_COMPOUND_F, type_size, dtype_id, error) ! - ! Insert memebers + ! Insert members ! - ! CHARACTER*2 memeber + ! CHARACTER*2 member ! offset = 0 CALL h5tinsert_f(dtype_id, "char_field", offset, dt5_id, error) ! ! INTEGER member ! - offset = offset + type_sizec ! Offset of the second memeber is 2 + offset = offset + type_sizec ! Offset of the second member is 2 CALL h5tinsert_f(dtype_id, "integer_field", offset, H5T_NATIVE_INTEGER, error) ! ! DOUBLE PRECISION member ! - offset = offset + type_sizei ! Offset of the third memeber is 6 + offset = offset + type_sizei ! Offset of the third member is 6 CALL h5tinsert_f(dtype_id, "double_field", offset, H5T_NATIVE_DOUBLE, error) ! ! REAL member @@ -198,7 +198,7 @@ ! CALL h5dopen_f(file_id, dsetname, dset_id, error) ! - ! Create memeory datatyoe to read character member of the compound datatype. + ! Create memory datatype to read character member of the compound datatype. ! CALL h5tcopy_f(H5T_NATIVE_CHARACTER, dt2_id, error) typesize = 2 @@ -208,7 +208,7 @@ offset = 0 CALL h5tinsert_f(dt1_id, "char_field", offset, dt2_id, error) ! - ! Read part of the datatset and display it. + ! Read part of the dataset and display it. ! CALL h5dread_f(dset_id, dt1_id, char_member_out, data_dims, error) write(*,*) (char_member_out(i), i=1, dimsize) diff --git a/fortran/examples/h5_cmprss.f90 b/fortran/examples/h5_cmprss.f90 index 61efc30..86d9cfa 100644 --- a/fortran/examples/h5_cmprss.f90 +++ b/fortran/examples/h5_cmprss.f90 @@ -51,7 +51,7 @@ PROGRAM h5_cmprss !INTEGER :: szip_pixels_per_block ! - !Initialize FORTRAN predifined datatypes + !Initialize FORTRAN predefined datatypes ! CALL h5open_f(error) ! diff --git a/fortran/examples/h5_crtatt.f90 b/fortran/examples/h5_crtatt.f90 index 996a506..64d2bde 100644 --- a/fortran/examples/h5_crtatt.f90 +++ b/fortran/examples/h5_crtatt.f90 @@ -35,7 +35,7 @@ PROGRAM H5_CRTATT INTEGER(HID_T) :: aspace_id ! Attribute Dataspace identifier INTEGER(HID_T) :: atype_id ! Attribute Dataspace identifier INTEGER(HSIZE_T), DIMENSION(1) :: adims = (/2/) ! Attribute dimension - INTEGER :: arank = 1 ! Attribure rank + INTEGER :: arank = 1 ! Attribute rank INTEGER(SIZE_T) :: attrlen ! Length of the attribute string CHARACTER(LEN=80), DIMENSION(2) :: attr_data ! Attribute data diff --git a/fortran/examples/h5_extend.f90 b/fortran/examples/h5_extend.f90 index 0ad6ef9..193f33e 100644 --- a/fortran/examples/h5_extend.f90 +++ b/fortran/examples/h5_extend.f90 @@ -86,7 +86,7 @@ PROGRAM H5_EXTEND INTEGER, DIMENSION(1:3,1:10) :: rdata ! - !Initialize FORTRAN predifined datatypes + !Initialize FORTRAN predefined datatypes ! CALL h5open_f(error) @@ -110,7 +110,7 @@ PROGRAM H5_EXTEND CALL h5pset_chunk_f(crp_list, RANK, dimsc, error) ! - !Create a dataset with 3X3 dimensions using cparms creation propertie . + !Create a dataset with 3X3 dimensions using cparms creation properties . ! CALL h5dcreate_f(file_id, dsetname, H5T_NATIVE_INTEGER, dataspace, & dset_id, error, crp_list ) diff --git a/fortran/examples/refobjexample.f90 b/fortran/examples/refobjexample.f90 index d420192..873b420 100644 --- a/fortran/examples/refobjexample.f90 +++ b/fortran/examples/refobjexample.f90 @@ -26,7 +26,7 @@ IMPLICIT NONE CHARACTER(LEN=10), PARAMETER :: filename = "FORTRAN.h5" ! File CHARACTER(LEN=8), PARAMETER :: dsetnamei = "INTEGERS" ! Dataset with the integer data - CHARACTER(LEN=17), PARAMETER :: dsetnamer = "OBJECT_REFERENCES" ! Dataset wtih object + CHARACTER(LEN=17), PARAMETER :: dsetnamer = "OBJECT_REFERENCES" ! Dataset with object ! references CHARACTER(LEN=6), PARAMETER :: groupname1 = "GROUP1" ! Groups in the file CHARACTER(LEN=6), PARAMETER :: groupname2 = "GROUP2" ! diff --git a/fortran/examples/testh5fc.sh.in b/fortran/examples/testh5fc.sh.in index db687d8..71f41f8 100644 --- a/fortran/examples/testh5fc.sh.in +++ b/fortran/examples/testh5fc.sh.in @@ -61,7 +61,7 @@ applib=libapp${H5TOOL}.a # short hands # Caution: if some *.h5 files must be cleaned here, list them by names. # Don't use the wildcard form of *.h5 as it will wipe out even *.h5 generated -# by otehr test programs. This will cause a racing condition error when +# by other test programs. This will cause a racing condition error when # parallel make (e.g., gmake -j 4) is used. temp_SRC="$hdf5main $appmain $prog1 $prog2" temp_OBJ=`echo $temp_SRC | sed -e "s/\.${suffix}/.o/g"` diff --git a/fortran/src/H5Af.c b/fortran/src/H5Af.c index f78ade2..db1c040 100644 --- a/fortran/src/H5Af.c +++ b/fortran/src/H5Af.c @@ -272,7 +272,7 @@ done: * PURPOSE * Call H5Aopen to open an attribute * INPUTS - * obj_id - Identifer for object to which attribute is attached + * obj_id - Identifier for object to which attribute is attached * attr_name - Attribute access property list * attr_namelen - size of attr_name * aapl_id - Link access property list @@ -317,7 +317,7 @@ done: * PURPOSE * Call h5adelete_by_name to remove an attribute from a specified location * INPUTS - * loc_id - identifer for object to which attribute is attached + * loc_id - identifier for object to which attribute is attached * obj_name - object identifier * obj_namelen - name length * attr_name - name of the attribute @@ -424,7 +424,7 @@ done: * INPUTS * * - * loc_id - Identifer for object to which attribute is attached + * loc_id - Identifier for object to which attribute is attached * obj_name - Name of object, relative to location, * from which attribute is to be removed *TEST* check NULL * idx_type - Type of index; Possible values are: @@ -532,7 +532,7 @@ done: * aapl_id - Attribute access property list * lapl_id - Link access property list * OUTPUTS - * attr_id - attribute identifer + * attr_id - attribute identifier * RETURNS * 0 on success, -1 on failure * AUTHOR diff --git a/fortran/src/H5Aff.F90 b/fortran/src/H5Aff.F90 index 8f56993..53f0a39 100644 --- a/fortran/src/H5Aff.F90 +++ b/fortran/src/H5Aff.F90 @@ -211,7 +211,7 @@ CONTAINS ! ! INPUTS ! obj_id - identifier of a group, dataset, or named -! datatype atttribute to be attached to +! datatype attribute to be attached to ! name - attribute name ! OUTPUTS ! attr_id - attribute identifier @@ -487,7 +487,7 @@ CONTAINS SUBROUTINE h5aget_name_by_idx_f(loc_id, obj_name, idx_type, order, & n, name, hdferr, size, lapl_id) IMPLICIT NONE - INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifer for object to which attribute is attached + INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifier for object to which attribute is attached CHARACTER(LEN=*), INTENT(IN) :: obj_name ! Name of object, relative to location, ! from which attribute is to be removed *TEST* check NULL INTEGER, INTENT(IN) :: idx_type ! Type of index; Possible values are: @@ -846,7 +846,7 @@ CONTAINS ! identifier and attribute name ! ! INPUTS -! obj_id - Identifer for object to which attribute is attached +! obj_id - Identifier for object to which attribute is attached ! attr_name - Name of attribute to open ! OUTPUTS ! attr_id - attribute identifier @@ -934,7 +934,7 @@ CONTAINS ! SOURCE SUBROUTINE H5Adelete_by_idx_f(loc_id, obj_name, idx_type, order, n, hdferr, lapl_id) IMPLICIT NONE - INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifer for object to which attribute is attached + INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifier for object to which attribute is attached CHARACTER(LEN=*), INTENT(IN) :: obj_name ! Name of object, relative to location, ! from which attribute is to be removed INTEGER, INTENT(IN) :: idx_type ! Type of index; Possible values are: @@ -991,7 +991,7 @@ CONTAINS ! Removes an attribute from a specified location ! ! INPUTS -! loc_id - Identifer for object to which attribute is attached +! loc_id - Identifier for object to which attribute is attached ! obj_name - Name of attribute to open ! attr_name - Attribute access property list ! lapl_id - Link access property list @@ -1005,7 +1005,7 @@ CONTAINS ! SOURCE SUBROUTINE H5Adelete_by_name_f(loc_id, obj_name, attr_name, hdferr, lapl_id) IMPLICIT NONE - INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifer for object to which attribute is attached + INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifier for object to which attribute is attached CHARACTER(LEN=*), INTENT(IN) :: obj_name ! Name of object, relative to location, ! from which attribute is to be removed CHARACTER(LEN=*), INTENT(IN) :: attr_name ! Name of attribute to delete @@ -1841,7 +1841,7 @@ CONTAINS ! dims parameter was added to make code portable; ! Aprile 4, 2001 ! -! Changed buf intent to INOUT to be consistant +! Changed buf intent to INOUT to be consistent ! with how the C functions handles it. The pg ! compiler will return 0 if a buf value is not set. ! February, 2008 diff --git a/fortran/src/H5Df.c b/fortran/src/H5Df.c index bcf792c..a780683 100644 --- a/fortran/src/H5Df.c +++ b/fortran/src/H5Df.c @@ -145,7 +145,7 @@ DONE: * Elena Pourmal * Tuesday, May 14, 2002 * HISTORY - * This function was added to accomodate h5dwrite_f with the + * This function was added to accommodate h5dwrite_f with the * dims argument being of INTEGER(HSIZE_T) type * SOURCE */ @@ -217,7 +217,7 @@ h5dwrite_ref_reg_c(hid_t_f *dset_id, hid_t_f *mem_type_id, hid_t_f *mem_space_id * Elena Pourmal * Wednesday, May 15, 2002 * HISTORY - * This function was added to accomodate h5dread_f subroutine + * This function was added to accommodate h5dread_f subroutine * with the dims parameter being of INTEGER(HSIZE_T_F) size. * SOURCE */ @@ -495,10 +495,10 @@ h5dget_storage_size_c(hid_t_f *dset_id, hsize_t_f *size) c_dset_id = (hid_t)*dset_id; c_size = H5Dget_storage_size(c_dset_id); - if (c_size == 0) - return ret_value; - *size = (hsize_t_f)c_size; - ret_value = 0; + if (c_size != 0) { + ret_value = 0; + } + *size = (hsize_t_f)c_size; return ret_value; } @@ -783,7 +783,7 @@ h5dwrite_vl_string_c(hid_t_f *dset_id, hid_t_f *mem_type_id, hid_t_f *mem_space_ return ret_value; } /* - * Move data from temorary buffer + * Move data from temporary buffer */ tmp_p = tmp; for (i = 0; i < num_elem; i++) { diff --git a/fortran/src/H5Dff.F90 b/fortran/src/H5Dff.F90 index 75af58e..6b77a8c 100644 --- a/fortran/src/H5Dff.F90 +++ b/fortran/src/H5Dff.F90 @@ -476,7 +476,7 @@ CONTAINS ! ! Changed name from the now obsolete h5dextend_f ! to h5dset_extent_f. Provided interface to old name -! for backward compatability. -MSB- March 14, 2008 +! for backward compatibility. -MSB- March 14, 2008 ! ! SOURCE SUBROUTINE h5dset_extent_f(dataset_id, size, hdferr) @@ -1195,7 +1195,7 @@ CONTAINS IMPLICIT NONE INTEGER(HID_T), INTENT(IN) :: dset_id ! Dataset identifier INTEGER(HID_T), INTENT(IN) :: mem_type_id ! Memory datatype identifier - INTEGER(HSIZE_T), DIMENSION(*), INTENT(IN) :: dims ! size of the bufffer buf + INTEGER(HSIZE_T), DIMENSION(*), INTENT(IN) :: dims ! size of the buffer buf TYPE(hobj_ref_t_f), DIMENSION(dims(1)), INTENT(IN), TARGET :: buf ! Data buffer INTEGER, INTENT(OUT) :: hdferr ! Error code INTEGER(HID_T), OPTIONAL, INTENT(IN) :: mem_space_id ! Memory dataspace identfier @@ -1227,7 +1227,7 @@ CONTAINS IMPLICIT NONE INTEGER(HID_T), INTENT(IN) :: dset_id ! Dataset identifier INTEGER(HID_T), INTENT(IN) :: mem_type_id ! Memory datatype identifier - INTEGER(HSIZE_T), DIMENSION(*), INTENT(IN) :: dims ! size of the bufffer buf + INTEGER(HSIZE_T), DIMENSION(*), INTENT(IN) :: dims ! size of the buffer buf TYPE(hdset_reg_ref_t_f), DIMENSION(dims(1)), INTENT(IN), TARGET :: buf ! Data buffer INTEGER, INTENT(OUT) :: hdferr ! Error code INTEGER(HID_T), OPTIONAL, INTENT(IN) :: mem_space_id ! Memory dataspace identfier @@ -1625,8 +1625,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier -! buf - data buffer iin memory ro apply selection to -! - of k-th dimension of the buf array +! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 @@ -1676,8 +1675,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier -! buf - data buffer iin memory ro apply selection to -! - of k-th dimension of the buf array +! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 @@ -1724,8 +1722,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier - ! buf - data buffer iin memory ro apply selection to - ! - of k-th dimension of the buf array + ! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 @@ -1799,8 +1796,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier -! buf - data buffer iin memory ro apply selection to -! - of k-th dimension of the buf array +! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 diff --git a/fortran/src/H5Ff.c b/fortran/src/H5Ff.c index f943200..339f8b7 100644 --- a/fortran/src/H5Ff.c +++ b/fortran/src/H5Ff.c @@ -583,7 +583,7 @@ h5fget_name_c(hid_t_f *obj_id, size_t_f *size, _fcd buf, size_t_f *buflen) int_f ret_value = 0; /* Return value */ /* - * Allocate buffer to hold name of an attribute + * Allocate buffer to hold name of file */ if (NULL == (c_buf = (char *)HDmalloc((size_t)*buflen + 1))) HGOTO_DONE(FAIL); @@ -591,7 +591,7 @@ h5fget_name_c(hid_t_f *obj_id, size_t_f *size, _fcd buf, size_t_f *buflen) /* * Call H5Fget_name function */ - if ((size_c = H5Fget_name((hid_t)*obj_id, c_buf, (size_t)*buflen)) < 0) + if ((size_c = H5Fget_name((hid_t)*obj_id, c_buf, (size_t)*buflen + 1)) < 0) HGOTO_DONE(FAIL); /* diff --git a/fortran/src/H5Fff.F90 b/fortran/src/H5Fff.F90 index a273431..ecb40b7 100644 --- a/fortran/src/H5Fff.F90 +++ b/fortran/src/H5Fff.F90 @@ -97,7 +97,7 @@ CONTAINS INTEGER(HID_T), INTENT(OUT) :: file_id ! File identifier INTEGER, INTENT(OUT) :: hdferr ! Error code INTEGER(HID_T), OPTIONAL, INTENT(IN) :: creation_prp - ! File creation propertly + ! File creation property ! list identifier INTEGER(HID_T), OPTIONAL, INTENT(IN) :: access_prp ! File access property list diff --git a/fortran/src/H5Gf.c b/fortran/src/H5Gf.c index f445fee..0201761 100644 --- a/fortran/src/H5Gf.c +++ b/fortran/src/H5Gf.c @@ -390,13 +390,13 @@ DONE: * Call H5Glink2 to link the specified type * INPUTS * cur_loc_id - identifier of file or group - * cur_name - name of the existing object for hard link releative + * cur_name - name of the existing object for hard link relative * to cur_loc_id location, * anything for the soft link * current_namelen - current name length * link_type - link type * new_loc_id - location identifier - * new_name - new name for the object releative to the new_loc_id + * new_name - new name for the object relative to the new_loc_id * location * new_namelen - new_name length * RETURNS @@ -879,7 +879,7 @@ done: * February 15, 2008 * HISTORY * - * - Added 'mounted' paramater + * - Added 'mounted' parameter * M. Scot Breitenfeld * July 16, 2008 * SOURCE @@ -1014,7 +1014,7 @@ done: * February 18, 2008 * HISTORY * - * - Added 'mounted' paramater + * - Added 'mounted' parameter * M. Scot Breitenfeld * July 16, 2008 * SOURCE diff --git a/fortran/src/H5Gff.F90 b/fortran/src/H5Gff.F90 index 9e7665f..bfca595 100644 --- a/fortran/src/H5Gff.F90 +++ b/fortran/src/H5Gff.F90 @@ -73,7 +73,7 @@ CONTAINS ! called C functions (it is needed for Windows ! port). March 5, 2001 ! -! Added additional optional paramaters in 1.8 +! Added additional optional parameters in 1.8 ! MSB - February 27, 2008 ! ! SOURCE @@ -491,8 +491,8 @@ CONTAINS INTEGER, INTENT(OUT) :: hdferr ! Error code !***** - INTEGER :: current_namelen ! Lenghth of the current_name string - INTEGER :: new_namelen ! Lenghth of the new_name string + INTEGER :: current_namelen ! Length of the current_name string + INTEGER :: new_namelen ! Length of the new_name string INTERFACE INTEGER FUNCTION h5glink_c(loc_id, link_type, current_name, & @@ -524,7 +524,7 @@ CONTAINS ! PURPOSE ! Creates a link of the specified type from new_name ! to current_name. current_name and new_name are interpreted -! releative to current and new location identifiers. +! relative to current and new location identifiers. ! ! INPUTS ! cur_loc_id - location identifier @@ -559,8 +559,8 @@ CONTAINS INTEGER, INTENT(OUT) :: hdferr ! Error code !***** - INTEGER :: cur_namelen ! Lenghth of the current_name string - INTEGER :: new_namelen ! Lenghth of the new_name string + INTEGER :: cur_namelen ! Length of the current_name string + INTEGER :: new_namelen ! Length of the new_name string INTERFACE INTEGER FUNCTION h5glink2_c(cur_loc_id, cur_name, cur_namelen, & @@ -617,7 +617,7 @@ CONTAINS CHARACTER(LEN=*), INTENT(IN) :: name ! Name of an object INTEGER, INTENT(OUT) :: hdferr ! Error code !***** - INTEGER :: namelen ! Lenghth of the name character string + INTEGER :: namelen ! Length of the name character string INTERFACE INTEGER FUNCTION h5gunlink_c(loc_id, name, namelen) BIND(C,NAME='h5gunlink_c') @@ -666,8 +666,8 @@ CONTAINS CHARACTER(LEN=*), INTENT(IN) :: new_name ! New name of an object INTEGER, INTENT(OUT) :: hdferr ! Error code !***** - INTEGER :: namelen ! Lenghth of the current_name string - INTEGER :: new_namelen ! Lenghth of the new_name string + INTEGER :: namelen ! Length of the current_name string + INTEGER :: new_namelen ! Length of the new_name string INTERFACE INTEGER FUNCTION h5gmove_c(loc_id, name, namelen, new_name, new_namelen) BIND(C,NAME='h5gmove_c') @@ -715,7 +715,7 @@ CONTAINS INTEGER, INTENT(OUT) :: hdferr ! Error code !***** INTEGER :: src_namelen ! Length of the current_name string - INTEGER :: dst_namelen ! Lenghth of the new_name string + INTEGER :: dst_namelen ! Length of the new_name string INTERFACE INTEGER FUNCTION h5gmove2_c(src_loc_id, src_name, src_namelen, & @@ -776,7 +776,7 @@ CONTAINS ! points to INTEGER, INTENT(OUT) :: hdferr ! Error code !***** - INTEGER :: namelen ! Lenghth of the current_name string + INTEGER :: namelen ! Length of the current_name string INTERFACE INTEGER FUNCTION h5gget_linkval_c(loc_id, name, namelen, size, buffer) BIND(C,NAME='h5gget_linkval_c') @@ -827,8 +827,8 @@ CONTAINS CHARACTER(LEN=*), INTENT(IN) :: comment ! New name of an object INTEGER, INTENT(OUT) :: hdferr ! Error code !***** - INTEGER :: namelen ! Lenghth of the current_name string - INTEGER :: commentlen ! Lenghth of the comment string + INTEGER :: namelen ! Length of the current_name string + INTEGER :: commentlen ! Length of the comment string INTERFACE INTEGER FUNCTION h5gset_comment_c(loc_id, name, namelen, & @@ -1024,7 +1024,7 @@ CONTAINS ! ! HISTORY ! -! - Added 'mounted' paramater +! - Added 'mounted' parameter ! M. Scot Breitenfeld ! July 16, 2008 ! @@ -1104,7 +1104,7 @@ CONTAINS ! February 18, 2008 ! ! HISTORY -! Added 'mounted' paramater +! Added 'mounted' parameter ! M. Scot Breitenfeld ! July 16, 2008 ! @@ -1206,7 +1206,7 @@ CONTAINS ! February 18, 2008 ! ! HISTORY -! Added 'mounted' paramater +! Added 'mounted' parameter ! M. Scot Breitenfeld ! July 16, 2008 ! SOURCE diff --git a/fortran/src/H5Lff.F90 b/fortran/src/H5Lff.F90 index 3bfcf65..a568824 100644 --- a/fortran/src/H5Lff.F90 +++ b/fortran/src/H5Lff.F90 @@ -469,7 +469,7 @@ CONTAINS ! SOURCE SUBROUTINE h5ldelete_by_idx_f(loc_id, group_name, index_field, order, n, hdferr, lapl_id) IMPLICIT NONE - INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifer for object to which attribute is attached + INTEGER(HID_T), INTENT(IN) :: loc_id ! Identifier for object to which attribute is attached CHARACTER(LEN=*), INTENT(IN) :: group_name ! Name of object, relative to location, ! from which attribute is to be removed INTEGER, INTENT(IN) :: index_field ! Type of index; Possible values are: @@ -1315,7 +1315,7 @@ CONTAINS ! Inputs: ! group_id - Identifier specifying subject group ! index_type - Type of index which determines the order: -! H5_INDEX_NAME_F - Alpha-numeric index on name +! H5_INDEX_NAME_F - Alphanumeric index on name ! H5_INDEX_CRT_ORDER_F - Index on creation order ! order - Order within index: ! H5_ITER_INC_F - Increasing order @@ -1391,7 +1391,7 @@ CONTAINS ! loc_id - File or group identifier specifying location of subject group ! group_name - Name of subject group ! index_type - Type of index which determines the order: -! H5_INDEX_NAME_F - Alpha-numeric index on name +! H5_INDEX_NAME_F - Alphanumeric index on name ! H5_INDEX_CRT_ORDER_F - Index on creation order ! order - Order within index: ! H5_ITER_INC_F - Increasing order diff --git a/fortran/src/H5Pf.c b/fortran/src/H5Pf.c index 1f74955..ab09ab9 100644 --- a/fortran/src/H5Pf.c +++ b/fortran/src/H5Pf.c @@ -145,7 +145,7 @@ done: * plist1_id - property list identifier * plist2_id - property list identifier * OUTPUTS - * c_flag - flag to indicate that lists are eqaul + * c_flag - flag to indicate that lists are equal * RETURNS * 0 on success, -1 on failure * AUTHOR @@ -936,7 +936,7 @@ h5pset_fapl_stdio_c(hid_t_f *prp_id) * NAME * h5pget_fapl_stdio_c * PURPOSE - * Call H5Pget_fapl_stdio to determine whther the low level file driver + * Call H5Pget_fapl_stdio to determine whether the low level file driver * uses the functions declared in the stdio.h * INPUTS * prp_id - property list identifier @@ -1013,7 +1013,7 @@ h5pset_fapl_sec2_c(hid_t_f *prp_id) * NAME * h5pget_fapl_sec2_c * PURPOSE - * Call H5Pget_fapl_stdio to determine whther the low level file driver + * Call H5Pget_fapl_stdio to determine whether the low level file driver * uses the functions declared in the unistd.h * INPUTS * prp_id - property list identifier @@ -2186,7 +2186,7 @@ h5pset_fclose_degree_c(hid_t_f *fapl_id, int_f *degree) * PURPOSE * Call H5Pset_buffer to set size of conversion buffer * INPUTS - * prp_id - t`dataset trasfer property list identifier + * prp_id - t`dataset transfer property list identifier * size - size of the buffer * OUTPUTS * NONE @@ -2221,7 +2221,7 @@ h5pset_buffer_c(hid_t_f *prp_id, hsize_t_f *size) * PURPOSE * Call H5Pget_buffer to get size of conversion buffer * INPUTS - * prp_id - t`dataset trasfer property list identifier + * prp_id - t`dataset transfer property list identifier * OUTPUTS * size - size of conversion buffer * RETURNS @@ -3005,11 +3005,11 @@ h5pget_nprops_c(hid_t_f *plist, size_t_f *nprops) * h5pget_class_parent_c * PURPOSE * Call H5Pget_class_parent to get the parent class of - * a genereic property class + * a generic property class * INPUTS * prp_id - property list to query * OUTPUTS - * parent_id - parent classs identifier + * parent_id - parent class identifier * RETURNS * 0 on success, -1 on failure * AUTHOR @@ -3505,7 +3505,7 @@ h5pset_family_offset_c(hid_t_f *prp_id, hsize_t_f *offset) * NAME * h5pset_fapl_multi_c * PURPOSE - * Call H5Pset_fapl_multi to set multi file dirver + * Call H5Pset_fapl_multi to set multi file driver * INPUTS * prp_id - file_creation property list identifier * mem_map - memory mapping array @@ -3570,7 +3570,7 @@ h5pset_fapl_multi_c(hid_t_f *prp_id, int_f *memb_map, hid_t_f *memb_fapl, _fcd m tmp_p = tmp_p + c_lenmax; } /* - * Take care of othe arguments + * Take care of other arguments */ tmp_max_addr = (long double)(HADDR_MAX); c_prp_id = (hid_t)*prp_id; @@ -3603,7 +3603,7 @@ DONE: * NAME * h5pset_fapl_multi_sc * PURPOSE - * Call H5Pset_fapl_multi to set multi file dirver + * Call H5Pset_fapl_multi to set multi file driver * INPUTS * prp_id - file_creation property list identifier * RETURNS @@ -3641,7 +3641,7 @@ h5pset_fapl_multi_sc(hid_t_f *prp_id, int_f *flag) * NAME * h5pget_fapl_multi_c * PURPOSE - * Call H5Pget_fapl_multi to set multi file dirver + * Call H5Pget_fapl_multi to set multi file driver * INPUTS * prp_id - file_creation property list identifier * lenmax - length of the name a sdeclared in Fortran @@ -4607,7 +4607,7 @@ h5pget_copy_object_c(hid_t_f *ocp_plist_id, int_f *copy_options) * INPUTS * * prp_id - property list identifier to query - * expression_len - buffer size transorm expression + * expression_len - buffer size transform expression * * Output: * expression - buffer to hold transform expression @@ -4670,7 +4670,7 @@ done: * * prp_id - property list identifier to query * expression - buffer to hold transform expression - * expression_len - buffer size transorm expression + * expression_len - buffer size transform expression * * Output: * @@ -5478,7 +5478,7 @@ h5pget_fapl_mpio_c(hid_t_f *prp_id, int_f *comm, int_f *info) * h5pset_dxpl_mpio_c * PURPOSE * Call H5Pset_dxpl_mpio to set transfer mode of the dataset - * trasfer property list + * transfer property list * INPUTS * prp_id - property list identifier * data_xfer_mode - transfer mode @@ -5530,7 +5530,7 @@ h5pset_dxpl_mpio_c(hid_t_f *prp_id, int_f *data_xfer_mode) * h5pget_dxpl_mpio_c * PURPOSE * Call H5Pget_dxpl_mpio to get transfer mode of the dataset - * trasfer property list + * transfer property list * INPUTS * prp_id - property list identifier * data_xfer_mode - buffer to retrieve transfer mode diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index a7f2366..c55b7d9 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -500,7 +500,7 @@ CONTAINS ! a chunked layout dataset. ! ! INPUTS -! prp_id - datatset creation property list identifier +! prp_id - dataset creation property list identifier ! ndims - number of dimensions for each chunk ! dims - array with dimension sizes for each chunk ! OUTPUTS @@ -1880,7 +1880,7 @@ CONTAINS ! INPUTS ! ! prp_id - file access property list identifier -! gc_reference - flag for stting garbage collection on +! gc_reference - flag for setting garbage collection on ! and off (1 or 0) ! OUTPUTS ! @@ -1931,7 +1931,7 @@ CONTAINS ! prp_id - file access property list identifier ! OUTPUTS ! -! gc_reference - flag for stting garbage collection on +! gc_reference - flag for setting garbage collection on ! and off (1 or 0) ! hdferr - error code ! Success: 0 @@ -2659,7 +2659,7 @@ CONTAINS ! h5pequal_f ! ! PURPOSE -! Checks if two property lists are eqaul +! Checks if two property lists are equal ! ! INPUTS ! @@ -3059,7 +3059,7 @@ CONTAINS ! INPUTS ! ! plist_id - file access property list identifier -! size - metatdata block size +! size - metadata block size ! OUTPUTS ! ! hdferr - error code @@ -3103,7 +3103,7 @@ CONTAINS ! plist_id - file access property list identifier ! OUTPUTS ! -! size - metatdata block size +! size - metadata block size ! hdferr - error code ! Success: 0 ! Failure: -1 @@ -3596,7 +3596,7 @@ CONTAINS ! h5pget_class_parent_f ! ! PURPOSE -! Retrieves the parent class of a genric property class. +! Retrieves the parent class of a generic property class. ! ! INPUTS ! @@ -5391,7 +5391,7 @@ CONTAINS ! ! HISTORY ! -! Should hdferr return just 0 or 1 and add another arguement for the size? +! Should hdferr return just 0 or 1 and add another argument for the size? ! Fortran90 Interface: SUBROUTINE h5pget_data_transform_f(plist_id, expression, hdferr, size) IMPLICIT NONE diff --git a/fortran/src/H5Sf.c b/fortran/src/H5Sf.c index 341acfd..ea42a31 100644 --- a/fortran/src/H5Sf.c +++ b/fortran/src/H5Sf.c @@ -388,7 +388,7 @@ done: * PURPOSE * Call H5Sget_select_elem_pointlist * get a list of element points in the - * current dataspace selectin. + * current dataspace selection. * Starting with the startpoint-th point in the * list of points, numpoints points are put into the user's * buffer. If the user's buffer fills up before numpoints @@ -759,7 +759,7 @@ done: * space_id - identifier of the dataspace * rank - dataspace rank * current_size - array with the new dimension sizes - * maximum_size - aray with maximum sizes of dimensions + * maximum_size - array with maximum sizes of dimensions * RETURNS * 0 on success, -1 on failure * AUTHOR @@ -804,7 +804,7 @@ done: * space_id - identifier of the dataspace * OUTPUTS * dims - array with the dimension sizes - * maxdims - aray with maximum sizes of dimensions + * maxdims - array with maximum sizes of dimensions * RETURNS * number of dataspace dimensions (rank) on success, -1 on failure * AUTHOR @@ -852,7 +852,7 @@ done: * NAME * h5sis_simple_c * PURPOSE - * Call H5Sis_simple to detrmine if the dataspace + * Call H5Sis_simple to determine if the dataspace * is simple. * INPUTS * space_id - identifier of the dataspace diff --git a/fortran/src/H5Sff.F90 b/fortran/src/H5Sff.F90 index 62d665a..76b0dea 100644 --- a/fortran/src/H5Sff.F90 +++ b/fortran/src/H5Sff.F90 @@ -447,7 +447,7 @@ CONTAINS ! INPUTS ! space_id - dataspace identifier ! startpoint - element point to start with -! num_points - number of elemnt points to get +! num_points - number of element points to get ! OUTPUTS ! buf - buffer with element points selected ! hdferr - Returns 0 if successful and -1 if fails diff --git a/fortran/src/H5Tf.c b/fortran/src/H5Tf.c index 1516cc7..f3016f1 100644 --- a/fortran/src/H5Tf.c +++ b/fortran/src/H5Tf.c @@ -1560,7 +1560,7 @@ h5tget_member_type_c(hid_t_f *type_id, int_f *field_idx, hid_t_f *datatype) * Call H5Tcreate to create a datatype * INPUTS * cls - class type - * size - size of the class memeber + * size - size of the class member * RETURNS * 0 on success, -1 on failure * AUTHOR @@ -2033,7 +2033,7 @@ h5tvlen_create_c(hid_t_f *type_id, hid_t_f *vltype_id) * NAME * h5tis_variable_str_c * PURPOSE - * Call H5Tis_variable_str to detrmine if the datatype + * Call H5Tis_variable_str to determine if the datatype * is a variable string. * INPUTS * type_id - identifier of the dataspace @@ -2069,7 +2069,7 @@ h5tis_variable_str_c(hid_t_f *type_id, int_f *flag) * NAME * h5tget_member_class_c * PURPOSE - * Call H5Tget_member_class to detrmine ithe class of the compound + * Call H5Tget_member_class to determine ithe class of the compound * datatype member * INPUTS * type_id - identifier of the dataspace diff --git a/fortran/src/H5Tff.F90 b/fortran/src/H5Tff.F90 index 0eab39e..7582dab 100644 --- a/fortran/src/H5Tff.F90 +++ b/fortran/src/H5Tff.F90 @@ -86,7 +86,7 @@ CONTAINS ! called C functions (it is needed for Windows ! port). March 7, 2001 ! -! Added optional parameter 'tapl_id' for compatability +! Added optional parameter 'tapl_id' for compatibility ! with H5Topen2. April 9, 2009. ! ! SOURCE @@ -212,7 +212,7 @@ CONTAINS ! h5tcopy_f ! ! PURPOSE -! Creates a copy of exisiting datatype. +! Creates a copy of existing datatype. ! ! INPUTS ! type_id - datatype identifier diff --git a/fortran/src/H5_buildiface.F90 b/fortran/src/H5_buildiface.F90 index fb3a002..090b6db 100644 --- a/fortran/src/H5_buildiface.F90 +++ b/fortran/src/H5_buildiface.F90 @@ -14,7 +14,7 @@ ! ! NOTES ! This program uses the Fortran 2008 intrinsic function STORAGE_SIZE or SIZEOF -! depending on availablity.It generates code that makes use of +! depending on availability.It generates code that makes use of ! STORAGE_SIZE/SIZEOF in H5_gen.F90. STORAGE_SIZE is standard ! compliant and should always be chosen over SIZEOF. ! @@ -437,7 +437,7 @@ PROGRAM H5_buildiface ! dims parameter was added to make code portable; ! Aprile 4, 2001 ! -! Changed buf intent to INOUT to be consistant +! Changed buf intent to INOUT to be consistent ! with how the C functions handles it. The pg ! compiler will return 0 if a buf value is not set. ! February, 2008 @@ -564,7 +564,7 @@ PROGRAM H5_buildiface ! dims parameter was added to make code portable; ! Aprile 4, 2001 ! -! Changed buf intent to INOUT to be consistant +! Changed buf intent to INOUT to be consistent ! with how the C functions handles it. The pg ! compiler will return 0 if a buf value is not set. ! February, 2008 @@ -677,7 +677,7 @@ PROGRAM H5_buildiface ! Optional parameters: ! mem_space_id - memory dataspace identifier ! file_space_id - file dataspace identifier -! xfer_prp - trasfer property list identifier +! xfer_prp - transfer property list identifier ! ! AUTHOR ! Elena Pourmal diff --git a/fortran/src/H5_ff.F90 b/fortran/src/H5_ff.F90 index f56cdd9..350d978 100644 --- a/fortran/src/H5_ff.F90 +++ b/fortran/src/H5_ff.F90 @@ -701,7 +701,7 @@ CONTAINS ! ! Outputs: ! majnum - major version of the library -! minum - minor version of the library +! minnum - minor version of the library ! relnum - release version of the library ! error - Returns 0 if successful and -1 if fails ! @@ -736,7 +736,7 @@ CONTAINS ! ! Inputs: ! majnum - major version of the library -! minum - minor version of the library +! minnum - minor version of the library ! relnum - release version of the library ! ! Outputs: diff --git a/fortran/src/H5f90global.F90 b/fortran/src/H5f90global.F90 index 8f2b5ae..eb3b87e 100644 --- a/fortran/src/H5f90global.F90 +++ b/fortran/src/H5f90global.F90 @@ -855,7 +855,7 @@ CONTAINS END SUBROUTINE H5_Fortran_string_f2c -! Copy Fortran string to C charater array, assuming the C array is one-char +! Copy Fortran string to C character array, assuming the C array is one-char ! longer for the terminating null char. ! fstring : the Fortran input string ! cstring : the C output string (with memory already allocated) @@ -883,7 +883,7 @@ CONTAINS !!$ cstring(j) = C_NULL_CHAR !!$end subroutine MPIR_Fortran_string_f2c !!$ -!!$! Copy C charater array to Fortran string +!!$! Copy C character array to Fortran string !!$subroutine MPIR_Fortran_string_c2f(cstring, fstring) !!$ implicit none !!$ character(kind=c_char), intent(in) :: cstring(:) diff --git a/fortran/src/H5f90proto.h b/fortran/src/H5f90proto.h index fbbf7c2..4aa33f6 100644 --- a/fortran/src/H5f90proto.h +++ b/fortran/src/H5f90proto.h @@ -540,7 +540,7 @@ H5_FCDLL int_f h5pget_dxpl_mpio_c(hid_t_f *prp_id, int_f *data_xfer_mode); H5_FCDLL int_f h5pset_dxpl_mpio_c(hid_t_f *prp_id, int_f *data_xfer_mode); #endif /* - * Functions frome H5Rf.c + * Functions from H5Rf.c */ H5_FCDLL int_f h5rcreate_region_c(int_f *ref, hid_t_f *loc_id, _fcd name, int_f *namelen, hid_t_f *space_id); H5_FCDLL int_f h5rcreate_ptr_c(void *ref, hid_t_f *loc_id, _fcd name, int_f *namelen, int_f *ref_type, diff --git a/fortran/src/h5fc.in b/fortran/src/h5fc.in index a56d38d..661fde5 100644 --- a/fortran/src/h5fc.in +++ b/fortran/src/h5fc.in @@ -38,7 +38,7 @@ HL="@HL@" ## $FLINKER $FCFLAGS $H5BLD_FCFLAGS $F9XSUFFIXFLAG $LDFLAGS $LIBS ## ## $fmodules $link_objs $link_args $shared_link ## ## ## -## These settings can be overriden by setting HDF5_FCFLAGS, ## +## These settings can be overridden by setting HDF5_FCFLAGS, ## ## HDF5_LDFLAGS, or HDF5_LIBS in the environment. ## ## ## ############################################################################ @@ -136,7 +136,7 @@ usage() { echo " shared libraries]" echo " " echo " You can also add or change paths and flags to the compile line using" - echo " the following environment varibles or by assigning them to their counterparts" + echo " the following environment variables or by assigning them to their counterparts" echo " in the 'Things You Can Modify to Override...'" section of $prog_name echo " " echo " Variable Current value to be replaced" @@ -298,7 +298,7 @@ fi if test "x$do_link" = "xyes"; then shared_link="" -# conditionnaly link with the hl library +# conditionally link with the hl library if test "X$HL" = "Xhl"; then libraries=" $libraries -lhdf5hl_fortran -lhdf5_hl -lhdf5_fortran -lhdf5 " else diff --git a/fortran/test/H5_test_buildiface.F90 b/fortran/test/H5_test_buildiface.F90 index 94beab1..302fa38 100644 --- a/fortran/test/H5_test_buildiface.F90 +++ b/fortran/test/H5_test_buildiface.F90 @@ -14,7 +14,7 @@ ! ! NOTES ! This program uses the Fortran 2008 intrinsic function STORAGE_SIZE or SIZEOF -! depending on availablity.It generates code that makes use of +! depending on availability.It generates code that makes use of ! STORAGE_SIZE/SIZEOF in H5fortran_detect.f90. STORAGE_SIZE is standard ! compliant and should always be chosen over SIZEOF. ! diff --git a/fortran/test/fflush1.F90 b/fortran/test/fflush1.F90 index c192d93..5684f81 100644 --- a/fortran/test/fflush1.F90 +++ b/fortran/test/fflush1.F90 @@ -88,7 +88,7 @@ data_dims(2) = NY ! - !Initialize FORTRAN predifined datatypes + !Initialize FORTRAN predefined datatypes ! CALL h5open_f(error) CALL check("h5open_f",error,total_error) diff --git a/fortran/test/fflush2.F90 b/fortran/test/fflush2.F90 index dc3bc85..a6e16c1 100644 --- a/fortran/test/fflush2.F90 +++ b/fortran/test/fflush2.F90 @@ -81,7 +81,7 @@ PROGRAM FFLUSH2EXAMPLE data_dims(2) = NY ! - !Initialize FORTRAN predifined datatypes + !Initialize FORTRAN predefined datatypes ! CALL h5open_f(error) CALL check("h5open_f",error,total_error) @@ -163,7 +163,7 @@ PROGRAM FFLUSH2EXAMPLE CALL check("h5fclose_f",error,total_error) ! - !Close FORTRAN predifined datatypes + !Close FORTRAN predefined datatypes ! CALL h5_cleanup_f(filename, H5P_DEFAULT_F, error) CALL h5close_f(error) diff --git a/fortran/test/t.c b/fortran/test/t.c index 6dce111..6c856fa 100644 --- a/fortran/test/t.c +++ b/fortran/test/t.c @@ -152,6 +152,6 @@ void nh5_env_nocleanup_c(int_f *status) { *status = (int_f)0; - if (HDgetenv("HDF5_NOCLEANUP")) + if (HDgetenv(HDF5_NOCLEANUP)) *status = (int_f)1; } /* h5_env_nocleanup_c */ diff --git a/fortran/test/tH5A.F90 b/fortran/test/tH5A.F90 index d5ce9a2..2e76dad 100644 --- a/fortran/test/tH5A.F90 +++ b/fortran/test/tH5A.F90 @@ -81,7 +81,7 @@ CONTAINS INTEGER(HID_T) :: atype5_id !Integer Attribute Datatype identifier INTEGER(HSIZE_T), DIMENSION(1) :: adims = (/2/) ! Attribute dimension INTEGER(HSIZE_T), DIMENSION(1) :: adims2 = (/1/) ! Attribute dimension - INTEGER :: arank = 1 ! Attribure rank + INTEGER :: arank = 1 ! Attribute rank INTEGER(SIZE_T) :: attrlen ! Length of the attribute string INTEGER(HID_T) :: attr_space !Returned String Attribute Space identifier diff --git a/fortran/test/tH5A_1_8.F90 b/fortran/test/tH5A_1_8.F90 index 4e02c58..b245b1c 100644 --- a/fortran/test/tH5A_1_8.F90 +++ b/fortran/test/tH5A_1_8.F90 @@ -64,7 +64,7 @@ SUBROUTINE attribute_test_1_8(cleanup, total_error) INTEGER :: ret_total_error ! ******************** -! test_attr equivelent +! test_attr equivalent ! ******************** ! WRITE(*,*) "TESTING ATTRIBUTES" @@ -777,7 +777,6 @@ SUBROUTINE test_attr_info_by_idx(new_format, fcpl, fapl, total_error) INTEGER :: Input1 INTEGER(HSIZE_T) :: hzero = 0_HSIZE_T INTEGER :: minusone = -1 - INTEGER(HSIZE_T) :: htmp data_dims = 0 @@ -840,10 +839,6 @@ SUBROUTINE test_attr_info_by_idx(new_format, fcpl, fapl, total_error) my_dataset = dset3 END SELECT - ! Check for query on non-existant attribute - - n = 0 - ! -- CHECK PASSING AN INTEGER CONSTANT IN DIFFERENT FORMS -- ! 1) call by passing an integer with the _hsize_t declaration @@ -896,8 +891,8 @@ SUBROUTINE test_attr_info_by_idx(new_format, fcpl, fapl, total_error) ! Verify information for new attribute !EP CALL attr_info_by_idx_check(my_dataset, attrname, INT(j,HSIZE_T), use_index(i), total_error ) - htmp = j - CALL attr_info_by_idx_check(my_dataset, attrname, htmp, use_index(i), total_error ) + n = INT(j, HSIZE_T) + CALL attr_info_by_idx_check(my_dataset, attrname, n, use_index(i), total_error ) !CHECK(ret, FAIL, "attr_info_by_idx_check"); ENDDO @@ -1111,7 +1106,7 @@ SUBROUTINE test_attr_shared_rename( fcpl, fapl, total_error) INTEGER :: test_shared INTEGER(HSIZE_T), DIMENSION(1) :: adims2 = (/1/) ! Attribute dimension - INTEGER :: arank = 1 ! Attribure rank + INTEGER :: arank = 1 ! Attribute rank ! Initialize "big" attribute data @@ -1497,7 +1492,7 @@ SUBROUTINE test_attr_delete_by_idx(new_format, fcpl, fapl, total_error) END SELECT - ! Check for deleting non-existant attribute + ! Check for deleting non-existent attribute !EP CALL H5Adelete_by_idx_f(my_dataset, '.', idx_type, order, 0_HSIZE_T,error, lapl_id=H5P_DEFAULT_F) CALL H5Adelete_by_idx_f(my_dataset, '.', idx_type, order, hzero,error, lapl_id=H5P_DEFAULT_F) CALL verify("H5Adelete_by_idx_f",error,minusone,total_error) @@ -1773,7 +1768,7 @@ SUBROUTINE test_attr_shared_delete(fcpl, fapl, total_error) INTEGER :: test_shared INTEGER(HSIZE_T), DIMENSION(1) :: adims2 = (/1/) ! Attribute dimension - INTEGER :: arank = 1 ! Attribure rank + INTEGER :: arank = 1 ! Attribute rank ! Output message about test being performed diff --git a/fortran/test/tH5D.F90 b/fortran/test/tH5D.F90 index 006d62a..eb9e831 100644 --- a/fortran/test/tH5D.F90 +++ b/fortran/test/tH5D.F90 @@ -187,7 +187,7 @@ CONTAINS DO i = 1, 4 DO j = 1, 6 IF (data_out(i,j) .NE. dset_data(i, j)) THEN - WRITE(*, *) "dataset test error occured" + WRITE(*, *) "dataset test error occurred" WRITE(*,*) "data read is not the same as the data written" END IF END DO @@ -196,7 +196,7 @@ CONTAINS ! Check if no change to null_dset_data ! IF (null_dset_data .NE. 1) THEN - WRITE(*, *) "null dataset test error occured" + WRITE(*, *) "null dataset test error occurred" END IF ! ! End access to the dataset and release resources used by it. @@ -258,7 +258,7 @@ CONTAINS INTEGER(HID_T) :: dset_id ! Dataset identifier INTEGER(HID_T) :: dataspace ! Dataspace identifier INTEGER(HID_T) :: memspace ! memory Dataspace identifier - INTEGER(HID_T) :: crp_list ! dataset creatation property identifier + INTEGER(HID_T) :: crp_list ! dataset creation property identifier ! !dataset dimensions at creation time @@ -313,7 +313,7 @@ CONTAINS END DO ! - !Initialize FORTRAN predifined datatypes + !Initialize FORTRAN predefined datatypes ! ! CALL h5init_types_f(error) ! CALL check("h5init_types_f",error,total_error) @@ -347,7 +347,7 @@ CONTAINS CALL check("h5pset_chunk_f",error,total_error) ! - !Create a dataset with 3X3 dimensions using cparms creation propertie . + !Create a dataset with 3X3 dimensions using cparms creation properties . ! CALL h5dcreate_f(file_id, dsetname, H5T_NATIVE_INTEGER, dataspace, dset_id, error, crp_list ) CALL check("h5dcreate_f",error,total_error) @@ -426,7 +426,7 @@ CONTAINS CALL h5sget_simple_extent_ndims_f(dataspace, rankr, error) CALL check("h5sget_simple_extent_ndims_f",error,total_error) IF (rankr .NE. RANK) THEN - WRITE(*,*) "dataset rank error occured" + WRITE(*,*) "dataset rank error occurred" STOP END IF @@ -436,7 +436,7 @@ CONTAINS CALL h5sget_simple_extent_dims_f(dataspace, dimsr, maxdimsr, error) CALL check("h5sget_simple_extent_dims_f",error,total_error) IF ((dimsr(1) .NE. dims1(1)) .OR. (dimsr(2) .NE. dims1(2))) THEN - WRITE(*,*) "dataset dimensions error occured" + WRITE(*,*) "dataset dimensions error occurred" STOP END IF @@ -466,7 +466,7 @@ CONTAINS DO ih = 1, dims1(1) DO jh = 1, dims1(2) IF (data_out(ih,jh) .NE. data_in(ih, jh)) THEN - WRITE(*, *) "extend dataset test error occured" + WRITE(*, *) "extend dataset test error occurred" WRITE(*, *) "read value is not the same as the written values" END IF END DO diff --git a/fortran/test/tH5F.F90 b/fortran/test/tH5F.F90 index 06dc6de..8d4845d 100644 --- a/fortran/test/tH5F.F90 +++ b/fortran/test/tH5F.F90 @@ -554,7 +554,7 @@ CONTAINS do i = 1, NX do j = 1, NY IF (data_out(i,j) .NE. dset_data(i, j)) THEN - write(*, *) "reopen test error occured" + write(*, *) "reopen test error occurred" END IF end do end do @@ -584,17 +584,23 @@ CONTAINS ! The following subroutine checks that h5fget_name_f produces ! correct output for a given obj_id and filename. ! - SUBROUTINE check_get_name(obj_id, fix_filename, total_error) + SUBROUTINE check_get_name(obj_id, fix_filename, len_filename, total_error) USE HDF5 ! This module contains all necessary modules USE TH5_MISC IMPLICIT NONE INTEGER(HID_T) :: obj_id ! Object identifier CHARACTER(LEN=80), INTENT(IN) :: fix_filename ! Expected filename + INTEGER, INTENT(IN) :: len_filename ! The length of the filename INTEGER, INTENT(INOUT) :: total_error ! Error count CHARACTER(LEN=80):: file_name ! Filename buffer INTEGER:: error ! HDF5 error code INTEGER(SIZE_T):: name_size ! Filename length + + INTEGER, PARAMETER :: sm_len = 2 + CHARACTER(LEN=len_filename) :: filename_exact + CHARACTER(LEN=len_filename-sm_len) :: filename_sm + ! !Get file name from the dataset identifier ! @@ -637,6 +643,30 @@ CONTAINS total_error = total_error + 1 END IF + ! Use a buffer which is the exact size needed to hold the filename + CALL h5fget_name_f(obj_id, filename_exact, name_size, error) + CALL check("h5fget_name_f",error,total_error) + IF(name_size .NE. len_filename)THEN + WRITE(*,*) " file name size obtained from the object id is incorrect" + total_error = total_error + 1 + ENDIF + IF(filename_exact .NE. TRIM(fix_filename)) THEN + WRITE(*,*) " file name obtained from the object id is incorrect" + total_error = total_error + 1 + END IF + + ! Use a buffer which is smaller than needed to hold the filename + CALL h5fget_name_f(obj_id, filename_sm, name_size, error) + CALL check("h5fget_name_f",error,total_error) + IF(name_size .NE. len_filename)THEN + WRITE(*,*) " file name size obtained from the object id is incorrect" + total_error = total_error + 1 + ENDIF + IF(filename_sm(1:len_filename-sm_len) .NE. fix_filename(1:len_filename-sm_len)) THEN + WRITE(*,*) " file name obtained from the object id is incorrect" + total_error = total_error + 1 + END IF + END SUBROUTINE check_get_name ! The following subroutine tests h5fget_name_f. @@ -653,6 +683,7 @@ CONTAINS CHARACTER(LEN=*), PARAMETER :: filename = "filename" CHARACTER(LEN=80) :: fix_filename + INTEGER :: len_filename INTEGER(HID_T) :: file_id ! File identifier INTEGER(HID_T) :: g_id ! Group identifier @@ -679,8 +710,9 @@ CONTAINS CALL h5gopen_f(file_id,"/",g_id, error) CALL check("h5gopen_f",error,total_error) - CALL check_get_name(file_id, fix_filename, total_error) - CALL check_get_name(g_id, fix_filename, total_error) + len_filename = LEN_TRIM(fix_filename) + CALL check_get_name(file_id, fix_filename, len_filename, total_error) + CALL check_get_name(g_id, fix_filename, len_filename, total_error) ! Close the group. ! diff --git a/fortran/test/tH5G_1_8.F90 b/fortran/test/tH5G_1_8.F90 index 534a2d4..222ba9d 100644 --- a/fortran/test/tH5G_1_8.F90 +++ b/fortran/test/tH5G_1_8.F90 @@ -654,7 +654,7 @@ SUBROUTINE group_info(cleanup, fapl, total_error) INTEGER(HID_T) :: file, scalar, grp, d1 CHARACTER(LEN=12), PARAMETER :: filename ='TestLinks.h5' INTEGER(HSIZE_T), DIMENSION(1) :: adims2 = (/1/) ! Attribute dimension - INTEGER :: arank = 1 ! Attribure rank + INTEGER :: arank = 1 ! Attribute rank INTEGER :: error INTEGER :: cset ! Indicates the character set used for the link’s name. diff --git a/fortran/test/tH5MISC_1_8.F90 b/fortran/test/tH5MISC_1_8.F90 index c294f21..e35ef1f 100644 --- a/fortran/test/tH5MISC_1_8.F90 +++ b/fortran/test/tH5MISC_1_8.F90 @@ -160,7 +160,7 @@ SUBROUTINE test_genprop_basic_class(total_error) CALL verify("H5Pequal_f", flag, .TRUE., total_error) - ! Make certain false postives aren't being returned + ! Make certain false positives aren't being returned CALL H5Pequal_f(cid2, H5P_FILE_CREATE_F, flag, error) CALL check("H5Pequal_f", error, total_error) CALL verify("H5Pequal_f", flag, .FALSE., total_error) diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index 1d1208c..75e4e72 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -349,8 +349,8 @@ SUBROUTINE multi_file_test(cleanup, total_error) DO i = 1, 4 DO j = 1, 6 IF (data_out(i,j) .NE. dset_data(i, j)) THEN - WRITE(*, *) "dataset test error occured" - WRITE(*,*) "data read is not the same as the data writen" + WRITE(*, *) "dataset test error occurred" + WRITE(*,*) "data read is not the same as the data written" END IF END DO END DO @@ -575,7 +575,7 @@ SUBROUTINE test_chunk_cache(cleanup, total_error) CALL H5Pclose_f(dapl2,error) CALL check("H5Pclose_f", error, total_error) - ! Similary, test use of H5Dcreate2 with H5P_DEFAULT + ! Similarly, test use of H5Dcreate2 with H5P_DEFAULT CALL H5Dclose_f(dsid, error) CALL check("H5Dclose_f", error, total_error) diff --git a/fortran/test/tH5P_F03.F90 b/fortran/test/tH5P_F03.F90 index 245a588..43dd1c0 100644 --- a/fortran/test/tH5P_F03.F90 +++ b/fortran/test/tH5P_F03.F90 @@ -1013,28 +1013,28 @@ SUBROUTINE test_vds(total_error) CALL check("H5Pget_virtual_filename_f", error, total_error) IF(nsize.NE.LEN(SRC_FILE_LEN_EXACT))THEN - PRINT*,"virtual filenname size is incorrect" + PRINT*,"virtual filename size is incorrect" total_error = total_error + 1 ENDIF ! check passing a buffer that is very small CALL H5Pget_virtual_filename_f(dcpl, INT(i-1, size_t), SRC_FILE_LEN_TINY, error) CALL check("H5Pget_virtual_filename_f", error, total_error) IF(SRC_FILE_LEN_TINY.NE.SRC_FILE(i)(1:LEN(SRC_FILE_LEN_TINY)))THEN - PRINT*,"virtual filenname returned is incorrect" + PRINT*,"virtual filename returned is incorrect" total_error = total_error + 1 ENDIF ! check passing a buffer that small by one CALL H5Pget_virtual_filename_f(dcpl, INT(i-1, size_t), SRC_FILE_LEN_SMALL, error) CALL check("H5Pget_virtual_filename_f", error, total_error) IF(SRC_FILE_LEN_SMALL.NE.SRC_FILE(i)(1:LEN(SRC_FILE_LEN_SMALL)))THEN - PRINT*,"virtual filenname returned is incorrect" + PRINT*,"virtual filename returned is incorrect" total_error = total_error + 1 ENDIF ! check passing a buffer that is exact CALL H5Pget_virtual_filename_f(dcpl, INT(i-1, size_t), SRC_FILE_LEN_EXACT, error) CALL check("H5Pget_virtual_filename_f", error, total_error) IF(SRC_FILE_LEN_EXACT.NE.SRC_FILE(i)(1:LEN(SRC_FILE_LEN_EXACT)))THEN - PRINT*,"virtual filenname returned is incorrect" + PRINT*,"virtual filename returned is incorrect" total_error = total_error + 1 ENDIF ! check passing a buffer that bigger by one @@ -1042,7 +1042,7 @@ SUBROUTINE test_vds(total_error) CALL check("H5Pget_virtual_filename_f", error, total_error) IF(SRC_FILE_LEN_LARGE(1:LEN(SRC_FILE_LEN_EXACT)).NE.SRC_FILE(i)(1:LEN(SRC_FILE_LEN_EXACT)).AND. & SRC_FILE_LEN_LARGE(LEN(SRC_FILE_LEN_EXACT):).NE.'')THEN - PRINT*,"virtual filenname returned is incorrect" + PRINT*,"virtual filename returned is incorrect" total_error = total_error + 1 ENDIF ! check passing a buffer that is very big @@ -1050,7 +1050,7 @@ SUBROUTINE test_vds(total_error) CALL check("H5Pget_virtual_filename_f", error, total_error) IF(SRC_FILE_LEN_HUGE(1:LEN(SRC_FILE_LEN_EXACT)).NE.SRC_FILE(i)(1:LEN(SRC_FILE_LEN_EXACT)).AND. & SRC_FILE_LEN_HUGE(LEN(SRC_FILE_LEN_EXACT):).NE.'')THEN - PRINT*,"virtual filenname returned is incorrect" + PRINT*,"virtual filename returned is incorrect" total_error = total_error + 1 ENDIF ! Get source dataset name diff --git a/fortran/test/tH5R.F90 b/fortran/test/tH5R.F90 index 5144d83..4e62b20 100644 --- a/fortran/test/tH5R.F90 +++ b/fortran/test/tH5R.F90 @@ -142,7 +142,7 @@ SUBROUTINE refobjtest(cleanup, total_error) CALL check("h5gclose_f",error,total_error) ! - ! Craete references to two groups, integer dataset and shared datatype + ! Create references to two groups, integer dataset and shared datatype ! and write it to the dataset in the file ! CALL h5rcreate_f(file_id, groupname1, ref(1), error) diff --git a/fortran/test/tH5S.F90 b/fortran/test/tH5S.F90 index 08f0b59..302db51 100644 --- a/fortran/test/tH5S.F90 +++ b/fortran/test/tH5S.F90 @@ -151,7 +151,7 @@ CONTAINS CALL h5sget_simple_extent_dims_f(space2_id, dimsout, maxdimsout, error) CALL check("h5sget_simple_extent_dims_f", error, total_error) IF ((dimsout(1) .NE. dims1(1)) .OR. (dimsout(2) .NE. dims1(2)) ) THEN - write(*,*)"error occured, copied dims not same" + write(*,*)"error occurred, copied dims not same" END IF ! @@ -159,14 +159,14 @@ CONTAINS ! CALL h5sget_simple_extent_ndims_f(space2_id, rank2, error) CALL check("h5sget_simple_extent_ndims_f", error, total_error) - IF (rank2 .NE. rank1) write(*,*)"error occured, copied ranks not same" + IF (rank2 .NE. rank1) write(*,*)"error occurred, copied ranks not same" ! !get the copied space's number of elements. ! CALL h5sget_simple_extent_npoints_f(space2_id, npoints, error) CALL check("h5sget_simple_extent_npoints_f", error, total_error) - IF (npoints .NE. 24) write(*,*)"error occured, number of elements not correct" + IF (npoints .NE. 24) write(*,*)"error occurred, number of elements not correct" ! @@ -194,7 +194,7 @@ CONTAINS CALL h5sget_simple_extent_dims_f(space2_id, dimsout, maxdimsout, error) CALL check("h5sget_simple_extent_dims_f", error, total_error) IF ((dimsout(1) .NE. dims2(1)) .OR. (dimsout(2) .NE. dims2(2)) ) THEN - write(*,*)"error occured, copied dims not same" + write(*,*)"error occurred, copied dims not same" END IF ! @@ -235,8 +235,8 @@ CONTAINS do i = 1, 4 do j = 1, 6 IF (data1_out(i,j) .NE. data1_in(i, j)) THEN - write(*, *) "dataset test error occured" - write(*,*) "data read is not the same as the data writen" + write(*, *) "dataset test error occurred" + write(*,*) "data read is not the same as the data written" END IF end do end do @@ -256,8 +256,8 @@ CONTAINS do i = 1, 6 do j = 1, 6 IF (data2_out(i,j) .NE. data2_in(i, j)) THEN - write(*, *) "dataset test error occured" - write(*,*) "data read is not the same as the data writen" + write(*, *) "dataset test error occurred" + write(*,*) "data read is not the same as the data written" END IF end do end do diff --git a/fortran/test/tH5Sselect.F90 b/fortran/test/tH5Sselect.F90 index 178ac32..41cd16f 100644 --- a/fortran/test/tH5Sselect.F90 +++ b/fortran/test/tH5Sselect.F90 @@ -149,7 +149,7 @@ CONTAINS ! ! - !Initialize FORTRAN predifined datatypes + !Initialize FORTRAN predefined datatypes ! ! CALL h5init_types_f(error) ! CALL check("h5init_types_f", error, total_error) @@ -430,7 +430,7 @@ CONTAINS end do ! - !Initialize FORTRAN predifined datatypes + !Initialize FORTRAN predefined datatypes ! ! CALL h5init_types_f(error) ! CALL check("h5init_types_f", error, total_error) @@ -912,7 +912,7 @@ CONTAINS ! CALL h5sget_select_hyper_nblocks_f(dataspace, num_blocks, error) CALL check("h5sget_select_hyper_nblocks_f", error, total_error) - IF (num_blocks .NE. 4) write (*,*) "error occured with num_blocks" + IF (num_blocks .NE. 4) write (*,*) "error occurred with num_blocks" !write(*,*) num_blocks !result of num_blocks is 4 @@ -945,11 +945,11 @@ CONTAINS CALL h5sget_select_bounds_f(dataspace, startout, endout, error) CALL check("h5sget_select_bounds_f", error, total_error) IF ( (startout(1) .ne. 1) .or. (startout(2) .ne. 1) ) THEN - write(*,*) "error occured to select_bounds's start position" + write(*,*) "error occurred to select_bounds's start position" END IF IF ( (endout(1) .ne. 5) .or. (endout(2) .ne. 5) ) THEN - write(*,*) "error occured to select_bounds's end position" + write(*,*) "error occurred to select_bounds's end position" END IF !write(*,*) (startout(i), i = 1, RANK) !result of startout is 0, 0 @@ -978,7 +978,7 @@ CONTAINS ! CALL h5sget_select_elem_npoints_f(dataspace, num_points, error) CALL check("h5sget_select_elem_npoints_f", error, total_error) - IF (num_points .NE. 10) write(*,*) "error occured with num_points" + IF (num_points .NE. 10) write(*,*) "error occurred with num_points" !write(*,*) num_points ! result of num_points is 10 diff --git a/fortran/test/tH5T.F90 b/fortran/test/tH5T.F90 index 8b10136..d24ac89 100644 --- a/fortran/test/tH5T.F90 +++ b/fortran/test/tH5T.F90 @@ -72,7 +72,7 @@ CONTAINS INTEGER(HID_T) :: dt4_id ! Memory datatype identifier (for real field) INTEGER(HID_T) :: dt5_id ! Memory datatype identifier INTEGER(HID_T) :: membtype_id ! Datatype identifier - INTEGER(HID_T) :: plist_id ! Dataset trasfer property + INTEGER(HID_T) :: plist_id ! Dataset transfer property INTEGER(HSIZE_T), DIMENSION(1) :: dims = (/dimsize/) ! Dataset dimensions @@ -98,7 +98,7 @@ CONTAINS INTEGER :: class ! Datatype class INTEGER :: num_members ! Number of members in the compound datatype CHARACTER(LEN=256) :: member_name - INTEGER :: len ! Lenght of the name of the compound datatype member + INTEGER :: len ! Length of the name of the compound datatype member INTEGER :: member_index ! index of the field INTEGER(HSIZE_T), DIMENSION(3) :: array_dims=(/2,3,4/) INTEGER :: array_dims_range = 3 @@ -219,9 +219,9 @@ CONTAINS CALL h5tcreate_f(H5T_COMPOUND_F, type_size, dtype_id, error) CALL check("h5tcreate_f", error, total_error) ! - ! Insert memebers + ! Insert members ! - ! CHARACTER*2 memeber + ! CHARACTER*2 member ! offset = 0 CALL h5tinsert_f(dtype_id, "char_field", offset, dt5_id, error) @@ -229,13 +229,13 @@ CONTAINS ! ! INTEGER member ! - offset = offset + type_sizec ! Offset of the second memeber is 2 + offset = offset + type_sizec ! Offset of the second member is 2 CALL h5tinsert_f(dtype_id, "integer_field", offset, H5T_NATIVE_INTEGER, error) CALL check("h5tinsert_f", error, total_error) ! ! DOUBLE PRECISION member ! - offset = offset + type_sizei ! Offset of the third memeber is 6 + offset = offset + type_sizei ! Offset of the third member is 6 CALL h5tinsert_f(dtype_id, "double_field", offset, H5T_NATIVE_DOUBLE, error) CALL check("h5tinsert_f", error, total_error) ! @@ -416,7 +416,7 @@ CONTAINS CALL h5tequal_f(membtype_id, H5T_NATIVE_INTEGER, flag, error) CALL check("h5tequal_f", error, total_error) if(.not. flag) then - write(*,*) "Wrong member type returned for integer memebr" + write(*,*) "Wrong member type returned for integer member" total_error = total_error + 1 endif CALL h5tget_member_class_f(dtype_id, i-1, class, error) @@ -435,7 +435,7 @@ CONTAINS CALL h5tequal_f(membtype_id, H5T_NATIVE_DOUBLE, flag, error) CALL check("h5tequal_f", error, total_error) if(.not. flag) then - write(*,*) "Wrong member type returned for double precision memebr" + write(*,*) "Wrong member type returned for double precision member" total_error = total_error + 1 endif CALL h5tget_member_class_f(dtype_id, i-1, class, error) @@ -454,7 +454,7 @@ CONTAINS CALL h5tequal_f(membtype_id, H5T_NATIVE_REAL, flag, error) CALL check("h5tequal_f", error, total_error) if(.not. flag) then - write(*,*) "Wrong member type returned for real memebr" + write(*,*) "Wrong member type returned for real member" total_error = total_error + 1 endif CALL h5tget_member_class_f(dtype_id, i-1, class, error) diff --git a/fortran/test/tH5VL.F90 b/fortran/test/tH5VL.F90 index 2269fec..eb0470c 100644 --- a/fortran/test/tH5VL.F90 +++ b/fortran/test/tH5VL.F90 @@ -147,7 +147,7 @@ CONTAINS CALL check("h5dvlen_get_max_len_f", error, total_error) if(max_len .ne. data_dims(1)) then total_error = total_error + 1 - write(*,*) "Wrong number of elemets returned by h5dvlen_get_max_len_f" + write(*,*) "Wrong number of elements returned by h5dvlen_get_max_len_f" endif ! ! Read the dataset. @@ -317,7 +317,7 @@ CONTAINS CALL check("h5dvlen_get_max_len_f", error, total_error) if(max_len .ne. data_dims(1)) then total_error = total_error + 1 - write(*,*) "Wrong number of elemets returned by h5dvlen_get_max_len_f" + write(*,*) "Wrong number of elements returned by h5dvlen_get_max_len_f" endif ! ! Read the dataset. diff --git a/fortran/test/tH5Z.F90 b/fortran/test/tH5Z.F90 index 8c39fea..799067a 100644 --- a/fortran/test/tH5Z.F90 +++ b/fortran/test/tH5Z.F90 @@ -374,7 +374,7 @@ CONTAINS do i = 1, N do j = 1, M IF (data_out(i,j) .NE. dset_data(i, j)) THEN - write(*, *) "dataset test error occured" + write(*, *) "dataset test error occurred" write(*,*) "data read is not the same as the data written" num_errors = num_errors + 1 IF (num_errors .GE. 512) THEN diff --git a/hl/c++/test/ptableTest.cpp b/hl/c++/test/ptableTest.cpp index ab49303..33199f1 100644 --- a/hl/c++/test/ptableTest.cpp +++ b/hl/c++/test/ptableTest.cpp @@ -223,7 +223,7 @@ TestGetNext() for (record = 1; record < 6; record++) wrapper.AppendPacket(&record); - /* Ensure that we can interate through the records and get the right ones */ + /* Ensure that we can iterate through the records and get the right ones */ for (i = 1; i < 6; i++) { wrapper.GetNextPacket(&record); if (record != i) @@ -237,7 +237,7 @@ TestGetNext() if (error < 0) goto error; - /* Ensure that we can interate through the records and get the right ones */ + /* Ensure that we can iterate through the records and get the right ones */ for (i = 1; i < 6; i++) { error = wrapper.GetNextPacket(&record); if (record != i || error < 0) @@ -622,7 +622,7 @@ TestHDFFV_9758() s1[i].a = static_cast(i); s1[i].b = 1.0F * static_cast(i * i); s1[i].c = 1.0 / static_cast(i + 1); - HDsprintf(s1[i].d, "string%" PRIuHSIZE "", i); + HDsnprintf(s1[i].d, STRING_LENGTH, "string%" PRIuHSIZE "", i); s1[i].e = static_cast(100 + i); } diff --git a/hl/examples/ex_image1.c b/hl/examples/ex_image1.c index ead1715..1294ab3 100644 --- a/hl/examples/ex_image1.c +++ b/hl/examples/ex_image1.c @@ -56,10 +56,10 @@ main(void) H5IMmake_image_8bit(file_id, "image1", (hsize_t)WIDTH, (hsize_t)HEIGHT, buf); /* make a palette */ - H5IMmake_palette(file_id, "pallete", pal_dims, pal); + H5IMmake_palette(file_id, "palette", pal_dims, pal); /* attach the palette to the image */ - H5IMlink_palette(file_id, "image1", "pallete"); + H5IMlink_palette(file_id, "image1", "palette"); /* close the file. */ H5Fclose(file_id); diff --git a/hl/fortran/src/H5DSff.F90 b/hl/fortran/src/H5DSff.F90 index b241e9a..bbd9918 100644 --- a/hl/fortran/src/H5DSff.F90 +++ b/hl/fortran/src/H5DSff.F90 @@ -216,7 +216,7 @@ CONTAINS END SUBROUTINE H5DSis_attached_f ! -! H5DSiterate_scales: Impliment in F2003 +! H5DSiterate_scales: Implement in F2003 ! !------------------------------------------------------------------------- diff --git a/hl/fortran/src/H5HL_buildiface.F90 b/hl/fortran/src/H5HL_buildiface.F90 index 4f34575..ca66ebf 100644 --- a/hl/fortran/src/H5HL_buildiface.F90 +++ b/hl/fortran/src/H5HL_buildiface.F90 @@ -14,7 +14,7 @@ ! ! NOTES ! This program uses the Fortran 2008 intrinsic function STORAGE_SIZE or SIZEOF -! depending on availablity.It generates code that makes use of +! depending on availability.It generates code that makes use of ! STORAGE_SIZE/SIZEOF in H5fortran_detect.f90. STORAGE_SIZE is standard ! compliant and should always be chosen over SIZEOF. ! diff --git a/hl/fortran/src/H5LTff.F90 b/hl/fortran/src/H5LTff.F90 index ba4b770..a4ab247 100644 --- a/hl/fortran/src/H5LTff.F90 +++ b/hl/fortran/src/H5LTff.F90 @@ -88,7 +88,7 @@ MODULE H5LT_CONST TYPE(C_PTR), VALUE :: buf ! data buffer CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(in) :: dtype ! flag indicating the datatype of the ! the buffer: - ! R=Real, D=DOUBLE, I=Interger, C=Character + ! R=Real, D=DOUBLE, I=Integer, C=Character INTEGER(size_t) :: SizeOf_buf ! Sizeof the buf datatype END FUNCTION h5ltset_attribute_c END INTERFACE @@ -107,7 +107,7 @@ MODULE H5LT_CONST TYPE(C_PTR), VALUE :: buf ! data buffer CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(in) :: dtype ! flag indicating the datatype of the ! the buffer: - ! R=Real, D=DOUBLE, I=Interger + ! R=Real, D=DOUBLE, I=Integer INTEGER(size_t), INTENT(in) :: SizeOf_buf ! Sizeof the buf data type END FUNCTION h5ltget_attribute_c END INTERFACE diff --git a/hl/fortran/src/H5TBfc.c b/hl/fortran/src/H5TBfc.c index 23e3377..91c420b 100644 --- a/hl/fortran/src/H5TBfc.c +++ b/hl/fortran/src/H5TBfc.c @@ -83,7 +83,7 @@ h5tbmake_table_c(size_t_f *namelen1, _fcd name1, hid_t_f *loc_id, size_t_f *name (tmp = (char *)HD5f2cstring(field_names, (size_t) * (max_char_size_field_names) * (size_t)num_elem))) HGOTO_DONE(FAIL) /* - * move data from temorary buffer + * move data from temporary buffer */ tmp_p = tmp; for (i = 0; i < num_elem; i++) { @@ -188,7 +188,7 @@ h5tbmake_table_ptr_c(size_t_f *namelen1, _fcd name1, hid_t_f *loc_id, size_t_f * (tmp = (char *)HD5f2cstring(field_names, (size_t) * (max_char_size_field_names) * (size_t)num_elem))) HGOTO_DONE(FAIL) /* - * move data from temorary buffer + * move data from temporary buffer */ tmp_p = tmp; for (i = 0; i < num_elem; i++) { diff --git a/hl/fortran/src/H5TBff.F90 b/hl/fortran/src/H5TBff.F90 index 35a88b8..82f34a8 100644 --- a/hl/fortran/src/H5TBff.F90 +++ b/hl/fortran/src/H5TBff.F90 @@ -961,7 +961,7 @@ CONTAINS INTEGER(size_t), DIMENSION(1:nfields), INTENT(inout) :: field_offsets ! field offsets INTEGER(size_t), INTENT(inout):: type_size ! type size INTEGER(size_t) :: namelen ! name length - INTEGER(size_t) :: maxlen ! maxiumum length of input field names + INTEGER(size_t) :: maxlen ! maximum length of input field names INTEGER(size_t), DIMENSION(1:nfields) :: namelen2 ! name lengths INTEGER(size_t) :: c_maxlen_out ! maximum character length of a field array element END FUNCTION h5tbget_field_info_c diff --git a/hl/fortran/test/tsttable.F90 b/hl/fortran/test/tsttable.F90 index a77068c..e7bd88e 100644 --- a/hl/fortran/test/tsttable.F90 +++ b/hl/fortran/test/tsttable.F90 @@ -172,11 +172,11 @@ SUBROUTINE test_table1() ! offset = 0 field_offset(1) = offset - offset = offset + type_sizec ! Offset of the second memeber is 2 + offset = offset + type_sizec ! Offset of the second member is 2 field_offset(2) = offset - offset = offset + type_sizei ! Offset of the second memeber is 6 + offset = offset + type_sizei ! Offset of the second member is 6 field_offset(3) = offset - offset = offset + type_sized ! Offset of the second memeber is 14 + offset = offset + type_sized ! Offset of the second member is 14 field_offset(4) = offset !------------------------------------------------------------------------- @@ -571,7 +571,7 @@ SUBROUTINE test_table2() INTEGER(HSIZE_T), PARAMETER :: nrecords = 8 ! nrecords CHARACTER(len=8), PARAMETER :: filename = "f2tab.h5" ! File name - CHARACTER(LEN=5), PARAMETER :: table_name = "tabel" ! table name + CHARACTER(LEN=5), PARAMETER :: table_name = "table" ! table name CHARACTER(LEN=10), PARAMETER :: table_name_fill = "tabel_fill" ! table name ! Define field information diff --git a/hl/src/H5DO.c b/hl/src/H5DO.c index 3bdb4e2..1703669 100644 --- a/hl/src/H5DO.c +++ b/hl/src/H5DO.c @@ -182,7 +182,7 @@ H5DOappend(hid_t dset_id, hid_t dxpl_id, unsigned axis, size_t extension, hid_t if (FAIL == H5Sselect_hyperslab(new_space_id, H5S_SELECT_SET, start, stride, count, block)) goto done; - /* The # of elemnts in the new extended dataspace */ + /* The # of elements in the new extended dataspace */ if ((snelmts = H5Sget_select_npoints(new_space_id)) < 0) goto done; nelmts = (hsize_t)snelmts; diff --git a/hl/src/H5DS.c b/hl/src/H5DS.c index c947d16..387419b 100644 --- a/hl/src/H5DS.c +++ b/hl/src/H5DS.c @@ -88,7 +88,7 @@ H5DSset_scale(hid_t dsid, const char *dimname) return FAIL; /*------------------------------------------------------------------------- - * check if the dataset is a dataset wich has references to dimension scales + * check if the dataset is a dataset which has references to dimension scales *------------------------------------------------------------------------- */ @@ -1122,7 +1122,7 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) goto out; } /* nelmts */ - /* Free refrences */ + /* Free references */ if (is_new_ref) { if (H5Treclaim(tid, sid, H5P_DEFAULT, ndsbuf) < 0) goto out; @@ -1189,7 +1189,7 @@ out: dsbuf = NULL; } if (buf) { - /* Failure occured before H5Treclaim was called; + /* Failure occurred before H5Treclaim was called; free the pointers allocated when we read data in */ for (i = 0; i < rank; i++) { if (buf[i].p) @@ -1633,7 +1633,7 @@ H5DSiterate_scales(hid_t did, unsigned int dim, int *ds_idx, H5DS_iterate_t visi if (H5DSwith_new_ref(did, &is_new_ref) < 0) return FAIL; - /* get the number of scales assotiated with this DIM */ + /* get the number of scales associated with this DIM */ if ((nscales = H5DSget_num_scales(did, dim)) < 0) return FAIL; diff --git a/hl/src/H5LTpublic.h b/hl/src/H5LTpublic.h index f19d353..a6c7c84 100644 --- a/hl/src/H5LTpublic.h +++ b/hl/src/H5LTpublic.h @@ -19,7 +19,7 @@ #define H5LT_FILE_IMAGE_DONT_COPY 0x0002 /* The HDF5 lib won't copy */ /* user supplied image buffer. The same image is open with the core driver. */ #define H5LT_FILE_IMAGE_DONT_RELEASE 0x0004 /* The HDF5 lib won't */ -/* deallocate user supplied image buffer. The user application is reponsible */ +/* deallocate user supplied image buffer. The user application is responsible */ /* for doing so. */ #define H5LT_FILE_IMAGE_ALL 0x0007 diff --git a/hl/test/test_ds.c b/hl/test/test_ds.c index c66ef49..12b6840 100644 --- a/hl/test/test_ds.c +++ b/hl/test/test_ds.c @@ -3021,7 +3021,7 @@ test_simple(void) } /*------------------------------------------------------------------------- - * dettach for DIM0 + * detach for DIM0 *------------------------------------------------------------------------- */ @@ -3319,11 +3319,11 @@ test_simple(void) goto out; dim = 0; - /* iterate trough the 1st dimension of "dset_a" and verify that its DS is valid */ + /* iterate through the 1st dimension of "dset_a" and verify that its DS is valid */ if (H5DSiterate_scales(did, dim, NULL, verify_scale, NULL) < 0) goto out; - /* iterate trough the 2nd dimension of "dset_a" and verify that its DS is valid + /* iterate through the 2nd dimension of "dset_a" and verify that its DS is valid start at DS index 2 */ dim = 1; scale_idx = 2; @@ -3344,11 +3344,11 @@ test_simple(void) goto out; dim = 0; - /* iterate trough the 1st dimension of "dset_a" and read the DS */ + /* iterate through the 1st dimension of "dset_a" and read the DS */ if (H5DSiterate_scales(did, dim, NULL, read_scale, s1_wbuf) < 0) goto out; - /* iterate trough the 2nd dimension of "dset_a" and read the DS + /* iterate through the 2nd dimension of "dset_a" and read the DS start at DS index 2 */ dim = 1; scale_idx = 2; @@ -3381,9 +3381,9 @@ test_simple(void) goto out; { int match_size; /* does this scale size matches the dataset DIM size */ - int idx = 0; /* scale index to start iterating, on return, index where iterator stoped */ + int idx = 0; /* scale index to start iterating, on return, index where iterator stopped */ - /* iterate trough all the dimensions */ + /* iterate through all the dimensions */ for (dim = 0; dim < (unsigned)rank; dim++) { if ((match_size = H5DSiterate_scales(did, dim, &idx, match_dim_scale, NULL)) < 0) goto out; @@ -3475,9 +3475,9 @@ test_simple(void) goto out; { int match_size; /* does this scale size matches the dataset DIM size */ - int idx; /* scale index to start iterating, on return, index where iterator stoped */ + int idx; /* scale index to start iterating, on return, index where iterator stopped */ - /* iterate trough all the dimensions */ + /* iterate through all the dimensions */ for (dim = 0; dim < (unsigned)rank; dim++) { /* always start at 1st scale */ idx = 0; @@ -4025,7 +4025,7 @@ test_errors(void) PASSED(); /*------------------------------------------------------------------------- - * try to attach a scale to an image, pallete or table + * try to attach a scale to an image, palette or table *------------------------------------------------------------------------- */ @@ -4036,7 +4036,7 @@ test_errors(void) goto out; /* make a palette */ - if (H5IMmake_palette(fid, "pallete", pal_dims, NULL) < 0) + if (H5IMmake_palette(fid, "palette", pal_dims, NULL) < 0) goto out; /* open the previous written "ds_b" */ @@ -4248,7 +4248,7 @@ test_iterators(void) if ((did = H5Dopen2(fid, "dset_a", H5P_DEFAULT)) < 0) goto out; - /* try to iterate trough the 1st dimension of "dset_a", return error */ + /* try to iterate through the 1st dimension of "dset_a", return error */ if (H5DSiterate_scales(did, 0, NULL, verify_scale, NULL) < 0) goto out; @@ -4269,7 +4269,7 @@ test_iterators(void) if ((did = H5Dopen2(fid, "dset_a", H5P_DEFAULT)) < 0) goto out; - /* try to iterate trough the 3rd dimension of "dset_a", return error */ + /* try to iterate through the 3rd dimension of "dset_a", return error */ if (H5DSiterate_scales(did, 3, NULL, verify_scale, NULL) == SUCCEED) goto out; @@ -4306,7 +4306,7 @@ test_iterators(void) goto out; } - /* iterate trough the 1st dimension of "dset_a" */ + /* iterate through the 1st dimension of "dset_a" */ if (H5DSiterate_scales(did, 0, NULL, op_continue, NULL) < 0) goto out; diff --git a/hl/test/test_file_image.c b/hl/test/test_file_image.c index bde8adc..4448e41 100644 --- a/hl/test/test_file_image.c +++ b/hl/test/test_file_image.c @@ -109,12 +109,13 @@ test_file_image(size_t open_images, size_t nflags, const unsigned *flags) input_flags[i] = flags[(nflags + i) % nflags]; /* allocate name buffer for image i */ - filename[i] = (char *)HDmalloc(sizeof(char) * 32); + size_t filenamelength = sizeof(char) * 32; + filename[i] = (char *)HDmalloc(filenamelength); if (!filename[i]) FAIL_PUTS_ERROR("HDmalloc() failed"); /* create file name */ - HDsprintf(filename[i], "image_file%d.h5", (int)i); + HDsnprintf(filename[i], filenamelength, "image_file%d.h5", (int)i); /* create file */ if ((file_id[i] = H5Fcreate(filename[i], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -243,7 +244,7 @@ test_file_image(size_t open_images, size_t nflags, const unsigned *flags) */ HDmemset((uint8_t *)tmp_ptr + SUPER_STATUS_FLAGS_OFF_V0_V1, (int)0, (size_t)SUPER_STATUS_FLAGS_SIZE_V0_V1); - /* Does the comparision */ + /* Does the comparison */ if (HDmemcmp(tmp_ptr, buf_ptr[i], (size_t)buf_size[i]) != 0) FAIL_PUTS_ERROR("comparison of TMP vfd and user buffer failed"); /* Free the temporary buffer */ diff --git a/hl/test/test_h5do_compat.c b/hl/test/test_h5do_compat.c index c554ae7..6f19d4b 100644 --- a/hl/test/test_h5do_compat.c +++ b/hl/test/test_h5do_compat.c @@ -31,7 +31,7 @@ * * Purpose: Test the basic functionality of H5DOwrite_chunk * - * Return: Success: An identifer for the dataset used in the tests + * Return: Success: An identifier for the dataset used in the tests * Failure: H5I_INVALID_HID * *------------------------------------------------------------------------- diff --git a/hl/test/test_image.c b/hl/test/test_image.c index 81340c1..4997df6 100644 --- a/hl/test/test_image.c +++ b/hl/test/test_image.c @@ -219,7 +219,7 @@ test_simple(void) *------------------------------------------------------------------------- */ - HL_TESTING2("pallete functions"); + HL_TESTING2("palette functions"); if (H5IMget_npalettes(fid, IMAGE1_NAME, &npals) < 0) goto out; @@ -422,7 +422,7 @@ test_data(void) /*------------------------------------------------------------------------- * palette #4. blue-red - * make a palette whith blue to red colors + * make a palette with blue to red colors *------------------------------------------------------------------------- */ for (i = 0, n = 0; i < 256 * 3; i += 3, n++) { @@ -650,7 +650,7 @@ test_generate(void) HL_TESTING2("make indexed image from land data"); for (i = 0; i < n_elements; i++) { - if (data[i] < 0.0f) + if (data[i] < 0.0F) image_data[i] = 0; else image_data[i] = (unsigned char)((255 * data[i]) / xmax); @@ -671,10 +671,10 @@ test_generate(void) HL_TESTING2("make indexed image from sea data"); for (i = 0; i < n_elements; i++) { - if (data[i] > 0.0f) + if (data[i] > 0.0F) image_data[i] = 0; else { - image_data[i] = (unsigned char)((255.0f * (data[i] - xmin)) / (xmax - xmin)); + image_data[i] = (unsigned char)((255.0F * (data[i] - xmin)) / (xmax - xmin)); } } diff --git a/hl/test/test_ld.c b/hl/test/test_ld.c index dc5755c..150d04c 100644 --- a/hl/test/test_ld.c +++ b/hl/test/test_ld.c @@ -189,7 +189,7 @@ typedef struct set_t { * This will fail on heiwa and amani when VALID_FIELDS1 is "field1,field3,field4" * because of alignment problems: * amani and heiwa - 8 byte alignment - * jam - 4 byte alignemnt + * jam - 4 byte alignment * This will need to be fixed in the library for H5Tget_native_type(). */ /* VALID_FIELDS1 "field1,field2.a,field3,field4" */ @@ -213,8 +213,8 @@ typedef struct test_valid_fields2 { #define TEST_BUF_SIZE 100 int * iibuf; /* buffer for storing retrieved elements */ int * ibuf; /* buffer for storing retrieved elements (integer) */ -set_t * cbuf; /* buffer for storing retrieved elemnets (compound) */ -set_t * ccbuf; /* buffer for storing retrieved elemnets (compound) */ +set_t * cbuf; /* buffer for storing retrieved elements (compound) */ +set_t * ccbuf; /* buffer for storing retrieved elements (compound) */ test_valid_fields1 *vbuf1; /* buffer for storing retrieved elements (FIELDS1) */ test_valid_fields2 *vbuf2; /* buffer for storing retrieved elements (FIELDS2) */ diff --git a/hl/test/test_packet.c b/hl/test/test_packet.c index 5f30d4b..e8b90f7 100644 --- a/hl/test/test_packet.c +++ b/hl/test/test_packet.c @@ -47,7 +47,7 @@ typedef struct particle_t { */ static particle_t testPart[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, - {"Four", 40, 40, 4.0F, 40.0}, {"Five", 50, 50, 5.0F, 50.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /*------------------------------------------------------------------------- diff --git a/hl/test/test_packet_vlen.c b/hl/test/test_packet_vlen.c index 6d6bf34..aa88a6c 100644 --- a/hl/test/test_packet_vlen.c +++ b/hl/test/test_packet_vlen.c @@ -109,7 +109,7 @@ test_VLof_atomic(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ @@ -246,7 +246,7 @@ test_VLof_comptype(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ @@ -418,7 +418,7 @@ test_compound_VL_VLtype(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ @@ -582,7 +582,7 @@ test_VLof_VLtype(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ @@ -677,7 +677,7 @@ error: /* An error has occurred. Clean up and exit. */ * * Description: * - Added a fixed-length packet table to the file for variety - * - Use the helper funtion verify_ptlengthtype to test H5PTis_varlen + * - Use the helper function verify_ptlengthtype to test H5PTis_varlen * on each packet table. * * 2016/01/27 -BMR @@ -1144,7 +1144,7 @@ testfl_VLof_atomic(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ @@ -1281,7 +1281,7 @@ testfl_VLof_comptype(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ @@ -1453,7 +1453,7 @@ testfl_compound_VL_VLtype(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ @@ -1617,7 +1617,7 @@ testfl_VLof_VLtype(void) if (ret < 0) goto error; - HDsprintf(msg, "The number of packets in the packet table must be %u\n", NRECORDS); + HDsnprintf(msg, sizeof(msg), "The number of packets in the packet table must be %u\n", NRECORDS); VERIFY(count == NRECORDS, msg); /* Read all five packets back */ diff --git a/hl/test/test_table.c b/hl/test/test_table.c index 6199e27..90a930c 100644 --- a/hl/test/test_table.c +++ b/hl/test/test_table.c @@ -1464,7 +1464,7 @@ test_table(hid_t fid, int do_write) HL_TESTING2("getting field info"); - /* alocate */ + /* allocate */ names_out = (char **)HDmalloc(sizeof(char *) * (size_t)NFIELDS); for (i = 0; i < NFIELDS; i++) { names_out[i] = (char *)HDmalloc(sizeof(char) * 255); diff --git a/hl/tools/gif2h5/decompress.c b/hl/tools/gif2h5/decompress.c index 64d9643..ca33f86 100644 --- a/hl/tools/gif2h5/decompress.c +++ b/hl/tools/gif2h5/decompress.c @@ -178,7 +178,7 @@ Decompress(GIFIMAGEDESC *GifImageDesc, GIFHEAD *GifHead) DataMask = (1 << ((GifHead->PackedField & 0x07) + 1)) - 1; Raster = GifImageDesc->GIFImage; - /* Check for image seperator */ + /* Check for image separator */ /* Now read in values from the image descriptor */ IWidth = GifImageDesc->ImageWidth; @@ -193,7 +193,7 @@ Decompress(GIFIMAGEDESC *GifImageDesc, GIFHEAD *GifHead) */ /* - * Start reading the raster data. First we get the intial code size and + * Start reading the raster data. First we get the initial code size and * compute decompressor constant values, based on this code size. */ diff --git a/hl/tools/gif2h5/gif.h b/hl/tools/gif2h5/gif.h index 1a8cfe4..5ea8633 100644 --- a/hl/tools/gif2h5/gif.h +++ b/hl/tools/gif2h5/gif.h @@ -129,7 +129,7 @@ typedef struct _GifCommentExtension { ** in the HDF file. ** I have assumed that the ImageDescriptors and GraphicControls follow ** one another, ie. I have not associated them with each other. The driver -** must assume a 1-1 correspondance. The same discussion with plain text +** must assume a 1-1 correspondence. The same discussion with plain text ** extension. */ typedef struct _GifToMem { diff --git a/hl/tools/gif2h5/gif2mem.c b/hl/tools/gif2h5/gif2mem.c index c66250b..2c2225e 100644 --- a/hl/tools/gif2h5/gif2mem.c +++ b/hl/tools/gif2h5/gif2mem.c @@ -40,7 +40,7 @@ Gif2Mem(GIFBYTE *MemGif, GIFTOMEM *GifMemoryStruct) { /* * The gif structure outline for passing data to memory is given in gif.h. - * These pointers are redunant, should take them out in ver. 2 + * These pointers are redundant, should take them out in ver. 2 */ GIFHEAD * gifHead; /* GIF Header structure */ GIFIMAGEDESC ** gifImageDesc; /* Logical Image Descriptor struct */ diff --git a/hl/tools/gif2h5/gifread.c b/hl/tools/gif2h5/gifread.c index dd9e5be..3a6bcae 100644 --- a/hl/tools/gif2h5/gifread.c +++ b/hl/tools/gif2h5/gifread.c @@ -50,15 +50,15 @@ GetByte(const GIFBYTE *MemGif) * Global Color Table (if any) from a GIF image file. The information * is stored in a GIFHEAD structure. * - * Returns: -1 if a FILE stream error occured during the read, - * otherwise 0 if no error occured. + * Returns: -1 if a FILE stream error occurred during the read, + * otherwise 0 if no error occurred. */ int ReadGifHeader(GIFHEAD * GifHead, /* Pointer to GIF header structure */ GIFBYTE **MemGif2) /* GIF image file input FILE stream */ { GIFWORD i; /* Loop counter */ - GIFWORD tableSize; /* Number of entires in the Global Color Table */ + GIFWORD tableSize; /* Number of entries in the Global Color Table */ GifHead->TableSize = 0; for (i = 0; i < 6; i++) { @@ -103,7 +103,7 @@ ReadGifHeader(GIFHEAD * GifHead, /* Pointer to GIF header structure */ return -1; #endif /* 0 */ - return 0; /* No FILE stream error occured */ + return 0; /* No FILE stream error occurred */ } /* @@ -116,8 +116,8 @@ ReadGifHeader(GIFHEAD * GifHead, /* Pointer to GIF header structure */ ** Note that the ImageSeparator field value in the GIFIMAGEDESC ** structure is assigned by the function calling ReadGifImageDesc(). ** -** Returns: -1 if a FILE stream error occured during the read, -** otherwise 0 if no error occured. +** Returns: -1 if a FILE stream error occurred during the read, +** otherwise 0 if no error occurred. */ int ReadGifImageDesc(GIFIMAGEDESC *GifImageDesc, /* Pointer to GIF image descriptor structure */ @@ -196,7 +196,7 @@ ReadGifImageDesc(GIFIMAGEDESC *GifImageDesc, /* Pointer to GIF image descriptor *TempPtr++ = *(*MemGif2)++; } while (ch1); - return (0); /* No FILE stream error occured */ + return (0); /* No FILE stream error occurred */ } /* @@ -205,8 +205,8 @@ ReadGifImageDesc(GIFIMAGEDESC *GifImageDesc, /* Pointer to GIF image descriptor ** Note that the Introducer and Label field values in the GIFGRAPHICCONTROL ** structure are assigned by the function calling ReadGifGraphicControl(). ** -** Returns: -1 if a FILE stream error occured during the read, -** otherwise 0 if no error occured. +** Returns: -1 if a FILE stream error occurred during the read, +** otherwise 0 if no error occurred. */ int ReadGifGraphicControl(GIFGRAPHICCONTROL *GifGraphicControl, /* Pointer to GC Extension structure */ @@ -219,7 +219,7 @@ ReadGifGraphicControl(GIFGRAPHICCONTROL *GifGraphicControl, /* Pointer to GC Ext GifGraphicControl->GCEDump[i] = *(*MemGif2)++; } - return (0); /* No FILE stream error occured */ + return (0); /* No FILE stream error occurred */ } /* @@ -228,8 +228,8 @@ ReadGifGraphicControl(GIFGRAPHICCONTROL *GifGraphicControl, /* Pointer to GC Ext ** Note that the Introducer and Label field values in the GIFLPLAINTEXT ** structure are assigned by the function calling ReadGifPlainText(). ** -** Returns: -1 if a FILE stream error occured during the read, -** otherwise 0 if no error occured. +** Returns: -1 if a FILE stream error occurred during the read, +** otherwise 0 if no error occurred. */ int ReadGifPlainText(GIFPLAINTEXT *GifPlainText, /* Pointer to Plain Text Extension structure */ @@ -256,7 +256,7 @@ ReadGifPlainText(GIFPLAINTEXT *GifPlainText, /* Pointer to Plain Text Extension return(-1); */ - return (0); /* No FILE stream error occured */ + return (0); /* No FILE stream error occurred */ } /* @@ -265,8 +265,8 @@ ReadGifPlainText(GIFPLAINTEXT *GifPlainText, /* Pointer to Plain Text Extension ** Note that the Introducer and Label field values in the GIFAPPLICATION ** structure are assigned by the function calling ReadGifApplication(). ** -** Returns: -1 if a FILE stream error occured during the read, -** otherwise 0 if no error occured. +** Returns: -1 if a FILE stream error occurred during the read, +** otherwise 0 if no error occurred. */ int ReadGifApplication(GIFAPPLICATION *GifApplication, /* Pointer to Application Extension structure */ @@ -292,7 +292,7 @@ ReadGifApplication(GIFAPPLICATION *GifApplication, /* Pointer to Application Ext return(-1); */ - return (0); /* No FILE stream error occured */ + return (0); /* No FILE stream error occurred */ } /* @@ -301,8 +301,8 @@ ReadGifApplication(GIFAPPLICATION *GifApplication, /* Pointer to Application Ext ** Note that the Introducer and Label field values in the GIFCOMMENT ** structure are assigned by the function calling ReadGifComment(). ** -** Returns: -1 if a FILE stream error occured during the read, -** otherwise 0 if no error occured. +** Returns: -1 if a FILE stream error occurred during the read, +** otherwise 0 if no error occurred. */ int ReadGifComment(GIFCOMMENT *GifComment, /* Pointer to GIF Comment Extension structure */ @@ -316,7 +316,7 @@ ReadGifComment(GIFCOMMENT *GifComment, /* Pointer to GIF Comment Extension struc GifComment->Terminator = 0; - return (0); /* No FILE stream error occured */ + return (0); /* No FILE stream error occurred */ } /* @@ -326,8 +326,8 @@ ReadGifComment(GIFCOMMENT *GifComment, /* Pointer to GIF Comment Extension struc ** A GIF "sub-block" is a single count byte followed by 1 to 255 ** additional data bytes. ** -** Returns: A NULL pointer if a memory allocation error occured, -** otherwise a valid pointer if no error occured. +** Returns: A NULL pointer if a memory allocation error occurred, +** otherwise a valid pointer if no error occurred. */ static GIFBYTE * ReadDataSubBlocks(GIFBYTE **MemGif2, /* GIF image file input FILE stream */ @@ -364,7 +364,7 @@ ReadDataSubBlocks(GIFBYTE **MemGif2, /* GIF image file input FILE stream if ((dataSize = *(*MemGif2)++) == 0) break; /* Block Terminator encountered */ - /* Increase the buffer size to accomodate the next sub-block */ + /* Increase the buffer size to accommodate the next sub-block */ if (!(ptr1 = ptr2 = (GIFBYTE *)realloc(ptr2, bufSize + dataSize + 1))) return ((GIFBYTE *)NULL); diff --git a/hl/tools/gif2h5/h52giftest.sh.in b/hl/tools/gif2h5/h52giftest.sh.in index 2f84ef6..4f4057f 100644 --- a/hl/tools/gif2h5/h52giftest.sh.in +++ b/hl/tools/gif2h5/h52giftest.sh.in @@ -46,7 +46,7 @@ TOOLTEST() # Verify the test runs with failure (return code is not 0) # Use for testing if tool can handle error conditions like -# illegal input, bad arguments, exeeding limits, ... +# illegal input, bad arguments, exceeding limits, ... TOOLTESTFAIL() { # for now, discard any error messages generated. diff --git a/hl/tools/gif2h5/writehdf.c b/hl/tools/gif2h5/writehdf.c index 6ec6af4..b49e2a6 100644 --- a/hl/tools/gif2h5/writehdf.c +++ b/hl/tools/gif2h5/writehdf.c @@ -27,7 +27,7 @@ * Programmer: Unknown * * Modifications: pvn - * Use the HDF5 IMAGE API to write the HDF5 image and pallete + * Use the HDF5 IMAGE API to write the HDF5 image and palette * * Date: January, 31, 2006 * @@ -68,7 +68,7 @@ WriteHDF(GIFTOMEM GifMemoryStruct, char *HDFName) #endif /* UNUSED */ if ((file_id = H5Fcreate(HDFName, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) { - /* error occured opening the HDF File for write */ + /* error occurred opening the HDF File for write */ fprintf(stderr, "HDF file could not be opened for writing\n"); fprintf(stderr, "NOTE: GIF file must be present in the same directory as the binary on UNIX systems.\n"); @@ -100,7 +100,7 @@ WriteHDF(GIFTOMEM GifMemoryStruct, char *HDFName) dims[1] = gifImageDesc->ImageWidth; /* create the image name */ - sprintf(ImageName, "Image%d", i); + snprintf(ImageName, sizeof(ImageName), "Image%d", i); /* write image */ if (H5IMmake_image_8bit(file_id, ImageName, dims[1], dims[0], (gifImageDesc->Image)) < 0) diff --git a/hl/tools/h5watch/CMakeLists.txt b/hl/tools/h5watch/CMakeLists.txt index be983ba..1ab473d 100644 --- a/hl/tools/h5watch/CMakeLists.txt +++ b/hl/tools/h5watch/CMakeLists.txt @@ -40,29 +40,6 @@ if (HDF5_ENABLE_FORMATTERS) endif () if (BUILD_TESTING AND HDF5_TEST_SWMR AND HDF5_TEST_SERIAL) - #-- Add swmr_check_compat_vfd program - set (hl_swmr_check_compat_vfd_SOURCES - ${HDF5_HL_TOOLS_H5WATCH_SOURCE_DIR}/swmr_check_compat_vfd.c - ) - add_executable (hl_swmr_check_compat_vfd ${hl_swmr_check_compat_vfd_SOURCES}) - target_compile_options(hl_swmr_check_compat_vfd PRIVATE "${HDF5_CMAKE_C_FLAGS}") - target_include_directories (hl_swmr_check_compat_vfd PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") - if (NOT ONLY_SHARED_LIBS) - TARGET_C_PROPERTIES (hl_swmr_check_compat_vfd STATIC) - target_link_libraries (hl_swmr_check_compat_vfd PRIVATE ${HDF5_LIB_TARGET}) - else () - TARGET_C_PROPERTIES (hl_swmr_check_compat_vfd SHARED) - target_link_libraries (hl_swmr_check_compat_vfd PRIVATE ${HDF5_LIBSH_TARGET}) - endif () - set_target_properties (hl_swmr_check_compat_vfd PROPERTIES FOLDER tools/hl) - - #----------------------------------------------------------------------------- - # Add Target to clang-format - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_HL_TOOLS_H5WATCH_hl_swmr_check_compat_vfd_FORMAT hl_swmr_check_compat_vfd) - endif () - #-- Add extend_dset program set (extend_dset_SOURCES ${HDF5_HL_TOOLS_H5WATCH_SOURCE_DIR}/extend_dset.c diff --git a/hl/tools/h5watch/CMakeTests.cmake b/hl/tools/h5watch/CMakeTests.cmake index 60c099b..b2f689b 100644 --- a/hl/tools/h5watch/CMakeTests.cmake +++ b/hl/tools/h5watch/CMakeTests.cmake @@ -149,18 +149,22 @@ add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tes # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -set (SWMR_INCOMPAT ${hl_swmr_check_compat_vfd}) +add_test ( + NAME H5WATCH-SWMR_INCOMPAT + COMMAND swmr_check_compat_vfd +) +set_tests_properties (H5WATCH-SWMR_INCOMPAT PROPERTIES FIXTURES_SETUP swmr_vfd_check_compat) -if (NOT SWMR_INCOMPAT) # Remove any output file left over from previous test run - add_test ( - NAME H5WATCH-clearall-objects - COMMAND ${CMAKE_COMMAND} -E remove WATCH.h5 - ) - if (last_test) - set_tests_properties (H5WATCH-clearall-objects PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "H5WATCH-clearall-objects") +add_test ( + NAME H5WATCH-clearall-objects + COMMAND ${CMAKE_COMMAND} -E remove WATCH.h5 +) +set_tests_properties (H5WATCH-clearall-objects PROPERTIES FIXTURES_REQUIRED swmr_vfd_check_compat) +if (last_test) + set_tests_properties (H5WATCH-clearall-objects PROPERTIES DEPENDS ${last_test}) +endif () +set (last_test "H5WATCH-clearall-objects") ################################################################################################# # # @@ -182,32 +186,32 @@ if (NOT SWMR_INCOMPAT) # # ################################################################################################# # create the output files to be used. - add_test (NAME H5WATCH-h5watchgentest COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) - set_tests_properties (H5WATCH-h5watchgentest PROPERTIES - WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles" - DEPENDS "H5WATCH-clearall-objects" - ) - set_tests_properties (H5WATCH-h5watchgentest PROPERTIES FIXTURES_SETUP gen_test_watch) - set (last_test "H5WATCH-h5watchgentest") +add_test (NAME H5WATCH-h5watchgentest COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) +set_tests_properties (H5WATCH-h5watchgentest PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles" + DEPENDS "H5WATCH-clearall-objects" +) +set_tests_properties (H5WATCH-h5watchgentest PROPERTIES FIXTURES_SETUP gen_test_watch) +set_tests_properties (H5WATCH-h5watchgentest PROPERTIES FIXTURES_REQUIRED swmr_vfd_check_compat) +set (last_test "H5WATCH-h5watchgentest") # Test on --help options - ADD_H5_TEST (w-help1 0 --help) +ADD_H5_TEST (w-help1 0 --help) # # Tests on expected failures - ADD_H5_ERR_TEST (w-err-dset1 1 WATCH.h5) - ADD_H5_ERR_TEST (w-err-dset2 1 WATCH.h5/group/DSET_CMPD) - ADD_H5_ERR_TEST (w-err-dset-none 1 WATCH.h5/DSET_NONE) - ADD_H5_ERR_TEST (w-err-dset-nomax 1 WATCH.h5/DSET_NOMAX) - ADD_H5_ERR_TEST (w-err-file 1 ../WATCH.h5/DSET_CMPD) - ADD_H5_TEST (w-err-width 1 --width=-8 WATCH.h5/DSET_ONE) - ADD_H5_TEST (w-err-poll 1 --polling=-8 WATCH.h5/DSET_ONE) - ADD_H5_TEST (w-err-poll0 1 --polling=0 WATCH.h5/DSET_ONE) +ADD_H5_ERR_TEST (w-err-dset1 1 WATCH.h5) +ADD_H5_ERR_TEST (w-err-dset2 1 WATCH.h5/group/DSET_CMPD) +ADD_H5_ERR_TEST (w-err-dset-none 1 WATCH.h5/DSET_NONE) +ADD_H5_ERR_TEST (w-err-dset-nomax 1 WATCH.h5/DSET_NOMAX) +ADD_H5_ERR_TEST (w-err-file 1 ../WATCH.h5/DSET_CMPD) +ADD_H5_TEST (w-err-width 1 --width=-8 WATCH.h5/DSET_ONE) +ADD_H5_TEST (w-err-poll 1 --polling=-8 WATCH.h5/DSET_ONE) +ADD_H5_TEST (w-err-poll0 1 --polling=0 WATCH.h5/DSET_ONE) # # Tests on invalid field names via --fields option for a compound typed dataset: DSET_CMPD - ADD_H5_ERR_TEST (w-err-cmpd1 1 --fields=fieldx WATCH.h5/DSET_CMPD) - ADD_H5_ERR_TEST (w-err-cmpd2 1 --fields=field1,field2. WATCH.h5/DSET_CMPD) - ADD_H5_ERR_TEST (w-err-cmpd3 1 --fields=field1,field2, WATCH.h5/DSET_CMPD) - ADD_H5_ERR_TEST (w-err-cmpd4 1 --fields=field1,field2.b.k WATCH.h5/DSET_CMPD) - ADD_H5_ERR_TEST (w-err-cmpd5 1 --fields=field1 --fields=field2.b.k WATCH.h5/DSET_CMPD) +ADD_H5_ERR_TEST (w-err-cmpd1 1 --fields=fieldx WATCH.h5/DSET_CMPD) +ADD_H5_ERR_TEST (w-err-cmpd2 1 --fields=field1,field2. WATCH.h5/DSET_CMPD) +ADD_H5_ERR_TEST (w-err-cmpd3 1 --fields=field1,field2, WATCH.h5/DSET_CMPD) +ADD_H5_ERR_TEST (w-err-cmpd4 1 --fields=field1,field2.b.k WATCH.h5/DSET_CMPD) +ADD_H5_ERR_TEST (w-err-cmpd5 1 --fields=field1 --fields=field2.b.k WATCH.h5/DSET_CMPD) # -endif () diff --git a/hl/tools/h5watch/Makefile.am b/hl/tools/h5watch/Makefile.am index 5112965..13bd820 100644 --- a/hl/tools/h5watch/Makefile.am +++ b/hl/tools/h5watch/Makefile.am @@ -24,7 +24,6 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib -I$(top_srcdir)/hl/sr # These are our main targets, the tools bin_PROGRAMS=h5watch -noinst_PROGRAMS=swmr_check_compat_vfd # Add h5watch specific linker flags here h5watch_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) @@ -38,7 +37,7 @@ if BUILD_TESTS_CONDITIONAL TEST_SCRIPT=testh5watch.sh check_SCRIPTS=$(TEST_SCRIPT) SCRIPT_DEPEND=swmr_check_compat_vfd$(EXEEXT) extend_dset$(EXEEXT) h5watch$(EXEEXT) - noinst_PROGRAMS+=h5watchgentest extend_dset + noinst_PROGRAMS=h5watchgentest extend_dset # Add extend_dset specific preprocessor flags here # (add the main test subdirectory to the include file path) extend_dset_CPPFLAGS=$(AM_CPPFLAGS) -I$(top_srcdir)/test diff --git a/hl/tools/h5watch/extend_dset.c b/hl/tools/h5watch/extend_dset.c index 282f5c2..7756506 100644 --- a/hl/tools/h5watch/extend_dset.c +++ b/hl/tools/h5watch/extend_dset.c @@ -83,7 +83,7 @@ extend_dset_two(const char *file, char *dname, int action1, int action2) size_t dtype_size; /* size of the dataset's datatype */ unsigned num_elmts; /* number of elements in the dataset */ int * ibuf = NULL; /* buffer for storing retrieved elements (integer) */ - set_t * cbuf = NULL; /* buffer for storing retrieved elemnets (compound) */ + set_t * cbuf = NULL; /* buffer for storing retrieved elements (compound) */ /* Allocate memory */ if (NULL == (ibuf = (int *)HDcalloc(TEST_BUF_SIZE, sizeof(int)))) @@ -237,7 +237,7 @@ extend_dset_one(const char *file, char *dname, int action) hsize_t count[1]; /* dimension sizes of appended data */ size_t dtype_size; /* size of the dataset's datatype */ int * ibuf = NULL; /* buffer for storing retrieved elements (integer) */ - set_t * cbuf = NULL; /* buffer for storing retrieved elemnets (compound) */ + set_t * cbuf = NULL; /* buffer for storing retrieved elements (compound) */ /* Allocate memory */ if (NULL == (ibuf = (int *)HDcalloc(TEST_BUF_SIZE, sizeof(int)))) @@ -392,7 +392,7 @@ error: *********************************************************************** */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { char *dname = NULL; char *fname = NULL; diff --git a/hl/tools/h5watch/h5watch.c b/hl/tools/h5watch/h5watch.c index afbc31d..5a6fe6f 100644 --- a/hl/tools/h5watch/h5watch.c +++ b/hl/tools/h5watch/h5watch.c @@ -51,7 +51,7 @@ static herr_t process_cmpd_fields(hid_t fid, char *dsetname); static herr_t check_dataset(hid_t fid, char *dsetname); static void leave(int ret); static void usage(const char *prog); -static void parse_command_line(int argc, const char *argv[]); +static void parse_command_line(int argc, const char *const *argv); /* * Command-line options: The user can only specify long-named parameters. @@ -174,9 +174,9 @@ doprint(hid_t did, const hsize_t *start, const hsize_t *block, int rank) } /* end else */ /* Floating point types should display full precision */ - sprintf(fmt_float, "%%1.%dg", FLT_DIG); + snprintf(fmt_float, sizeof(fmt_float), "%%1.%dg", FLT_DIG); info.fmt_float = fmt_float; - sprintf(fmt_double, "%%1.%dg", DBL_DIG); + snprintf(fmt_double, sizeof(fmt_double), "%%1.%dg", DBL_DIG); info.fmt_double = fmt_double; info.dset_format = "DSET-%s "; @@ -665,7 +665,7 @@ usage(const char *prog) *------------------------------------------------------------------------- */ static void -parse_command_line(int argc, const char *argv[]) +parse_command_line(int argc, const char *const *argv) { int opt; /* Command line option */ int tmp; @@ -790,7 +790,7 @@ catch_signal(int H5_ATTR_UNUSED signo) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { char drivername[50]; /* VFD name */ char *fname = NULL; /* File name */ @@ -819,7 +819,7 @@ main(int argc, const char *argv[]) } /* parse command line options */ - parse_command_line(argc, argv); + parse_command_line(argc, (const char *const *)argv); if (argc <= H5_optind) { error_msg("missing dataset name\n"); @@ -921,7 +921,7 @@ main(int argc, const char *argv[]) } } - /* If everything is fine, start monitoring the datset */ + /* If everything is fine, start monitoring the dataset */ if (h5tools_getstatus() != EXIT_FAILURE) if (monitor_dataset(fid, dname) < 0) h5tools_setstatus(EXIT_FAILURE); diff --git a/hl/tools/h5watch/swmr_check_compat_vfd.c b/hl/tools/h5watch/swmr_check_compat_vfd.c deleted file mode 100644 index a2340bf..0000000 --- a/hl/tools/h5watch/swmr_check_compat_vfd.c +++ /dev/null @@ -1,55 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* Purpose: This is a small program that checks if the HDF5_DRIVER - * environment variable is set to a value that supports SWMR. - * - * It is intended for use in shell scripts. - */ - -#include - -#include "H5private.h" - -/* This file needs to access the file driver testing code */ -#define H5FD_FRIEND /*suppress error about including H5FDpkg */ -#define H5FD_TESTING -#include "H5FDpkg.h" /* File drivers */ - -/*------------------------------------------------------------------------- - * Function: main - * - * Purpose: Inspects the HDF5_DRIVER environment variable, which - * determines the VFD that the test harness will use with - * the majority of the tests. - * - * Return: VFD supports SWMR: EXIT_SUCCESS - * - * VFD does not support SWMR - * or failure: EXIT_FAILURE - * - *------------------------------------------------------------------------- - */ -int -main(void) -{ - char *driver = NULL; - - driver = HDgetenv("HDF5_DRIVER"); - - if (H5FD__supports_swmr_test(driver)) - return EXIT_SUCCESS; - else - return EXIT_FAILURE; - -} /* end main() */ diff --git a/hl/tools/h5watch/testh5watch.sh.in b/hl/tools/h5watch/testh5watch.sh.in index 67ffcc3..0432384 100644 --- a/hl/tools/h5watch/testh5watch.sh.in +++ b/hl/tools/h5watch/testh5watch.sh.in @@ -13,10 +13,11 @@ # # Tests for the h5watch tool # +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -./swmr_check_compat_vfd +$utils_testdir/swmr_check_compat_vfd rc=$? if [ $rc != 0 ] ; then echo @@ -66,10 +67,6 @@ fi nerrors=0 verbose=yes -# The build (current) directory might be different than the source directory. -if test -z "$srcdir"; then - srcdir=. -fi test -d ../testfiles || mkdir ../testfiles # Print a line-line message left justified in a field of 70 characters diff --git a/java/examples/datasets/H5Ex_D_Sofloat.java b/java/examples/datasets/H5Ex_D_Sofloat.java index a42aba4..8edde09 100644 --- a/java/examples/datasets/H5Ex_D_Sofloat.java +++ b/java/examples/datasets/H5Ex_D_Sofloat.java @@ -21,6 +21,9 @@ ************************************************************/ package examples.datasets; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Locale; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; @@ -123,8 +126,9 @@ public class H5Ex_D_Sofloat { } // Print the maximum value. - System.out.println("Maximum value in write buffer is: " + max); - System.out.println("Minimum value in write buffer is: " + min); + DecimalFormat df = new DecimalFormat("#,##0.000000", new DecimalFormatSymbols(Locale.US)); + System.out.println("Maximum value in write buffer is: " + df.format(max)); + System.out.println("Minimum value in write buffer is: " + df.format(min)); // Create a new file using the default properties. try { @@ -308,8 +312,9 @@ public class H5Ex_D_Sofloat { } // Print the maximum value. - System.out.println("Maximum value in " + DATASETNAME + " is: " + max); - System.out.println("Minimum value in " + DATASETNAME + " is: " + min); + DecimalFormat df = new DecimalFormat("#,##0.000000", new DecimalFormatSymbols(Locale.US)); + System.out.println("Maximum value in " + DATASETNAME + " is: " + df.format(max)); + System.out.println("Minimum value in " + DATASETNAME + " is: " + df.format(min)); // End access to the dataset and release resources used by it. try { diff --git a/java/examples/datasets/H5Ex_D_UnlimitedMod.java b/java/examples/datasets/H5Ex_D_UnlimitedMod.java index 273ac3e..0708898 100644 --- a/java/examples/datasets/H5Ex_D_UnlimitedMod.java +++ b/java/examples/datasets/H5Ex_D_UnlimitedMod.java @@ -17,7 +17,7 @@ file. Next, it reopens the file, reads back the data, outputs it to the screen, extends the dataset, and writes new data to the entire extended dataset. Finally it - reopens the file again, reads back the data, and utputs it + reopens the file again, reads back the data, and outputs it to the screen. ************************************************************/ package examples.datasets; diff --git a/java/examples/datasets/JavaDatasetExample.sh.in b/java/examples/datasets/JavaDatasetExample.sh.in index f29739a..9a5ddd8 100644 --- a/java/examples/datasets/JavaDatasetExample.sh.in +++ b/java/examples/datasets/JavaDatasetExample.sh.in @@ -57,8 +57,8 @@ $top_builddir/java/src/jni/.libs/libhdf5_java.* $top_builddir/java/src/$JARFILE " LIST_JAR_TESTFILES=" -$HDFLIB_HOME/slf4j-api-1.7.25.jar -$HDFLIB_HOME/ext/slf4j-simple-1.7.25.jar +$HDFLIB_HOME/slf4j-api-1.7.33.jar +$HDFLIB_HOME/ext/slf4j-simple-1.7.33.jar " LIST_DATA_FILES=" $HDFTEST_HOME/../testfiles/examples.datasets.H5Ex_D_Alloc.txt @@ -220,7 +220,7 @@ JAVAEXEFLAGS=@H5_JAVAFLAGS@ COPY_LIBFILES_TO_BLDLIBDIR COPY_DATAFILES_TO_BLDDIR -CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.25.jar:"$BLDLIBDIR"/slf4j-simple-1.7.25.jar:"$TESTJARFILE"" +CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.33.jar:"$BLDLIBDIR"/slf4j-simple-1.7.33.jar:"$TESTJARFILE"" TEST=/usr/bin/test if [ ! -x /usr/bin/test ] diff --git a/java/examples/datasets/Makefile.am b/java/examples/datasets/Makefile.am index 41a914b..8b71ced 100644 --- a/java/examples/datasets/Makefile.am +++ b/java/examples/datasets/Makefile.am @@ -27,7 +27,7 @@ classes: pkgpath = examples/datasets hdfjarfile = jar$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).jar -CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.25.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.25.jar:$$CLASSPATH +CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.33.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.33.jar:$$CLASSPATH jarfile = jar$(PACKAGE_TARNAME)datasets.jar diff --git a/java/examples/datatypes/H5Ex_T_ObjectReference.java b/java/examples/datatypes/H5Ex_T_ObjectReference.java index 38536b8..b0f98de 100644 --- a/java/examples/datatypes/H5Ex_T_ObjectReference.java +++ b/java/examples/datatypes/H5Ex_T_ObjectReference.java @@ -70,7 +70,7 @@ public class H5Ex_T_ObjectReference { long group_id = HDF5Constants.H5I_INVALID_HID; long dataset_id = HDF5Constants.H5I_INVALID_HID; long[] dims = { DIM0 }; - byte[][] dset_data = new byte[DIM0][8]; + byte[][] dset_data = new byte[DIM0][HDF5Constants.H5R_REF_BUF_SIZE]; // Create a new file using default properties. try { @@ -111,50 +111,58 @@ public class H5Ex_T_ObjectReference { e.printStackTrace(); } - // Create references to the previously created objects. Passing -1 - // as space_id causes this parameter to be ignored. Other values - // besides valid dataspaces result in an error. try { if (file_id >= 0) { - byte rbuf0[] = H5.H5Rcreate(file_id, GROUPNAME, HDF5Constants.H5R_OBJECT, -1); - byte rbuf1[] = H5.H5Rcreate(file_id, DATASETNAME2, HDF5Constants.H5R_OBJECT, -1); - for (int indx = 0; indx < 8; indx++) { - dset_data[0][indx] = rbuf0[indx]; - dset_data[1][indx] = rbuf1[indx]; + try { + dset_data[0] = H5.H5Rcreate_object(file_id, GROUPNAME, HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + } + + try { + dset_data[1] = H5.H5Rcreate_object(file_id, DATASETNAME2, HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); } } + + // Create dataspace. Setting maximum size to NULL sets the maximum + // size to be the current size. + try { + filespace_id = H5.H5Screate_simple(RANK, dims, null); + } + catch (Exception e) { + e.printStackTrace(); + } + + // Create the dataset. + try { + if ((file_id >= 0) && (filespace_id >= 0)) + dataset_id = H5.H5Dcreate(file_id, DATASETNAME, HDF5Constants.H5T_STD_REF, filespace_id, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + } + catch (Exception e) { + e.printStackTrace(); + } + + // Write the object references to it. + try { + if (dataset_id >= 0) + H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, dset_data); + } + catch (Exception e) { + e.printStackTrace(); + } } - catch (Exception e) { - e.printStackTrace(); - } - - // Create dataspace. Setting maximum size to NULL sets the maximum - // size to be the current size. - try { - filespace_id = H5.H5Screate_simple(RANK, dims, null); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Create the dataset. - try { - if ((file_id >= 0) && (filespace_id >= 0)) - dataset_id = H5.H5Dcreate(file_id, DATASETNAME, HDF5Constants.H5T_STD_REF_OBJ, filespace_id, - HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Write the object references to it. - try { - if (dataset_id >= 0) - H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF_OBJ, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, dset_data); + catch (Exception ex) { + ex.printStackTrace(); } - catch (Exception e) { - e.printStackTrace(); + finally { + try {H5.H5Rdestroy(dset_data[1]);} catch (Exception ex) {} + try {H5.H5Rdestroy(dset_data[0]);} catch (Exception ex) {} } // End access to the dataset and release resources used by it. @@ -192,140 +200,91 @@ public class H5Ex_T_ObjectReference { int object_type = -1; long object_id = HDF5Constants.H5I_INVALID_HID; long[] dims = { DIM0 }; - byte[][] dset_data; + byte[][] dset_data = new byte[DIM0][HDF5Constants.H5R_REF_BUF_SIZE]; // Open an existing file. try { file_id = H5.H5Fopen(FILENAME, HDF5Constants.H5F_ACC_RDONLY, HDF5Constants.H5P_DEFAULT); - } - catch (Exception e) { - e.printStackTrace(); - } - // Open an existing dataset. - try { - if (file_id >= 0) - dataset_id = H5.H5Dopen(file_id, DATASETNAME, HDF5Constants.H5P_DEFAULT); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Get dataspace and allocate memory for read buffer. - try { - if (dataset_id >= 0) - dataspace_id = H5.H5Dget_space(dataset_id); - } - catch (Exception e) { - e.printStackTrace(); - } - - try { - if (dataspace_id >= 0) - H5.H5Sget_simple_extent_dims(dataspace_id, dims, null); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Allocate array of pointers to two-dimensional arrays (the - // elements of the dataset. - dset_data = new byte[(int)dims[0]][8]; - - // Read the data using the default properties. - try { - if (dataset_id >= 0) { - H5.H5Dread(dataset_id, HDF5Constants.H5T_STD_REF_OBJ, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, dset_data); - } - } - catch (Exception e) { - e.printStackTrace(); - } - - // Output the data to the screen. - for (int indx = 0; indx < dims[0]; indx++) { - System.out.println(DATASETNAME + "[" + indx + "]:"); - System.out.print(" ->"); - // Open the referenced object, get its name and type. + // Open an existing dataset. try { - if (dataset_id >= 0) { - object_id = H5.H5Rdereference(dataset_id, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5R_OBJECT, dset_data[indx]); - object_type = H5.H5Rget_obj_type(dataset_id, HDF5Constants.H5R_OBJECT, dset_data[indx]); - } - String obj_name = null; - if (object_type >= 0) { - // Get the length of the name and retrieve the name. - obj_name = H5.H5Iget_name(object_id); - } - if ((object_id >= 0) && (object_type >= -1)) { - switch (H5G_obj.get(object_type)) { - case H5G_GROUP: - System.out.print("H5G_GROUP"); - try { - if (object_id >= 0) - H5.H5Gclose(object_id); - } - catch (Exception e) { - e.printStackTrace(); - } - break; - case H5G_DATASET: - System.out.print("H5G_DATASET"); + dataset_id = H5.H5Dopen(file_id, DATASETNAME, HDF5Constants.H5P_DEFAULT); + + try { + // Get dataspace and allocate memory for read buffer. + dataspace_id = H5.H5Dget_space(dataset_id); + H5.H5Sget_simple_extent_dims(dataspace_id, dims, null); + + // Read data. + H5.H5Dread(dataset_id, HDF5Constants.H5T_STD_REF, HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, dset_data); + + // Output the data to the screen. + for (int indx = 0; indx < dims[0]; indx++) { + System.out.println(DATASETNAME + "[" + indx + "]:"); + System.out.print(" ->"); + // Open the referenced object, get its name and type. try { - if (object_id >= 0) - H5.H5Dclose(object_id); + object_id = H5.H5Ropen_object(dset_data[indx], HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + try { + object_type = H5.H5Rget_obj_type3(dset_data[indx], HDF5Constants.H5R_OBJECT); + String obj_name = null; + if (object_type >= 0) { + // Get the name. + obj_name = H5.H5Iget_name(object_id); + } + if ((object_id >= 0) && (object_type >= -1)) { + switch (H5G_obj.get(object_type)) { + case H5G_GROUP: + System.out.print("H5G_GROUP"); + break; + case H5G_DATASET: + System.out.print("H5G_DATASET"); + break; + case H5G_TYPE: + System.out.print("H5G_TYPE"); + break; + default: + System.out.print("UNHANDLED"); + } + } + // Print the name. + System.out.println(": " + obj_name); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + try {H5.H5Oclose(object_id);} catch (Exception e) {} + } } - catch (Exception e) { - e.printStackTrace(); + catch (Exception e4) { + e4.printStackTrace(); } - break; - case H5G_TYPE: - System.out.print("H5G_TYPE"); - try { - if (object_id >= 0) - H5.H5Tclose(object_id); - } - catch (Exception e) { - e.printStackTrace(); + finally { + try {H5.H5Rdestroy(dset_data[indx]);} catch (Exception e4) {} } - break; - default: - System.out.print("UNHANDLED"); - } + } // end for + } + catch (Exception e3) { + e3.printStackTrace(); + } + finally { + try {H5.H5Sclose(dataspace_id);} catch (Exception e3) {} } - // Print the name. - System.out.println(": " + obj_name); } - catch (Exception e) { - e.printStackTrace(); + catch (Exception e2) { + e2.printStackTrace(); + } + finally { + try {H5.H5Dclose(dataset_id);} catch (Exception e2) {} } } - - // End access to the dataset and release resources used by it. - try { - if (dataspace_id >= 0) - H5.H5Sclose(dataspace_id); - } - catch (Exception e) { - e.printStackTrace(); - } - - try { - if (dataset_id >= 0) - H5.H5Dclose(dataset_id); - } - catch (Exception e) { - e.printStackTrace(); + catch (Exception e1) { + e1.printStackTrace(); } - - // Close the file. - try { - if (file_id >= 0) - H5.H5Fclose(file_id); - } - catch (Exception e) { - e.printStackTrace(); + finally { + try {H5.H5Fclose(file_id);} catch (Exception e1) {} } } diff --git a/java/examples/datatypes/H5Ex_T_ObjectReferenceAttribute.java b/java/examples/datatypes/H5Ex_T_ObjectReferenceAttribute.java index b38b0a0..f61ae0d 100644 --- a/java/examples/datatypes/H5Ex_T_ObjectReferenceAttribute.java +++ b/java/examples/datatypes/H5Ex_T_ObjectReferenceAttribute.java @@ -18,7 +18,6 @@ Next, it reopens the file, dereferences the references, and outputs the names of their targets to the screen. ************************************************************/ - package examples.datatypes; import java.util.EnumSet; @@ -72,7 +71,7 @@ public class H5Ex_T_ObjectReferenceAttribute { long dataset_id = HDF5Constants.H5I_INVALID_HID; long attribute_id = HDF5Constants.H5I_INVALID_HID; long[] dims = { DIM0 }; - byte[][] dset_data = new byte[DIM0][8]; + byte[][] dset_data = new byte[DIM0][HDF5Constants.H5R_REF_BUF_SIZE]; // Create a new file using default properties. try { @@ -86,7 +85,7 @@ public class H5Ex_T_ObjectReferenceAttribute { // Create dataset with a scalar dataspace. try { dataspace_id = H5.H5Screate(HDF5Constants.H5S_SCALAR); - if (dataspace_id >= 0) { + if ((file_id >= 0) && (dataspace_id >= 0)) { dataset_id = H5.H5Dcreate(file_id, DATASETNAME2, HDF5Constants.H5T_STD_I32LE, dataspace_id, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); if (dataset_id >= 0) @@ -113,64 +112,72 @@ public class H5Ex_T_ObjectReferenceAttribute { e.printStackTrace(); } - // Create references to the previously created objects. Passing -1 - // as space_id causes this parameter to be ignored. Other values - // besides valid dataspaces result in an error. try { if (file_id >= 0) { - byte rbuf0[] = H5.H5Rcreate(file_id, GROUPNAME, HDF5Constants.H5R_OBJECT, -1); - byte rbuf1[] = H5.H5Rcreate(file_id, DATASETNAME2, HDF5Constants.H5R_OBJECT, -1); - for (int indx = 0; indx < 8; indx++) { - dset_data[0][indx] = rbuf0[indx]; - dset_data[1][indx] = rbuf1[indx]; + try { + dset_data[0] = H5.H5Rcreate_object(file_id, GROUPNAME, HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + } + + try { + dset_data[1] = H5.H5Rcreate_object(file_id, DATASETNAME2, HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); } } - } - catch (Exception e) { - e.printStackTrace(); - } - // Create dataset with a scalar dataspace to serve as the parent - // for the attribute. - try { - dataspace_id = H5.H5Screate(HDF5Constants.H5S_SCALAR); - if (dataspace_id >= 0) { - dataset_id = H5.H5Dcreate(file_id, DATASETNAME, HDF5Constants.H5T_STD_I32LE, dataspace_id, - HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); - H5.H5Sclose(dataspace_id); - dataspace_id = HDF5Constants.H5I_INVALID_HID; + // Create dataset with a scalar dataspace to serve as the parent + // for the attribute. + try { + dataspace_id = H5.H5Screate(HDF5Constants.H5S_SCALAR); + if (dataspace_id >= 0) { + dataset_id = H5.H5Dcreate(file_id, DATASETNAME, HDF5Constants.H5T_STD_I32LE, dataspace_id, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + H5.H5Sclose(dataspace_id); + dataspace_id = HDF5Constants.H5I_INVALID_HID; + } + } + catch (Exception e) { + e.printStackTrace(); + } + + // Create dataspace. Setting maximum size to NULL sets the maximum + // size to be the current size. + try { + dataspace_id = H5.H5Screate_simple(RANK, dims, null); + } + catch (Exception e) { + e.printStackTrace(); + } + + // Create the attribute and write the array data to it. + try { + if ((dataset_id >= 0) && (dataspace_id >= 0)) + attribute_id = H5.H5Acreate(dataset_id, ATTRIBUTENAME, HDF5Constants.H5T_STD_REF, dataspace_id, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + } + catch (Exception e) { + e.printStackTrace(); + } + + // Write the dataset. + try { + if (attribute_id >= 0) + H5.H5Awrite(attribute_id, HDF5Constants.H5T_STD_REF, dset_data); + } + catch (Exception e) { + e.printStackTrace(); } } - catch (Exception e) { - e.printStackTrace(); - } - - // Create dataspace. Setting maximum size to NULL sets the maximum - // size to be the current size. - try { - dataspace_id = H5.H5Screate_simple(RANK, dims, null); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Create the attribute and write the array data to it. - try { - if ((dataset_id >= 0) && (dataspace_id >= 0)) - attribute_id = H5.H5Acreate(dataset_id, ATTRIBUTENAME, HDF5Constants.H5T_STD_REF_OBJ, dataspace_id, - HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Write the dataset. - try { - if (attribute_id >= 0) - H5.H5Awrite(attribute_id, HDF5Constants.H5T_STD_REF_OBJ, dset_data); + catch (Exception ex) { + ex.printStackTrace(); } - catch (Exception e) { - e.printStackTrace(); + finally { + try {H5.H5Rdestroy(dset_data[1]);} catch (Exception ex) {} + try {H5.H5Rdestroy(dset_data[0]);} catch (Exception ex) {} } // End access to the dataset and release resources used by it. @@ -190,7 +197,6 @@ public class H5Ex_T_ObjectReferenceAttribute { e.printStackTrace(); } - // Terminate access to the data space. try { if (dataspace_id >= 0) H5.H5Sclose(dataspace_id); @@ -207,7 +213,6 @@ public class H5Ex_T_ObjectReferenceAttribute { catch (Exception e) { e.printStackTrace(); } - } private static void ReadDataset() { @@ -218,158 +223,102 @@ public class H5Ex_T_ObjectReferenceAttribute { int object_type = -1; long object_id = HDF5Constants.H5I_INVALID_HID; long[] dims = { DIM0 }; - byte[][] dset_data; + byte[][] dset_data = new byte[DIM0][HDF5Constants.H5R_REF_BUF_SIZE]; // Open an existing file. try { file_id = H5.H5Fopen(FILENAME, HDF5Constants.H5F_ACC_RDONLY, HDF5Constants.H5P_DEFAULT); - } - catch (Exception e) { - e.printStackTrace(); - } - // Open an existing dataset. - try { - if (file_id >= 0) - dataset_id = H5.H5Dopen(file_id, DATASETNAME, HDF5Constants.H5P_DEFAULT); - } - catch (Exception e) { - e.printStackTrace(); - } - - try { - if (dataset_id >= 0) - attribute_id = H5.H5Aopen_by_name(dataset_id, ".", ATTRIBUTENAME, HDF5Constants.H5P_DEFAULT, - HDF5Constants.H5P_DEFAULT); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Get dataspace and allocate memory for read buffer. - try { - if (attribute_id >= 0) - dataspace_id = H5.H5Aget_space(attribute_id); - } - catch (Exception e) { - e.printStackTrace(); - } - - try { - if (dataspace_id >= 0) - H5.H5Sget_simple_extent_dims(dataspace_id, dims, null); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Allocate array of pointers to two-dimensional arrays (the - // elements of the dataset. - dset_data = new byte[(int) dims[0]][8]; - - // Read data. - try { - if (attribute_id >= 0) - H5.H5Aread(attribute_id, HDF5Constants.H5T_STD_REF_OBJ, dset_data); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Output the data to the screen. - for (int indx = 0; indx < dims[0]; indx++) { - System.out.println(ATTRIBUTENAME + "[" + indx + "]:"); - System.out.print(" ->"); - // Open the referenced object, get its name and type. + // Open an existing dataset. try { - if (dataset_id >= 0) { - object_id = H5.H5Rdereference(dataset_id, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5R_OBJECT, dset_data[indx]); - object_type = H5.H5Rget_obj_type(dataset_id, HDF5Constants.H5R_OBJECT, dset_data[indx]); + dataset_id = H5.H5Dopen(file_id, DATASETNAME, HDF5Constants.H5P_DEFAULT); + + try { + attribute_id = H5.H5Aopen_by_name(dataset_id, ".", ATTRIBUTENAME, HDF5Constants.H5P_DEFAULT, + HDF5Constants.H5P_DEFAULT); + + // Get dataspace and allocate memory for read buffer. + try { + dataspace_id = H5.H5Aget_space(attribute_id); + H5.H5Sget_simple_extent_dims(dataspace_id, dims, null); + + // Read data. + H5.H5Aread(attribute_id, HDF5Constants.H5T_STD_REF, dset_data); + + // Output the data to the screen. + for (int indx = 0; indx < dims[0]; indx++) { + System.out.println(ATTRIBUTENAME + "[" + indx + "]:"); + System.out.print(" ->"); + // Open the referenced object, get its name and type. + try { + object_id = H5.H5Ropen_object(dset_data[indx], HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + try { + object_type = H5.H5Rget_obj_type3(dset_data[indx], HDF5Constants.H5R_OBJECT); + String obj_name = null; + if (object_type >= 0) { + // Get the name. + obj_name = H5.H5Iget_name(object_id); + } + if ((object_id >= 0) && (object_type >= -1)) { + switch (H5G_obj.get(object_type)) { + case H5G_GROUP: + System.out.print("H5G_GROUP"); + break; + case H5G_DATASET: + System.out.print("H5G_DATASET"); + break; + case H5G_TYPE: + System.out.print("H5G_TYPE"); + break; + default: + System.out.print("UNHANDLED"); + } + } + // Print the name. + System.out.println(": " + obj_name); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + try {H5.H5Oclose(object_id);} catch (Exception e) {} + } + } + catch (Exception e5) { + e5.printStackTrace(); + } + finally { + try {H5.H5Rdestroy(dset_data[indx]);} catch (Exception e5) {} + } + } // end for + } + catch (Exception e4) { + e4.printStackTrace(); + } + finally { + try {H5.H5Sclose(dataspace_id);} catch (Exception e3) {} + } } - String obj_name = null; - if (object_type >= 0) { - // Get the length of the name and retrieve the name. - obj_name = H5.H5Iget_name(object_id); + catch (Exception e3) { + e3.printStackTrace(); } - if ((object_id >= 0) && (object_type >= -1)) { - switch (H5G_obj.get(object_type)) { - case H5G_GROUP: - System.out.print("H5G_GROUP"); - try { - if (object_id >= 0) - H5.H5Gclose(object_id); - } - catch (Exception e) { - e.printStackTrace(); - } - break; - case H5G_DATASET: - System.out.print("H5G_DATASET"); - try { - if (object_id >= 0) - H5.H5Dclose(object_id); - } - catch (Exception e) { - e.printStackTrace(); - } - break; - case H5G_TYPE: - System.out.print("H5G_TYPE"); - try { - if (object_id >= 0) - H5.H5Tclose(object_id); - } - catch (Exception e) { - e.printStackTrace(); - } - break; - default: - System.out.print("UNHANDLED"); - } + finally { + try {H5.H5Aclose(attribute_id);} catch (Exception e4) {} } - // Print the name. - System.out.println(": " + obj_name); } - catch (Exception e) { - e.printStackTrace(); + catch (Exception e2) { + e2.printStackTrace(); + } + finally { + try {H5.H5Dclose(dataset_id);} catch (Exception e2) {} } } - - // End access to the dataset and release resources used by it. - try { - if (attribute_id >= 0) - H5.H5Aclose(attribute_id); - } - catch (Exception e) { - e.printStackTrace(); - } - - try { - if (dataset_id >= 0) - H5.H5Dclose(dataset_id); - } - catch (Exception e) { - e.printStackTrace(); - } - - // Terminate access to the data space. - try { - if (dataspace_id >= 0) - H5.H5Sclose(dataspace_id); + catch (Exception e1) { + e1.printStackTrace(); } - catch (Exception e) { - e.printStackTrace(); + finally { + try {H5.H5Fclose(file_id);} catch (Exception e1) {} } - - // Close the file. - try { - if (file_id >= 0) - H5.H5Fclose(file_id); - } - catch (Exception e) { - e.printStackTrace(); - } - } public static void main(String[] args) { diff --git a/java/examples/datatypes/JavaDatatypeExample.sh.in b/java/examples/datatypes/JavaDatatypeExample.sh.in index e26d8c0..c6f5dbc 100644 --- a/java/examples/datatypes/JavaDatatypeExample.sh.in +++ b/java/examples/datatypes/JavaDatatypeExample.sh.in @@ -54,8 +54,8 @@ $top_builddir/java/src/jni/.libs/libhdf5_java.* $top_builddir/java/src/$JARFILE " LIST_JAR_TESTFILES=" -$HDFLIB_HOME/slf4j-api-1.7.25.jar -$HDFLIB_HOME/ext/slf4j-simple-1.7.25.jar +$HDFLIB_HOME/slf4j-api-1.7.33.jar +$HDFLIB_HOME/ext/slf4j-simple-1.7.33.jar " LIST_DATA_FILES=" $HDFTEST_HOME/../testfiles/examples.datatypes.H5Ex_T_Array.txt @@ -216,7 +216,7 @@ JAVAEXEFLAGS=@H5_JAVAFLAGS@ COPY_LIBFILES_TO_BLDLIBDIR COPY_DATAFILES_TO_BLDDIR -CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.25.jar:"$BLDLIBDIR"/slf4j-simple-1.7.25.jar:"$TESTJARFILE"" +CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.33.jar:"$BLDLIBDIR"/slf4j-simple-1.7.33.jar:"$TESTJARFILE"" TEST=/usr/bin/test if [ ! -x /usr/bin/test ] diff --git a/java/examples/datatypes/Makefile.am b/java/examples/datatypes/Makefile.am index 90790f7..55ff91f 100644 --- a/java/examples/datatypes/Makefile.am +++ b/java/examples/datatypes/Makefile.am @@ -27,7 +27,7 @@ classes: pkgpath = examples/datatypes hdfjarfile = jar$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).jar -CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.25.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.25.jar:$$CLASSPATH +CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.33.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.33.jar:$$CLASSPATH jarfile = jar$(PACKAGE_TARNAME)datatypes.jar diff --git a/java/examples/groups/H5Ex_G_Corder.java b/java/examples/groups/H5Ex_G_Corder.java index 5df850e..4fa5aa2 100644 --- a/java/examples/groups/H5Ex_G_Corder.java +++ b/java/examples/groups/H5Ex_G_Corder.java @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /************************************************************ Creating a file with creation properties and traverse the - groups in alpabetical and creation order. + groups in alphabetical and creation order. ************************************************************/ package examples.groups; diff --git a/java/examples/groups/JavaGroupExample.sh.in b/java/examples/groups/JavaGroupExample.sh.in index 3b0e9d1..86f90bf 100644 --- a/java/examples/groups/JavaGroupExample.sh.in +++ b/java/examples/groups/JavaGroupExample.sh.in @@ -56,8 +56,8 @@ $top_builddir/java/src/jni/.libs/libhdf5_java.* $top_builddir/java/src/$JARFILE " LIST_JAR_TESTFILES=" -$HDFLIB_HOME/slf4j-api-1.7.25.jar -$HDFLIB_HOME/ext/slf4j-simple-1.7.25.jar +$HDFLIB_HOME/slf4j-api-1.7.33.jar +$HDFLIB_HOME/ext/slf4j-simple-1.7.33.jar " LIST_ITER_FILES=" $HDFTEST_HOME/h5ex_g_iterate.h5 @@ -255,7 +255,7 @@ COPY_LIBFILES_TO_BLDLIBDIR COPY_DATAFILES_TO_BLDDIR COPY_ITERFILES_TO_BLDITERDIR -CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.25.jar:"$BLDLIBDIR"/slf4j-simple-1.7.25.jar:"$TESTJARFILE"" +CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.33.jar:"$BLDLIBDIR"/slf4j-simple-1.7.33.jar:"$TESTJARFILE"" TEST=/usr/bin/test if [ ! -x /usr/bin/test ] diff --git a/java/examples/groups/Makefile.am b/java/examples/groups/Makefile.am index bfde9ae..c520860 100644 --- a/java/examples/groups/Makefile.am +++ b/java/examples/groups/Makefile.am @@ -27,7 +27,7 @@ classes: pkgpath = examples/groups hdfjarfile = jar$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).jar -CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.25.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.25.jar:$$CLASSPATH +CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.33.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.33.jar:$$CLASSPATH jarfile = jar$(PACKAGE_TARNAME)groups.jar diff --git a/java/examples/intro/JavaIntroExample.sh.in b/java/examples/intro/JavaIntroExample.sh.in index db741e5..a095f27 100644 --- a/java/examples/intro/JavaIntroExample.sh.in +++ b/java/examples/intro/JavaIntroExample.sh.in @@ -54,8 +54,8 @@ $top_builddir/java/src/jni/.libs/libhdf5_java.* $top_builddir/java/src/$JARFILE " LIST_JAR_TESTFILES=" -$HDFLIB_HOME/slf4j-api-1.7.25.jar -$HDFLIB_HOME/ext/slf4j-simple-1.7.25.jar +$HDFLIB_HOME/slf4j-api-1.7.33.jar +$HDFLIB_HOME/ext/slf4j-simple-1.7.33.jar " LIST_DATA_FILES=" $HDFTEST_HOME/../testfiles/examples.intro.H5_CreateDataset.txt @@ -205,7 +205,7 @@ JAVAEXEFLAGS=@H5_JAVAFLAGS@ COPY_LIBFILES_TO_BLDLIBDIR COPY_DATAFILES_TO_BLDDIR -CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.25.jar:"$BLDLIBDIR"/slf4j-simple-1.7.25.jar:"$TESTJARFILE"" +CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/slf4j-api-1.7.33.jar:"$BLDLIBDIR"/slf4j-simple-1.7.33.jar:"$TESTJARFILE"" TEST=/usr/bin/test if [ ! -x /usr/bin/test ] diff --git a/java/examples/intro/Makefile.am b/java/examples/intro/Makefile.am index 7d1aeab..741f122 100644 --- a/java/examples/intro/Makefile.am +++ b/java/examples/intro/Makefile.am @@ -27,7 +27,7 @@ classes: pkgpath = examples/intro hdfjarfile = jar$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).jar -CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.25.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.25.jar:$$CLASSPATH +CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/slf4j-api-1.7.33.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.33.jar:$$CLASSPATH jarfile = jar$(PACKAGE_TARNAME)intro.jar diff --git a/java/examples/testfiles/examples.datasets.H5Ex_D_Sofloat.txt b/java/examples/testfiles/examples.datasets.H5Ex_D_Sofloat.txt index 4d4b5d6..9025ce2 100644 --- a/java/examples/testfiles/examples.datasets.H5Ex_D_Sofloat.txt +++ b/java/examples/testfiles/examples.datasets.H5Ex_D_Sofloat.txt @@ -1,6 +1,6 @@ -Maximum value in write buffer is: 106.66666666666667 -Minimum value in write buffer is: 1.7692307692307692 +Maximum value in write buffer is: 106.666667 +Minimum value in write buffer is: 1.769231 Filter type is: H5Z_FILTER_SCALEOFFSET -Maximum value in DS1 is: 106.66169811320755 -Minimum value in DS1 is: 1.7692307692307692 +Maximum value in DS1 is: 106.661698 +Minimum value in DS1 is: 1.769231 diff --git a/java/lib/ext/slf4j-nop-1.7.25.jar b/java/lib/ext/slf4j-nop-1.7.25.jar deleted file mode 100644 index 78c7295..0000000 Binary files a/java/lib/ext/slf4j-nop-1.7.25.jar and /dev/null differ diff --git a/java/lib/ext/slf4j-nop-1.7.33.jar b/java/lib/ext/slf4j-nop-1.7.33.jar new file mode 100644 index 0000000..aa8fc09 Binary files /dev/null and b/java/lib/ext/slf4j-nop-1.7.33.jar differ diff --git a/java/lib/ext/slf4j-simple-1.7.25.jar b/java/lib/ext/slf4j-simple-1.7.25.jar deleted file mode 100644 index b29ca12..0000000 Binary files a/java/lib/ext/slf4j-simple-1.7.25.jar and /dev/null differ diff --git a/java/lib/ext/slf4j-simple-1.7.33.jar b/java/lib/ext/slf4j-simple-1.7.33.jar new file mode 100644 index 0000000..ebf4846 Binary files /dev/null and b/java/lib/ext/slf4j-simple-1.7.33.jar differ diff --git a/java/lib/slf4j-api-1.7.25.jar b/java/lib/slf4j-api-1.7.25.jar deleted file mode 100644 index 7e62f13..0000000 Binary files a/java/lib/slf4j-api-1.7.25.jar and /dev/null differ diff --git a/java/lib/slf4j-api-1.7.33.jar b/java/lib/slf4j-api-1.7.33.jar new file mode 100644 index 0000000..db3730d Binary files /dev/null and b/java/lib/slf4j-api-1.7.33.jar differ diff --git a/java/src/Makefile.am b/java/src/Makefile.am index a0abb32..5bb72ad 100644 --- a/java/src/Makefile.am +++ b/java/src/Makefile.am @@ -36,7 +36,7 @@ jarfile = jar$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).jar hdf5_javadir = $(libdir) pkgpath = hdf/hdf5lib -CLASSPATH_ENV=CLASSPATH=.:$(top_srcdir)/java/lib/slf4j-api-1.7.25.jar:$$CLASSPATH +CLASSPATH_ENV=CLASSPATH=.:$(top_srcdir)/java/lib/slf4j-api-1.7.33.jar:$$CLASSPATH AM_JAVACFLAGS = $(H5_JAVACFLAGS) -deprecation diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 4129f50..3180ede 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -161,7 +161,7 @@ import hdf.hdf5lib.structs.H5O_token_t; * disk (source) and in memory (destination). *

* For Java, this ``ANY'' is a problem, as the type of data must always be declared. Furthermore, multidimensional - * arrays are definitely not layed out contiguously in memory. It would be infeasible to declare a separate + * arrays are definitely not laid out contiguously in memory. It would be infeasible to declare a separate * routine for every combination of number type and dimensionality. For that reason, the * HDFArray class is used to discover the type, shape, and size of the * data array at run time, and to convert to and from a contiguous array of bytes in synchronized static native C order. @@ -216,7 +216,7 @@ import hdf.hdf5lib.structs.H5O_token_t; * exception handlers to print out the HDF-5 error stack. *


* - * @version HDF5 1.13.1
+ * @version HDF5 1.13.2
* See also: hdf.hdf5lib.HDFArray
* hdf.hdf5lib.HDF5Constants
* hdf.hdf5lib.HDF5CDataTypes
@@ -239,7 +239,7 @@ public class H5 implements java.io.Serializable { * * Make sure to update the versions number when a different library is used. */ - public final static int LIB_VERSION[] = { 1, 13, 1 }; + public final static int LIB_VERSION[] = { 1, 13, 2 }; /** * add system property to load library by path @@ -442,7 +442,7 @@ public class H5 implements java.io.Serializable { /** * Turn on error handling. By default, the C library prints the error stack of the HDF-5 C library on stdout. This - * behavior may be reenabled by calling H5error_on(). + * behavior may be re-enabled by calling H5error_on(). */ public synchronized static native void H5error_on(); @@ -2097,7 +2097,7 @@ public class H5 implements java.io.Serializable { * IN: Identifier for object to which attributes are attached; may be group, dataset, or named datatype. * @param idx_type * IN: The type of index specified by idx_type can be one of the following: - * H5_INDEX_NAME An alpha-numeric index by attribute name. + * H5_INDEX_NAME An alphanumeric index by attribute name. * H5_INDEX_CRT_ORDER An index by creation order. * @param order * IN: The order in which the index is to be traversed, as specified by order, can be one of the following: @@ -2137,7 +2137,7 @@ public class H5 implements java.io.Serializable { * IN: Name of object, relative to location. * @param idx_type * IN: The type of index specified by idx_type can be one of the following: - * H5_INDEX_NAME An alpha-numeric index by attribute name. + * H5_INDEX_NAME An alphanumeric index by attribute name. * H5_INDEX_CRT_ORDER An index by creation order. * @param order * IN: The order in which the index is to be traversed, as specified by order, can be one of the following: @@ -4577,7 +4577,7 @@ public class H5 implements java.io.Serializable { throws HDF5LibraryException, NullPointerException; /** - * Given a mount point, H5Funmount dissassociates the mount point's file from the file mounted there. + * Given a mount point, H5Funmount disassociates the mount point's file from the file mounted there. * * @param loc_id * The identifier for the location at which the specified file is to be unmounted. @@ -5205,7 +5205,7 @@ public class H5 implements java.io.Serializable { */ /** * retrieves information of all objects (recurvisely) under the group (name) located in the file or group specified - * by loc_id upto maximum specified by objMax. + * by loc_id up to maximum specified by objMax. * * @param loc_id * IN: File or group identifier @@ -7182,7 +7182,7 @@ public class H5 implements java.io.Serializable { * @param size * IN: Size the property value. * @param def_value - * IN: Defaul value of the property + * IN: Default value of the property * * @exception HDF5LibraryException * - Error from the HDF-5 Library. @@ -7202,7 +7202,7 @@ public class H5 implements java.io.Serializable { * @param size * IN: Size the property value. * @param value - * IN: Defaul value of the property + * IN: Default value of the property * * @exception HDF5LibraryException * - Error from the HDF-5 Library. @@ -7245,7 +7245,7 @@ public class H5 implements java.io.Serializable { * @param ocpl_id * IN: : Object (dataset or group) creation property list identifier * @param attributes - * The maximun and minimum no. of attributes to be stored. + * The maximum and minimum no. of attributes to be stored. * *
      *      attributes[0] =  The maximum number of attributes to be stored in compact storage
@@ -7353,7 +7353,7 @@ public class H5 implements java.io.Serializable {
      *              transfer property list.  The FLAGS argument specifies certain
      *              general properties of the filter and is documented below.
      *              The CD_VALUES is an array of CD_NELMTS integers which are
-     *              auxiliary data for the filter.  The integer vlues will be
+     *              auxiliary data for the filter.  The integer values will be
      *              stored in the dataset object header as part of the filter
      *              information.
      *

@@ -8054,9 +8054,9 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. * @exception NullPointerException - * - aligment array is null. + * - alignment array is null. * @exception IllegalArgumentException - * - aligment array is invalid. + * - alignment array is invalid. **/ public synchronized static native int H5Pget_alignment(long plist, long[] alignment) throws HDF5LibraryException, NullPointerException, IllegalArgumentException; @@ -8202,7 +8202,7 @@ public class H5 implements java.io.Serializable { throws HDF5LibraryException; /** - * H5Pget_gc_references Returns the current setting for the garbage collection refernces property from a file access + * H5Pget_gc_references Returns the current setting for the garbage collection references property from a file access * property list. * * @param fapl_id @@ -9254,7 +9254,7 @@ public class H5 implements java.io.Serializable { * the raw data chunk cache on a per-datset basis. * * @param dapl_id - * IN: Identifier of the datset access property list. + * IN: Identifier of the dataset access property list. * @param rdcc_nslots * IN: Number of elements (objects) in the raw data chunk cache. * @param rdcc_nbytes diff --git a/java/src/hdf/hdf5lib/exceptions/HDF5LibraryException.java b/java/src/hdf/hdf5lib/exceptions/HDF5LibraryException.java index 532355e..9f94882 100644 --- a/java/src/hdf/hdf5lib/exceptions/HDF5LibraryException.java +++ b/java/src/hdf/hdf5lib/exceptions/HDF5LibraryException.java @@ -114,7 +114,7 @@ public class HDF5LibraryException extends HDF5Exception { return "special zero no error"; } else if (err_code == HDF5Constants.H5E_UNINITIALIZED) { - return "information is unitialized"; + return "information is uninitialized"; } else if (err_code == HDF5Constants.H5E_UNSUPPORTED) { return "feature is unsupported"; diff --git a/java/src/hdf/hdf5lib/package-info.java b/java/src/hdf/hdf5lib/package-info.java index 838a266..2863f5a 100644 --- a/java/src/hdf/hdf5lib/package-info.java +++ b/java/src/hdf/hdf5lib/package-info.java @@ -114,7 +114,7 @@ * disk (source) and in memory (destination). *

* For Java, this ``ANY'' is a problem, as the type of data must always be declared. Furthermore, multidimensional - * arrays are definitely not layed out contiguously in memory. It would be infeasible to declare a separate + * arrays are definitely not laid out contiguously in memory. It would be infeasible to declare a separate * routine for every combination of number type and dimensionality. For that reason, the * HDFArray class is used to discover the type, shape, and size of the * data array at run time, and to convert to and from a contiguous array of bytes in synchronized static native C order. @@ -171,4 +171,4 @@ * * See also: http://hdfgroup.org/HDF5" **/ -package hdf.hdf5lib; \ No newline at end of file +package hdf.hdf5lib; diff --git a/java/src/hdf/hdf5lib/structs/H5F_info2_t.java b/java/src/hdf/hdf5lib/structs/H5F_info2_t.java index f951bb4..78e6d6c 100644 --- a/java/src/hdf/hdf5lib/structs/H5F_info2_t.java +++ b/java/src/hdf/hdf5lib/structs/H5F_info2_t.java @@ -40,7 +40,7 @@ public class H5F_info2_t implements Serializable{ public H5_ih_info_t sohm_msgs_info; /** - * Constructor fot current "global" information about file + * Constructor for current "global" information about file * @param super_version: Superblock version number * @param super_size: Superblock size * @param super_ext_size: Superblock extension size diff --git a/java/src/jni/h5util.c b/java/src/jni/h5util.c index 7313c28..5a01dc9 100644 --- a/java/src/jni/h5util.c +++ b/java/src/jni/h5util.c @@ -182,7 +182,7 @@ size_t h5str_convert(JNIEnv *env, char **in_str, hid_t container, hid_t tid, void *out_buf, size_t out_buf_offset) { unsigned char *ucptr = NULL; - static char fmt_llong[8], fmt_ullong[8]; + char fmt_llong[8], fmt_ullong[8]; H5T_class_t tclass = H5T_NO_CLASS; const char delimiter[] = " ," H5_COMPOUND_BEGIN_INDICATOR H5_COMPOUND_END_INDICATOR H5_ARRAY_BEGIN_INDICATOR H5_ARRAY_END_INDICATOR H5_VLEN_BEGIN_INDICATOR H5_VLEN_END_INDICATOR; @@ -211,10 +211,10 @@ h5str_convert(JNIEnv *env, char **in_str, hid_t container, hid_t tid, void *out_ /* Build default formats for long long types */ if (!fmt_llong[0]) { - if (HDsprintf(fmt_llong, "%%%sd", H5_PRINTF_LL_WIDTH) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_convert: HDsprintf failure"); - if (HDsprintf(fmt_ullong, "%%%su", H5_PRINTF_LL_WIDTH) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_convert: HDsprintf failure"); + if (HDsnprintf(fmt_llong, sizeof(fmt_llong), "%%%sd", H5_PRINTF_LL_WIDTH) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_convert: HDsnprintf failure"); + if (HDsnprintf(fmt_ullong, sizeof(fmt_ullong), "%%%su", H5_PRINTF_LL_WIDTH) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_convert: HDsnprintf failure"); } /* end if */ switch (tclass) { @@ -759,7 +759,7 @@ size_t h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *in_buf, int expand_data) { unsigned char *ucptr = (unsigned char *)in_buf; - static char fmt_llong[8], fmt_ullong[8]; + char fmt_llong[8], fmt_ullong[8]; H5T_class_t tclass = H5T_NO_CLASS; size_t typeSize = 0; H5T_sign_t nsign = H5T_SGN_ERROR; @@ -794,11 +794,12 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i HDmemcpy(&tmp_float, cptr, sizeof(float)); - if (NULL == (this_str = (char *)HDmalloc(25))) + size_t this_len = 25; + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%g", tmp_float) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%g", tmp_float) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); break; } @@ -808,11 +809,12 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i HDmemcpy(&tmp_double, cptr, sizeof(double)); - if (NULL == (this_str = (char *)HDmalloc(25))) + size_t this_len = 25; + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%g", tmp_double) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, "%g", this_len, tmp_double) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); break; } @@ -822,11 +824,12 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i HDmemcpy(&tmp_ldouble, cptr, sizeof(long double)); - if (NULL == (this_str = (char *)HDmalloc(27))) + size_t this_len = 27; + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%Lg", tmp_ldouble) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%Lg", tmp_ldouble) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); break; } @@ -888,25 +891,26 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i unsigned char tmp_uchar = 0; char tmp_char = 0; + size_t this_len = 7; if (H5T_SGN_NONE == nsign) { HDmemcpy(&tmp_uchar, cptr, sizeof(unsigned char)); - if (NULL == (this_str = (char *)HDmalloc(7))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%hhu", tmp_uchar) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%hhu", tmp_uchar) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } else { HDmemcpy(&tmp_char, cptr, sizeof(char)); - if (NULL == (this_str = (char *)HDmalloc(7))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%hhd", tmp_char) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%hhd", tmp_char) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } break; @@ -916,25 +920,26 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i unsigned short tmp_ushort = 0; short tmp_short = 0; + size_t this_len = 9; if (H5T_SGN_NONE == nsign) { HDmemcpy(&tmp_ushort, cptr, sizeof(unsigned short)); - if (NULL == (this_str = (char *)HDmalloc(9))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%hu", tmp_ushort) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%hu", tmp_ushort) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } else { HDmemcpy(&tmp_short, cptr, sizeof(short)); - if (NULL == (this_str = (char *)HDmalloc(9))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%hd", tmp_short) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%hd", tmp_short) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } break; @@ -944,25 +949,26 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i unsigned int tmp_uint = 0; int tmp_int = 0; + size_t this_len = 14; if (H5T_SGN_NONE == nsign) { HDmemcpy(&tmp_uint, cptr, sizeof(unsigned int)); - if (NULL == (this_str = (char *)HDmalloc(14))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%u", tmp_uint) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%u", tmp_uint) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } else { HDmemcpy(&tmp_int, cptr, sizeof(int)); - if (NULL == (this_str = (char *)HDmalloc(14))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%d", tmp_int) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%d", tmp_int) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } break; @@ -972,25 +978,26 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i unsigned long tmp_ulong = 0; long tmp_long = 0; + size_t this_len = 23; if (H5T_SGN_NONE == nsign) { HDmemcpy(&tmp_ulong, cptr, sizeof(unsigned long)); - if (NULL == (this_str = (char *)HDmalloc(23))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%lu", tmp_ulong) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%lu", tmp_ulong) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } else { HDmemcpy(&tmp_long, cptr, sizeof(long)); - if (NULL == (this_str = (char *)HDmalloc(23))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%ld", tmp_long) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%ld", tmp_long) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } break; @@ -1001,25 +1008,26 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i unsigned long long tmp_ullong = 0; long long tmp_llong = 0; + size_t this_len = 25; if (H5T_SGN_NONE == nsign) { HDmemcpy(&tmp_ullong, cptr, sizeof(unsigned long long)); - if (NULL == (this_str = (char *)HDmalloc(25))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, fmt_ullong, tmp_ullong) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, fmt_ullong, tmp_ullong) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } else { HDmemcpy(&tmp_llong, cptr, sizeof(long long)); - if (NULL == (this_str = (char *)HDmalloc(25))) + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, fmt_llong, tmp_llong) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, fmt_llong, tmp_llong) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } break; @@ -1079,17 +1087,18 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i else { size_t i; - if (NULL == (this_str = (char *)HDmalloc(4 * (typeSize + 1)))) + size_t this_len = 4 * (typeSize + 1); + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); if (1 == typeSize) { - if (HDsprintf(this_str, "%#02x", ucptr[0]) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%#02x", ucptr[0]) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } else { for (i = 0; i < typeSize; i++) - if (HDsprintf(this_str, "%s%02x", i ? ":" : "", ucptr[i]) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%s%02x", i ? ":" : "", ucptr[i]) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } } @@ -1134,11 +1143,12 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i else H5_LIBRARY_ERROR(ENVONLY); - if (NULL == (this_str = (char *)HDmalloc(14))) + size_t this_len = 14; + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR( ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%u-", (unsigned)oi.type) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%u-", (unsigned)oi.type) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); if (!h5str_append(out_str, this_str)) H5_ASSERTION_ERROR(ENVONLY, "Unable to append string."); HDfree(this_str); @@ -1150,11 +1160,13 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i H5Otoken_to_str(tid, &oi.token, &token_str); - if (NULL == (this_str = (char *)HDmalloc(64 + strlen(token_str) + 1))) + size_t this_len = 64 + strlen(token_str) + 1; + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR( ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsprintf(this_str, "%lu:%s", oi.fileno, token_str) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%lu:%s", oi.fileno, token_str) < + 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); H5free_memory(token_str); } @@ -1307,17 +1319,18 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i /* All other types get printed as hexadecimal */ - if (NULL == (this_str = (char *)HDmalloc(4 * (typeSize + 1)))) + size_t this_len = 4 * (typeSize + 1); + if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); if (1 == typeSize) { - if (HDsprintf(this_str, "%#02x", ucptr[0]) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%#02x", ucptr[0]) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } else { for (i = 0; i < typeSize; i++) - if (HDsprintf(this_str, "%s%02x", i ? ":" : "", ucptr[i]) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsprintf failure"); + if (HDsnprintf(this_str, this_len, "%s%02x", i ? ":" : "", ucptr[i]) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); } break; @@ -1348,7 +1361,7 @@ done: * This is a special case subfunction to print the data in a region reference of type blocks. * * Return: - * The function returns FAIL if there was an error, otherwise SUCEED + * The function returns FAIL if there was an error, otherwise SUCCEED *------------------------------------------------------------------------- */ static int @@ -1519,8 +1532,9 @@ h5str_dump_region_blocks(JNIEnv *env, h5str_t *str, hid_t region_space, hid_t re for (j = 0; j < ndims; j++) { tmp_str[0] = '\0'; - if (HDsprintf(tmp_str, "%s%lu", j ? "," : "(", (unsigned long)ptdata[i * 2 * ndims + j]) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_dump_region_blocks: HDsprintf failure"); + if (HDsnprintf(tmp_str, sizeof(tmp_str), "%s%lu", j ? "," : "(", + (unsigned long)ptdata[i * 2 * ndims + j]) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_dump_region_blocks: HDsnprintf failure"); if (!h5str_append(str, tmp_str)) H5_ASSERTION_ERROR(ENVONLY, "Unable to append string."); @@ -1529,9 +1543,9 @@ h5str_dump_region_blocks(JNIEnv *env, h5str_t *str, hid_t region_space, hid_t re for (j = 0; j < ndims; j++) { tmp_str[0] = '\0'; - if (HDsprintf(tmp_str, "%s%lu", j ? "," : ")-(", - (unsigned long)ptdata[i * 2 * ndims + j + ndims]) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_dump_region_blocks: HDsprintf failure"); + if (HDsnprintf(tmp_str, sizeof(tmp_str), "%s%lu", j ? "," : ")-(", + (unsigned long)ptdata[i * 2 * ndims + j + ndims]) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_dump_region_blocks: HDsnprintf failure"); if (!h5str_append(str, tmp_str)) H5_ASSERTION_ERROR(ENVONLY, "Unable to append string."); @@ -1696,8 +1710,9 @@ h5str_dump_region_points(JNIEnv *env, h5str_t *str, hid_t region_space, hid_t re for (j = 0; j < ndims; j++) { tmp_str[0] = '\0'; - if (HDsprintf(tmp_str, "%s%lu", j ? "," : "(", (unsigned long)(ptdata[i * ndims + j])) < 0) - H5_JNI_FATAL_ERROR(ENVONLY, "h5str_dump_region_points: HDsprintf failure"); + if (HDsnprintf(tmp_str, sizeof(tmp_str), "%s%lu", j ? "," : "(", + (unsigned long)(ptdata[i * ndims + j])) < 0) + H5_JNI_FATAL_ERROR(ENVONLY, "h5str_dump_region_points: HDsnprintf failure"); if (!h5str_append(str, tmp_str)) H5_ASSERTION_ERROR(ENVONLY, "Unable to append string."); @@ -1771,7 +1786,7 @@ done: * Purpose: Recursive check for variable length string of a datatype. * * Return: - * TRUE : type conatains any variable length string + * TRUE : type contains any variable length string * FALSE : type doesn't contain any variable length string * Negative value: error occur * @@ -2300,7 +2315,7 @@ done: * This is a special case subfunction to print the data in a region reference of type blocks. * * Return: - * The function returns FAIL if there was an error, otherwise SUCEED + * The function returns FAIL if there was an error, otherwise SUCCEED * *------------------------------------------------------------------------- */ diff --git a/java/test/CMakeLists.txt b/java/test/CMakeLists.txt index c6cf607..b1186a4 100644 --- a/java/test/CMakeLists.txt +++ b/java/test/CMakeLists.txt @@ -39,6 +39,7 @@ set (HDF5_JAVA_TEST_SOURCES TestH5Oparams TestH5Obasic TestH5Ocreate + TestH5OcopyOld TestH5Ocopy TestH5PL TestH5VL diff --git a/java/test/Makefile.am b/java/test/Makefile.am index ac3b619..4a6785d 100644 --- a/java/test/Makefile.am +++ b/java/test/Makefile.am @@ -27,7 +27,7 @@ classes: pkgpath = test hdfjarfile = jar$(PACKAGE_TARNAME)-$(PACKAGE_VERSION).jar -CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/junit.jar:$(top_srcdir)/java/lib/hamcrest-core.jar:$(top_srcdir)/java/lib/slf4j-api-1.7.25.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.25.jar:$$CLASSPATH +CLASSPATH_ENV=CLASSPATH=.:$(JAVAROOT):$(top_builddir)/java/src/$(hdfjarfile):$(top_srcdir)/java/lib/junit.jar:$(top_srcdir)/java/lib/hamcrest-core.jar:$(top_srcdir)/java/lib/slf4j-api-1.7.33.jar:$(top_srcdir)/java/lib/ext/slf4j-simple-1.7.33.jar:$$CLASSPATH jarfile = jar$(PACKAGE_TARNAME)test.jar @@ -71,6 +71,7 @@ noinst_JAVA = \ TestH5Oparams.java \ TestH5Obasic.java \ TestH5Ocreate.java \ + TestH5OcopyOld.java \ TestH5Ocopy.java \ TestH5PL.java \ TestH5VL.java \ diff --git a/java/test/TestH5.java b/java/test/TestH5.java index 1f81f09..168f28a 100644 --- a/java/test/TestH5.java +++ b/java/test/TestH5.java @@ -287,7 +287,7 @@ public class TestH5 { */ @Test public void testH5get_libversion() { - int libversion[] = { 1, 13, 1 }; + int libversion[] = { 1, 13, 2 }; try { H5.H5get_libversion(libversion); @@ -326,7 +326,7 @@ public class TestH5 { */ @Test public void testH5check_version() { - int majnum = 1, minnum = 13, relnum = 1; + int majnum = 1, minnum = 13, relnum = 2; try { H5.H5check_version(majnum, minnum, relnum); diff --git a/java/test/TestH5Ocopy.java b/java/test/TestH5Ocopy.java index b3b1acd..821cad3 100644 --- a/java/test/TestH5Ocopy.java +++ b/java/test/TestH5Ocopy.java @@ -136,33 +136,46 @@ public class TestH5Ocopy { @Test public void testH5OcopyRefsAttr() { long ocp_plist_id = HDF5Constants.H5I_INVALID_HID; - byte rbuf0[]=null , rbuf1[] = null; - byte[] dset_data = new byte[16]; + byte[][] dset_data = new byte[2][HDF5Constants.H5R_REF_BUF_SIZE]; long attribute_id = HDF5Constants.H5I_INVALID_HID; - - try { - rbuf0 = H5.H5Rcreate(H5fid, "/G1", HDF5Constants.H5R_OBJECT, -1); - rbuf1 = H5.H5Rcreate(H5fid, "DS2", HDF5Constants.H5R_OBJECT, -1); - //System.arraycopy(rbuf0, 0, dset_data, 0, 8); - System.arraycopy(rbuf1, 0, dset_data, 8, 8); - } - catch (Exception ex) { - fail("testH5OcopyRefsAttr: H5Rcreate failed"); - } - try { - attribute_id = H5.H5Acreate(H5did2, "A1", HDF5Constants.H5T_STD_REF_OBJ, H5dsid, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); - assertTrue("testH5OcopyRefsAttr.H5Acreate: ", attribute_id >= 0); - H5.H5Awrite(attribute_id, HDF5Constants.H5T_STD_REF_OBJ, dset_data); - - H5.H5Aclose(attribute_id); + try { + dset_data[0] = H5.H5Rcreate_object(H5fid, "/G1", HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5OcopyRefsAttr: H5Rcreate_object " + err); + } + + try { + dset_data[1] = H5.H5Rcreate_object(H5fid, "DS2", HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5OcopyRefsAttr: H5Rcreate_object " + err); + } + + try { + attribute_id = H5.H5Acreate(H5did2, "A1", HDF5Constants.H5T_STD_REF, H5dsid, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsAttr.H5Acreate: ", attribute_id >= 0); + H5.H5Awrite(attribute_id, HDF5Constants.H5T_STD_REF, dset_data); + + H5.H5Aclose(attribute_id); + } + catch (Exception ex) { + fail("testH5OcopyRefsAttr: H5Awrite failed"); + } + finally { + try {H5.H5Aclose(attribute_id);} catch (Exception exx) {} + } } catch (Exception ex) { - fail("testH5OcopyRefsAttr: H5Awrite failed"); + ex.printStackTrace(); } finally { - try {H5.H5Aclose(attribute_id);} catch (Exception exx) {} + try {H5.H5Rdestroy(dset_data[1]);} catch (Exception ex) {} + try {H5.H5Rdestroy(dset_data[0]);} catch (Exception ex) {} } try { @@ -181,30 +194,50 @@ public class TestH5Ocopy { @Test public void testH5OcopyRefsDatasettodiffFile() { - byte rbuf1[] = null; - byte[] dset_data = new byte[16]; + byte[][] dset_data = new byte[2][HDF5Constants.H5R_REF_BUF_SIZE]; long ocp_plist_id = HDF5Constants.H5I_INVALID_HID; long dataset_id = HDF5Constants.H5I_INVALID_HID; long H5fid2 = HDF5Constants.H5I_INVALID_HID; try { - rbuf1 = H5.H5Rcreate(H5fid, "DS2", HDF5Constants.H5R_OBJECT, -1); - System.arraycopy(rbuf1, 0, dset_data, 8, 8); - - dataset_id = H5.H5Dcreate(H5fid, "DSREF", - HDF5Constants.H5T_STD_REF_OBJ, H5dsid, - HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); - assertTrue("testH5OcopyRefsDatasettodiffFile.H5Dcreate: ", dataset_id >= 0); - H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF_OBJ, - HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, dset_data); - H5.H5Dclose(dataset_id); + try { + dset_data[0] = H5.H5Rcreate_object(H5fid, "/G1", HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5OcopyRefsDatasettodiffFile: H5Rcreate_object " + err); + } + try { + dset_data[1] = H5.H5Rcreate_object(H5fid, "DS2", HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5OcopyRefsDatasettodiffFile: H5Rcreate_object " + err); + } + + try { + dataset_id = H5.H5Dcreate(H5fid, "DSREF", + HDF5Constants.H5T_STD_REF, H5dsid, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsDatasettodiffFile.H5Dcreate: ", dataset_id >= 0); + H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF, + HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, dset_data); + H5.H5Dclose(dataset_id); + } + catch (Exception ex) { + fail("testH5OcopyRefsDatasettodiffFile: create dataset failed"); + } + finally { + try {H5.H5Dclose(dataset_id);} catch (Exception exx) {} + } } catch (Exception ex) { - fail("testH5OcopyRefsDatasettodiffFile: create dataset failed"); + ex.printStackTrace(); } finally { - try {H5.H5Dclose(dataset_id);} catch (Exception exx) {} + try {H5.H5Rdestroy(dset_data[0]);} catch (Exception ex) {} + try {H5.H5Rdestroy(dset_data[1]);} catch (Exception ex) {} } try { @@ -241,123 +274,154 @@ public class TestH5Ocopy { @Test public void testH5OcopyRefsDatasettosameFile() { - byte rbuf0[]=null , rbuf1[] = null; - byte[] dset_data = new byte[16]; + byte[][] dset_data = new byte[2][HDF5Constants.H5R_REF_BUF_SIZE]; + byte[][] read_data = new byte[2][HDF5Constants.H5R_REF_BUF_SIZE]; long ocp_plist_id = HDF5Constants.H5I_INVALID_HID; long dataset_id = HDF5Constants.H5I_INVALID_HID; long did = HDF5Constants.H5I_INVALID_HID; int obj_type = -1; - byte[] read_data = new byte[16]; try { - rbuf0 = H5.H5Rcreate(H5fid, "/G1", HDF5Constants.H5R_OBJECT, -1); - rbuf1 = H5.H5Rcreate(H5fid, "DS2", HDF5Constants.H5R_OBJECT, -1); - System.arraycopy(rbuf0, 0, dset_data, 0, 8); - System.arraycopy(rbuf1, 0, dset_data, 8, 8); - - //Create a dataset and write object references to it. - dataset_id = H5.H5Dcreate(H5fid, "DSREF", - HDF5Constants.H5T_STD_REF_OBJ, H5dsid, - HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); - assertTrue("testH5OcopyRefsDatasettosameFile.H5Dcreate: ", dataset_id >= 0); - H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF_OBJ, - HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, - HDF5Constants.H5P_DEFAULT, dset_data); - //Close the dataset. - H5.H5Dclose(dataset_id); + try { + dset_data[0] = H5.H5Rcreate_object(H5fid, "/G1", HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5OcopyRefsDatasettosameFile: H5Rcreate_object " + err); + } + + try { + dset_data[1] = H5.H5Rcreate_object(H5fid, "DS2", HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5OcopyRefsDatasettosameFile: H5Rcreate_object " + err); + } + + try { + //Create a dataset and write object references to it. + dataset_id = H5.H5Dcreate(H5fid, "DSREF", + HDF5Constants.H5T_STD_REF, H5dsid, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsDatasettosameFile.H5Dcreate: ", dataset_id >= 0); + H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF, + HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, dset_data); + //Close the dataset. + H5.H5Dclose(dataset_id); + } + catch (Exception ex) { + fail("testH5OcopyRefsDatasettosameFile: create dataset failed"); + } + finally { + try {H5.H5Dclose(dataset_id);} catch (Exception exx) {} + } } catch (Exception ex) { - try {H5.H5Dclose(dataset_id);} catch (Exception exx) {} - fail("testH5OcopyRefsDatasettosameFile: create dataset failed"); + ex.printStackTrace(); + } + finally { + try {H5.H5Rdestroy(dset_data[1]);} catch (Exception ex) {} + try {H5.H5Rdestroy(dset_data[0]);} catch (Exception ex) {} } try { ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); assertTrue("testH5OcopyRefsDatasettosameFile.H5Pcreate: ", ocp_plist_id >= 0); H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); + //Perform copy function. + try { + H5.H5Ocopy(H5fid, "DSREF", H5fid, "CPYREFD", ocp_plist_id, HDF5Constants.H5P_DEFAULT); + } + catch(Exception ex) { + fail("testH5OcopyRefsDatasettosameFile: H5Ocopy failed"); + } } catch (Exception ex) { - try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} fail("testH5OcopyRefsDatasettosameFile: H5Pset_copy_object failed"); } - - //Perform copy function. - try { - H5.H5Ocopy(H5fid, "DSREF", H5fid, "CPYREFD", ocp_plist_id, HDF5Constants.H5P_DEFAULT); - } - catch(Exception ex) { + finally { try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} - fail("testH5OcopyRefsDatasettosameFile: H5Ocopy failed"); } - //Open the dataset that has been copied try { - did = H5.H5Dopen(H5fid, "DSREF", HDF5Constants.H5P_DEFAULT); - assertTrue("testH5OcopyRefsDatasettosameFile.H5Dopen: ", did >= 0); + //Open the dataset that has been copied + try { + did = H5.H5Dopen(H5fid, "DSREF", HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsDatasettosameFile.H5Dopen: ", did >= 0); + } + catch (Exception e) { + e.printStackTrace(); + fail("testH5OcopyRefsDatasettosameFile: H5Dopen failed"); + } + + //Read the dataset object references in the read_data buffer. + try { + H5.H5Dread(did, HDF5Constants.H5T_STD_REF, HDF5Constants.H5S_ALL,HDF5Constants.H5S_ALL, HDF5Constants.H5P_DEFAULT, read_data); + } + catch (Exception e) { + e.printStackTrace(); + fail("testH5OcopyRefsDatasettosameFile: H5Dread failed"); + } } - catch (Exception e) { - try {H5.H5Dclose(did);} catch (Exception exx) {} - e.printStackTrace(); - fail("testH5OcopyRefsDatasettosameFile: H5Dopen failed"); + catch (Exception ex) { + ex.printStackTrace(); + fail("testH5OcopyRefsDatasettosameFile: open and read dataset failed"); + } + finally { + try {H5.H5Dclose(did);} catch (Exception ex) {} } try { - //Read the dataset object references in the read_data buffer. - H5.H5Dread(did, HDF5Constants.H5T_STD_REF_OBJ, HDF5Constants.H5S_ALL,HDF5Constants.H5S_ALL, HDF5Constants.H5P_DEFAULT, read_data); - System.arraycopy(read_data, 0, rbuf0, 0, 8); - System.arraycopy(read_data, 8, rbuf1, 0, 8); - //Get the type of object the reference points to. - obj_type = H5.H5Rget_obj_type(H5fid, HDF5Constants.H5R_OBJECT, rbuf1); + obj_type = H5.H5Rget_obj_type3(read_data[1], HDF5Constants.H5R_OBJECT); assertEquals(obj_type, HDF5Constants.H5O_TYPE_DATASET); - - obj_type = H5.H5Rget_obj_type(H5fid, HDF5Constants.H5R_OBJECT, rbuf0); + + obj_type = H5.H5Rget_obj_type3(read_data[0], HDF5Constants.H5R_OBJECT); assertEquals(obj_type, HDF5Constants.H5O_TYPE_GROUP); } catch (Exception ex) { ex.printStackTrace(); } finally { - try {H5.H5Dclose(did);} catch (Exception ex) {} - try {H5.H5Pclose(ocp_plist_id);} catch (Exception ex) {} + try {H5.H5Rdestroy(read_data[1]);} catch (Exception ex) {} + try {H5.H5Rdestroy(read_data[0]);} catch (Exception ex) {} + } + } + + @Test + public void testH5OcopyNullRef() throws Throwable { + final long _pid_ = HDF5Constants.H5P_DEFAULT; + long sid = HDF5Constants.H5I_INVALID_HID; + long did = HDF5Constants.H5I_INVALID_HID; + long aid = HDF5Constants.H5I_INVALID_HID; + + try { + sid = H5.H5Screate_simple(1, new long[] {1}, null); + assertTrue("testH5OcopyNullRef.H5Screate_simple: ", sid >= 0); + did = H5.H5Dcreate(H5fid, "Dataset_with_null_Ref", HDF5Constants.H5T_NATIVE_INT, sid, _pid_, _pid_, _pid_); + assertTrue("testH5OcopyNullRef.H5Dcreate: ", did > 0); + aid = H5.H5Acreate(did, "Null_Ref", HDF5Constants.H5T_STD_REF, sid, _pid_, _pid_); + assertTrue("testH5OcopyNullRef.H5Acreate: ", aid > 0); } - } - -// @Ignore because of JIRA HDF5-9547 -// @Test(expected = HDF5LibraryException.class) -// public void testH5OcopyInvalidRef() throws Throwable { -// final long _pid_ = HDF5Constants.H5P_DEFAULT; -// long sid = HDF5Constants.H5I_INVALID_HID; -// long did = HDF5Constants.H5I_INVALID_HID; -// long aid = HDF5Constants.H5I_INVALID_HID; -// -// try { -// sid = H5.H5Screate_simple(1, new long[] {1}, null); -// assertTrue("testH5OcopyInvalidRef.H5Screate_simple: ", sid >= 0); -// did = H5.H5Dcreate(H5fid, "Dataset_with_invalid_Ref", HDF5Constants.H5T_NATIVE_INT, sid, _pid_, _pid_, _pid_); -// assertTrue("testH5OcopyInvalidRef.H5Dcreate: ", did > 0); -// aid = H5.H5Acreate(did, "Invalid_Ref", HDF5Constants.H5T_STD_REF_OBJ, sid, _pid_, _pid_); -// assertTrue("testH5OcopyInvalidRef.H5Acreate: ", aid > 0); -// H5.H5Awrite(aid, HDF5Constants.H5T_STD_REF_OBJ, new long[]{-1}); -// } -// catch (Exception ex) { -// ex.printStackTrace(); -// } -// finally { -// try {H5.H5Dclose(did);} catch (Exception exx) {} -// try {H5.H5Aclose(aid);} catch (Exception exx) {} -// try {H5.H5Sclose(sid);} catch (Exception exx) {} -// } -// -// long ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); -// assertTrue("testH5OcopyInvalidRef.H5Pcreate: ", ocp_plist_id >= 0); -// H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); -// try { -// H5.H5Ocopy(H5fid, "/Dataset_with_invalid_Ref", H5fid, "/Dataset_with_invalid_Ref_cp", ocp_plist_id, _pid_); -// } -// finally { -// try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} -// } -// } + catch (Exception ex) { + ex.printStackTrace(); + } + finally { + try {H5.H5Dclose(did);} catch (Exception exx) {} + try {H5.H5Aclose(aid);} catch (Exception exx) {} + try {H5.H5Sclose(sid);} catch (Exception exx) {} + } + + long ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); + assertTrue("testH5OcopyNullRef.H5Pcreate: ", ocp_plist_id >= 0); + H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); + try { + H5.H5Ocopy(H5fid, "/Dataset_with_null_Ref", H5fid, "/Dataset_with_null_Ref_cp", ocp_plist_id, _pid_); + } + finally { + try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} + } + } } diff --git a/java/test/TestH5OcopyOld.java b/java/test/TestH5OcopyOld.java new file mode 100644 index 0000000..e547051 --- /dev/null +++ b/java/test/TestH5OcopyOld.java @@ -0,0 +1,397 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +package test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; + +import hdf.hdf5lib.H5; +import hdf.hdf5lib.HDF5Constants; +import hdf.hdf5lib.exceptions.HDF5Exception; +import hdf.hdf5lib.exceptions.HDF5LibraryException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +public class TestH5OcopyOld { + @Rule public TestName testname = new TestName(); + private static final String FILENAME = "testRefsattributeO.h5"; + private static final int DIM_X = 4; + private static final int DIM_Y = 6; + long H5fid = HDF5Constants.H5I_INVALID_HID; + long H5dsid = HDF5Constants.H5I_INVALID_HID; + long H5did1 = HDF5Constants.H5I_INVALID_HID; + long H5did2 = HDF5Constants.H5I_INVALID_HID; + long H5gcpl = HDF5Constants.H5I_INVALID_HID; + long H5gid = HDF5Constants.H5I_INVALID_HID; + long H5dsid2 = HDF5Constants.H5I_INVALID_HID; + long[] dims = { 2 }; + + private final void _deleteFile(String filename) { + File file = new File(filename); + + if (file.exists()) { + try { + file.delete(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + } + + private final long _createDataset(long fid, long dsid, String name, long dapl) { + long did = HDF5Constants.H5I_INVALID_HID; + try { + did = H5.H5Dcreate(fid, name, + HDF5Constants.H5T_STD_I32BE, dsid, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, dapl); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Dcreate: " + err); + } + assertTrue("TestH5O._createDataset: ",did >= 0); + + return did; + } + + private final long _createGroup(long fid, String name) { + long gid = HDF5Constants.H5I_INVALID_HID; + try { + H5gcpl = HDF5Constants.H5P_DEFAULT; + gid = H5.H5Gcreate(fid, name, HDF5Constants.H5P_DEFAULT, + H5gcpl, HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5.H5Gcreate: " + err); + } + assertTrue("TestH5O._createGroup: ",gid >= 0); + + return gid; + } + + @Before + public void createH5file() + throws NullPointerException, HDF5Exception { + assertTrue("H5 open ids is 0",H5.getOpenIDCount()==0); + System.out.print(testname.getMethodName()); + try { + H5fid = H5.H5Fcreate(FILENAME, HDF5Constants.H5F_ACC_TRUNC, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + H5dsid2 = H5.H5Screate(HDF5Constants.H5S_SCALAR); + H5did1 = _createDataset(H5fid, H5dsid2, "DS2", HDF5Constants.H5P_DEFAULT); + H5dsid = H5.H5Screate_simple(1, dims, null); + H5gid = _createGroup(H5fid, "/G1"); + H5did2 = _createDataset(H5gid, H5dsid, "DS1", HDF5Constants.H5P_DEFAULT); + } + catch (Throwable err) { + err.printStackTrace(); + fail("TestH5O.createH5file: " + err); + } + assertTrue("TestH5O.createH5file: H5.H5Fcreate: ",H5fid >= 0); + assertTrue("TestH5O.createH5file: H5.H5Screate_simple: ",H5dsid >= 0); + assertTrue("TestH5O.createH5file: H5.H5Gcreate: ",H5gid >= 0); + + H5.H5Fflush(H5fid, HDF5Constants.H5F_SCOPE_LOCAL); + } + + @After + public void deleteH5file() throws HDF5LibraryException { + if (H5gid > 0) + try {H5.H5Gclose(H5gid);} catch (Exception ex) {} + if (H5did2 > 0) + try {H5.H5Dclose(H5did2);} catch (Exception ex) {} + if (H5dsid > 0) + try {H5.H5Sclose(H5dsid);} catch (Exception ex) {} + if (H5dsid2 > 0) + try {H5.H5Sclose(H5dsid2);} catch (Exception ex) {} + if (H5did1 > 0) + try {H5.H5Dclose(H5did1);} catch (Exception ex) {} + if (H5fid > 0) + try {H5.H5Fclose(H5fid);} catch (Exception ex) {} + + _deleteFile(FILENAME); + System.out.println(); + } + + @Test + public void testH5OcopyRefsAttr() { + long ocp_plist_id = HDF5Constants.H5I_INVALID_HID; + byte rbuf0[]=null , rbuf1[] = null; + byte[] dset_data = new byte[16]; + long attribute_id = HDF5Constants.H5I_INVALID_HID; + + try { + rbuf0 = H5.H5Rcreate(H5fid, "/G1", HDF5Constants.H5R_OBJECT, -1); + rbuf1 = H5.H5Rcreate(H5fid, "DS2", HDF5Constants.H5R_OBJECT, -1); + //System.arraycopy(rbuf0, 0, dset_data, 0, 8); + System.arraycopy(rbuf1, 0, dset_data, 8, 8); + } + catch (Exception ex) { + fail("testH5OcopyRefsAttr: H5Rcreate failed"); + } + + try { + attribute_id = H5.H5Acreate(H5did2, "A1", HDF5Constants.H5T_STD_REF_OBJ, H5dsid, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsAttr.H5Acreate: ", attribute_id >= 0); + H5.H5Awrite(attribute_id, HDF5Constants.H5T_STD_REF_OBJ, dset_data); + + H5.H5Aclose(attribute_id); + } + catch (Exception ex) { + fail("testH5OcopyRefsAttr: H5Awrite failed"); + } + finally { + try {H5.H5Aclose(attribute_id);} catch (Exception exx) {} + } + + try { + ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); + assertTrue("testH5OcopyRefsAttr.H5Pcreate: ", ocp_plist_id >= 0); + H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); + H5.H5Ocopy(H5fid, ".", H5fid, "CPYREF", ocp_plist_id, HDF5Constants.H5P_DEFAULT); + } + catch (Exception ex) { + fail("testH5OcopyRefsAttr: H5Ocopy failed"); + } + finally { + try {H5.H5Pclose(ocp_plist_id);} catch (Exception ex) {} + } + } + + @Test + public void testH5OcopyRefsDatasettodiffFile() { + byte rbuf1[] = null; + byte[] dset_data = new byte[16]; + long ocp_plist_id = HDF5Constants.H5I_INVALID_HID; + long dataset_id = HDF5Constants.H5I_INVALID_HID; + long H5fid2 = HDF5Constants.H5I_INVALID_HID; + + try { + rbuf1 = H5.H5Rcreate(H5fid, "DS2", HDF5Constants.H5R_OBJECT, -1); + System.arraycopy(rbuf1, 0, dset_data, 8, 8); + + dataset_id = H5.H5Dcreate(H5fid, "DSREF", + HDF5Constants.H5T_STD_REF_OBJ, H5dsid, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsDatasettodiffFile.H5Dcreate: ", dataset_id >= 0); + H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF_OBJ, + HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, dset_data); + H5.H5Dclose(dataset_id); + } + catch (Exception ex) { + fail("testH5OcopyRefsDatasettodiffFile: create dataset failed"); + } + finally { + try {H5.H5Dclose(dataset_id);} catch (Exception exx) {} + } + + try { + //create new file + H5fid2 = H5.H5Fcreate("copy.h5", HDF5Constants.H5F_ACC_TRUNC, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsDatasettodiffFile.H5Fcreate: ", H5fid2 >= 0); + H5.H5Fflush(H5fid2, HDF5Constants.H5F_SCOPE_LOCAL); + } + catch (Exception ex) { + try {H5.H5Fclose(H5fid2);} catch (Exception exx) {} + fail("testH5OcopyRefsDatasettodiffFile: H5Fcreate failed"); + } + + try { + //create object copy property list id and set the flags. + ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); + assertTrue("testH5OcopyRefsDatasettodiffFile.H5Pcreate: ", ocp_plist_id >= 0); + H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); + + //Perform copy function. + H5.H5Ocopy(H5fid, ".", H5fid2, "CPYREFD", ocp_plist_id, HDF5Constants.H5P_DEFAULT); + } + catch (Exception ex){ + ex.printStackTrace(); + fail("testH5OcopyRefsDatasettodiffFile: H5Ocopy failed"); + } + finally { + try {H5.H5Pclose(ocp_plist_id);} catch (Exception ex) {} + try {H5.H5Fclose(H5fid2);} catch (Exception ex) {} + } + _deleteFile("copy.h5"); + } + + @Test + public void testH5OcopyRefsDatasettosameFile() { + byte rbuf0[]=null , rbuf1[] = null; + byte[] dset_data = new byte[16]; + long ocp_plist_id = HDF5Constants.H5I_INVALID_HID; + long dataset_id = HDF5Constants.H5I_INVALID_HID; + long did = HDF5Constants.H5I_INVALID_HID; + int obj_type = -1; + byte[] read_data = new byte[16]; + + try { + rbuf0 = H5.H5Rcreate(H5fid, "/G1", HDF5Constants.H5R_OBJECT, -1); + rbuf1 = H5.H5Rcreate(H5fid, "DS2", HDF5Constants.H5R_OBJECT, -1); + System.arraycopy(rbuf0, 0, dset_data, 0, 8); + System.arraycopy(rbuf1, 0, dset_data, 8, 8); + + //Create a dataset and write object references to it. + dataset_id = H5.H5Dcreate(H5fid, "DSREF", + HDF5Constants.H5T_STD_REF_OBJ, H5dsid, + HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsDatasettosameFile.H5Dcreate: ", dataset_id >= 0); + H5.H5Dwrite(dataset_id, HDF5Constants.H5T_STD_REF_OBJ, + HDF5Constants.H5S_ALL, HDF5Constants.H5S_ALL, + HDF5Constants.H5P_DEFAULT, dset_data); + //Close the dataset. + H5.H5Dclose(dataset_id); + } + catch (Exception ex) { + try {H5.H5Dclose(dataset_id);} catch (Exception exx) {} + fail("testH5OcopyRefsDatasettosameFile: create dataset failed"); + } + + try { + ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); + assertTrue("testH5OcopyRefsDatasettosameFile.H5Pcreate: ", ocp_plist_id >= 0); + H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); + } + catch (Exception ex) { + try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} + fail("testH5OcopyRefsDatasettosameFile: H5Pset_copy_object failed"); + } + + //Perform copy function. + try { + H5.H5Ocopy(H5fid, "DSREF", H5fid, "CPYREFD", ocp_plist_id, HDF5Constants.H5P_DEFAULT); + } + catch(Exception ex) { + try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} + fail("testH5OcopyRefsDatasettosameFile: H5Ocopy failed"); + } + + //Open the dataset that has been copied + try { + did = H5.H5Dopen(H5fid, "DSREF", HDF5Constants.H5P_DEFAULT); + assertTrue("testH5OcopyRefsDatasettosameFile.H5Dopen: ", did >= 0); + } + catch (Exception e) { + try {H5.H5Dclose(did);} catch (Exception exx) {} + e.printStackTrace(); + fail("testH5OcopyRefsDatasettosameFile: H5Dopen failed"); + } + + try { + //Read the dataset object references in the read_data buffer. + H5.H5Dread(did, HDF5Constants.H5T_STD_REF_OBJ, HDF5Constants.H5S_ALL,HDF5Constants.H5S_ALL, HDF5Constants.H5P_DEFAULT, read_data); + System.arraycopy(read_data, 0, rbuf0, 0, 8); + System.arraycopy(read_data, 8, rbuf1, 0, 8); + + //Get the type of object the reference points to. + obj_type = H5.H5Rget_obj_type(H5fid, HDF5Constants.H5R_OBJECT, rbuf1); + assertEquals(obj_type, HDF5Constants.H5O_TYPE_DATASET); + + obj_type = H5.H5Rget_obj_type(H5fid, HDF5Constants.H5R_OBJECT, rbuf0); + assertEquals(obj_type, HDF5Constants.H5O_TYPE_GROUP); + } + catch (Exception ex) { + ex.printStackTrace(); + } + finally { + try {H5.H5Dclose(did);} catch (Exception ex) {} + try {H5.H5Pclose(ocp_plist_id);} catch (Exception ex) {} + } + } + + @Test + public void testH5OcopyNullRef() throws Throwable { + final long _pid_ = HDF5Constants.H5P_DEFAULT; + long sid = HDF5Constants.H5I_INVALID_HID; + long did = HDF5Constants.H5I_INVALID_HID; + long aid = HDF5Constants.H5I_INVALID_HID; + + try { + sid = H5.H5Screate_simple(1, new long[] {1}, null); + assertTrue("testH5OcopyNullRef.H5Screate_simple: ", sid >= 0); + did = H5.H5Dcreate(H5fid, "Dataset_with_null_Ref", HDF5Constants.H5T_NATIVE_INT, sid, _pid_, _pid_, _pid_); + assertTrue("testH5OcopyNullRef.H5Dcreate: ", did > 0); + aid = H5.H5Acreate(did, "Null_Ref", HDF5Constants.H5T_STD_REF_OBJ, sid, _pid_, _pid_); + assertTrue("testH5OcopyNullRef.H5Acreate: ", aid > 0); + } + catch (Exception ex) { + ex.printStackTrace(); + } + finally { + try {H5.H5Dclose(did);} catch (Exception exx) {} + try {H5.H5Aclose(aid);} catch (Exception exx) {} + try {H5.H5Sclose(sid);} catch (Exception exx) {} + } + + long ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); + assertTrue("testH5OcopyNullRef.H5Pcreate: ", ocp_plist_id >= 0); + H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); + try { + H5.H5Ocopy(H5fid, "/Dataset_with_null_Ref", H5fid, "/Dataset_with_null_Ref_cp", ocp_plist_id, _pid_); + } + finally { + try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} + } + } + +// @Ignore because of JIRA HDF5-9547 +// @Test(expected = HDF5LibraryException.class) +// public void testH5OcopyInvalidRef() throws Throwable { +// final long _pid_ = HDF5Constants.H5P_DEFAULT; +// long sid = HDF5Constants.H5I_INVALID_HID; +// long did = HDF5Constants.H5I_INVALID_HID; +// long aid = HDF5Constants.H5I_INVALID_HID; +// +// try { +// sid = H5.H5Screate_simple(1, new long[] {1}, null); +// assertTrue("testH5OcopyInvalidRef.H5Screate_simple: ", sid >= 0); +// did = H5.H5Dcreate(H5fid, "Dataset_with_invalid_Ref", HDF5Constants.H5T_NATIVE_INT, sid, _pid_, _pid_, _pid_); +// assertTrue("testH5OcopyInvalidRef.H5Dcreate: ", did > 0); +// aid = H5.H5Acreate(did, "Invalid_Ref", HDF5Constants.H5T_STD_REF_OBJ, sid, _pid_, _pid_); +// assertTrue("testH5OcopyInvalidRef.H5Acreate: ", aid > 0); +// H5.H5Awrite(aid, HDF5Constants.H5T_STD_REF_OBJ, new long[]{-1}); +// } +// catch (Exception ex) { +// ex.printStackTrace(); +// } +// finally { +// try {H5.H5Dclose(did);} catch (Exception exx) {} +// try {H5.H5Aclose(aid);} catch (Exception exx) {} +// try {H5.H5Sclose(sid);} catch (Exception exx) {} +// } +// +// long ocp_plist_id = H5.H5Pcreate(HDF5Constants.H5P_OBJECT_COPY); +// assertTrue("testH5OcopyInvalidRef.H5Pcreate: ", ocp_plist_id >= 0); +// H5.H5Pset_copy_object(ocp_plist_id, HDF5Constants.H5O_COPY_EXPAND_REFERENCE_FLAG); +// try { +// H5.H5Ocopy(H5fid, "/Dataset_with_invalid_Ref", H5fid, "/Dataset_with_invalid_Ref_cp", ocp_plist_id, _pid_); +// } +// finally { +// try {H5.H5Pclose(ocp_plist_id);} catch (Exception exx) {} +// } +// } + +} diff --git a/java/test/TestH5P.java b/java/test/TestH5P.java index 3879128..6ae93d9 100644 --- a/java/test/TestH5P.java +++ b/java/test/TestH5P.java @@ -1173,12 +1173,12 @@ public class TestH5P { strategy = H5.H5Pget_file_space_strategy(fcpl_id, persist, threshold); assertTrue("strategy(default): "+strategy, strategy == HDF5Constants.H5F_FSPACE_STRATEGY_FSM_AGGR); assertTrue("persist(default): "+persist[0], persist[0] == false); - assertTrue("theshold(default): "+threshold[0], threshold[0] == 1); + assertTrue("threshold(default): "+threshold[0], threshold[0] == 1); H5.H5Pset_file_space_strategy(fcpl_id, HDF5Constants.H5F_FSPACE_STRATEGY_PAGE, true, 1); strategy = H5.H5Pget_file_space_strategy(fcpl_id, persist, threshold); assertTrue("strategy: "+strategy, strategy == HDF5Constants.H5F_FSPACE_STRATEGY_PAGE); assertTrue("persist: "+persist[0], persist[0] == true); - assertTrue("theshold: "+threshold[0], threshold[0] == 1); + assertTrue("threshold: "+threshold[0], threshold[0] == 1); } catch (Throwable err) { err.printStackTrace(); diff --git a/java/test/TestH5Pfapl.java b/java/test/TestH5Pfapl.java index a65b52e..f6be24f 100644 --- a/java/test/TestH5Pfapl.java +++ b/java/test/TestH5Pfapl.java @@ -892,7 +892,7 @@ public class TestH5Pfapl { try { H5.H5Pset_fapl_multi(fapl_id, member_map, member_fapl, member_name, member_addr, true); long driver_type = H5.H5Pget_driver(fapl_id); - assertTrue("H5Pget_driver: muti = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); + assertTrue("H5Pget_driver: multi = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); } catch (Throwable err) { err.printStackTrace(); @@ -932,7 +932,7 @@ public class TestH5Pfapl { try { H5.H5Pset_fapl_multi(fapl_id, member_map, member_fapl, member_name, member_addr, true); long driver_type = H5.H5Pget_driver(fapl_id); - assertTrue("H5Pget_driver: muti = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); + assertTrue("H5Pget_driver: multi = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); } catch (Throwable err) { err.printStackTrace(); @@ -1001,7 +1001,7 @@ public class TestH5Pfapl { try { H5.H5Pset_fapl_multi(fapl_id, member_map, member_fapl, member_name, member_addr, true); long driver_type = H5.H5Pget_driver(fapl_id); - assertTrue("H5Pget_driver: muti = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); + assertTrue("H5Pget_driver: multi = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); } catch (Throwable err) { err.printStackTrace(); @@ -1223,7 +1223,7 @@ public class TestH5Pfapl { try { H5.H5Pset_fapl_multi(fapl_id, member_map, member_fapl, member_name, member_addr, true); long driver_type = H5.H5Pget_driver(fapl_id); - assertTrue("H5Pget_driver: muti = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); + assertTrue("H5Pget_driver: multi = "+ driver_type, HDF5Constants.H5FD_MULTI==driver_type); } catch (Throwable err) { err.printStackTrace(); diff --git a/java/test/TestH5Plist.java b/java/test/TestH5Plist.java index 0d53071..03e1c8f 100644 --- a/java/test/TestH5Plist.java +++ b/java/test/TestH5Plist.java @@ -148,7 +148,7 @@ public class TestH5Plist { } assertTrue("H5Pequal cid2", status >= 0); - // Make certain false postives aren't being returned + // Make certain false positives aren't being returned try { status = H5.H5Pequal(cid2, HDF5Constants.H5P_FILE_CREATE); } @@ -295,7 +295,7 @@ public class TestH5Plist { } assertTrue("H5Pget_nprops: "+nprops, nprops==0); - // Check the existance of the first property (should fail) + // Check the existence of the first property (should fail) try { status = H5.H5Pexist(plist_class_id, PROP1_NAME); } @@ -326,7 +326,7 @@ public class TestH5Plist { catch (Throwable err) { } - // Check the existance of the first property + // Check the existence of the first property try { status = H5.H5Pexist(plist_class_id, PROP1_NAME); } @@ -377,7 +377,7 @@ public class TestH5Plist { catch (Throwable err) { } - // Check the existance of the second property + // Check the existence of the second property try { status = H5.H5Pexist(plist_class_id, PROP2_NAME); } @@ -418,7 +418,7 @@ public class TestH5Plist { fail("H5Pregister2 plist_class_id: "+PROP3_NAME + err); } - // Check the existance of the third property + // Check the existence of the third property try { status = H5.H5Pexist(plist_class_id, PROP3_NAME); } diff --git a/java/test/junit.sh.in b/java/test/junit.sh.in index 39db296..ca2b3a9 100644 --- a/java/test/junit.sh.in +++ b/java/test/junit.sh.in @@ -66,8 +66,8 @@ $top_builddir/java/src/jni/.libs/libhdf5_java.* LIST_JAR_TESTFILES=" $HDFLIB_HOME/hamcrest-core.jar $HDFLIB_HOME/junit.jar -$HDFLIB_HOME/slf4j-api-1.7.25.jar -$HDFLIB_HOME/ext/slf4j-simple-1.7.25.jar +$HDFLIB_HOME/slf4j-api-1.7.33.jar +$HDFLIB_HOME/ext/slf4j-simple-1.7.33.jar " LIST_JAR_FILES=" $top_builddir/java/src/$JARFILE @@ -108,6 +108,7 @@ $HDFTEST_HOME/testfiles/JUnit-TestH5Arw.txt $HDFTEST_HOME/testfiles/JUnit-TestH5Oparams.txt $HDFTEST_HOME/testfiles/JUnit-TestH5Obasic.txt $HDFTEST_HOME/testfiles/JUnit-TestH5Ocreate.txt +$HDFTEST_HOME/testfiles/JUnit-TestH5OcopyOld.txt $HDFTEST_HOME/testfiles/JUnit-TestH5Ocopy.txt $HDFTEST_HOME/testfiles/JUnit-TestH5PL.txt $HDFTEST_HOME/testfiles/JUnit-TestH5VL.txt @@ -298,7 +299,7 @@ JAVAEXEFLAGS=@H5_JAVAFLAGS@ COPY_LIBFILES_TO_BLDLIBDIR COPY_DATAFILES_TO_BLDDIR -CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/junit.jar:"$BLDLIBDIR"/hamcrest-core.jar:"$BLDLIBDIR"/slf4j-api-1.7.25.jar:"$BLDLIBDIR"/slf4j-simple-1.7.25.jar:"$TESTJARFILE"" +CPATH=".:"$BLDLIBDIR"/"$JARFILE":"$BLDLIBDIR"/junit.jar:"$BLDLIBDIR"/hamcrest-core.jar:"$BLDLIBDIR"/slf4j-api-1.7.33.jar:"$BLDLIBDIR"/slf4j-simple-1.7.33.jar:"$TESTJARFILE"" TEST=/usr/bin/test if [ ! -x /usr/bin/test ] @@ -1030,6 +1031,27 @@ else test yes = "$verbose" && $DIFF JUnit-TestH5Ocreate.txt JUnit-TestH5Ocreate.out |sed 's/^/ /' fi +echo "$JAVAEXE $JAVAEXEFLAGS -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=trace -Djava.library.path=$BLDLIBDIR -cp $CLASSPATH -ea org.junit.runner.JUnitCore test.TestH5OcopyOld" +TESTING JUnit-TestH5OcopyOld +($RUNSERIAL $JAVAEXE $JAVAEXEFLAGS -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=trace -Djava.library.path=$BLDLIBDIR -cp $CLASSPATH -ea org.junit.runner.JUnitCore test.TestH5OcopyOld > JUnit-TestH5OcopyOld.ext) + +# Extract file name, line number, version and thread IDs because they may be different +sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \ + -e 's/line [0-9]*/line (number)/' \ + -e 's/Time: [0-9]*[\.,[0-9]*]*/Time: XXXX/' \ + -e 's/v[1-9]*\.[0-9]*\./version (number)\./' \ + -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ + JUnit-TestH5OcopyOld.ext > JUnit-TestH5OcopyOld.out + +if diff JUnit-TestH5OcopyOld.out JUnit-TestH5OcopyOld.txt > /dev/null; then + echo " PASSED JUnit-TestH5OcopyOld" +else + echo "**FAILED** JUnit-TestH5OcopyOld" + echo " Expected result differs from actual result" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF JUnit-TestH5OcopyOld.txt JUnit-TestH5OcopyOld.out |sed 's/^/ /' +fi + echo "$JAVAEXE $JAVAEXEFLAGS -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=trace -Djava.library.path=$BLDLIBDIR -cp $CLASSPATH -ea org.junit.runner.JUnitCore test.TestH5Ocopy" TESTING JUnit-TestH5Ocopy ($RUNSERIAL $JAVAEXE $JAVAEXEFLAGS -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=trace -Djava.library.path=$BLDLIBDIR -cp $CLASSPATH -ea org.junit.runner.JUnitCore test.TestH5Ocopy > JUnit-TestH5Ocopy.ext) diff --git a/java/test/testfiles/JUnit-TestH5Ocopy.txt b/java/test/testfiles/JUnit-TestH5Ocopy.txt index 32dfde7..7f66410 100644 --- a/java/test/testfiles/JUnit-TestH5Ocopy.txt +++ b/java/test/testfiles/JUnit-TestH5Ocopy.txt @@ -1,9 +1,10 @@ JUnit version 4.11 .testH5OcopyRefsDatasettosameFile +.testH5OcopyNullRef .testH5OcopyRefsDatasettodiffFile .testH5OcopyRefsAttr Time: XXXX -OK (3 tests) +OK (4 tests) diff --git a/java/test/testfiles/JUnit-TestH5OcopyOld.txt b/java/test/testfiles/JUnit-TestH5OcopyOld.txt new file mode 100644 index 0000000..7f66410 --- /dev/null +++ b/java/test/testfiles/JUnit-TestH5OcopyOld.txt @@ -0,0 +1,10 @@ +JUnit version 4.11 +.testH5OcopyRefsDatasettosameFile +.testH5OcopyNullRef +.testH5OcopyRefsDatasettodiffFile +.testH5OcopyRefsAttr + +Time: XXXX + +OK (4 tests) + diff --git a/m4/aclocal_fc.f90 b/m4/aclocal_fc.f90 index 240a768..bcefab5 100644 --- a/m4/aclocal_fc.f90 +++ b/m4/aclocal_fc.f90 @@ -120,7 +120,7 @@ PROGRAM FC_AVAIL_KINDS k = SELECTED_REAL_KIND(ik,jk) IF(k.LT.0) EXIT exp IF(k.NE.prev_rkind)THEN - ! Check if we aleady have that kind + ! Check if we already have that kind new_kind = .TRUE. DO kk = 1, num_rkinds IF(k.EQ.list_rkinds(kk))THEN diff --git a/m4/aclocal_fc.m4 b/m4/aclocal_fc.m4 index 49ff485..9d01f47 100644 --- a/m4/aclocal_fc.m4 +++ b/m4/aclocal_fc.m4 @@ -20,7 +20,7 @@ dnl Contains Macros for HDF5 Fortran dnl ********************************* dnl dnl Special characteristics that have no autoconf counterpart but that -dnl we need as part of the C++ support. To distinquish these, they +dnl we need as part of the C++ support. To distinguish these, they dnl have a [PAC] prefix. dnl dnl ------------------------------------------------------------------------- @@ -260,8 +260,8 @@ TEST_SRC="`sed -n '/PROGRAM FC_AVAIL_KINDS/,/END PROGRAM FC_AVAIL_KINDS/p' $srcd AC_RUN_IFELSE([$TEST_SRC], [ dnl The output from the above program will be: - dnl -- LINE 1 -- valid integer kinds (comma seperated list) - dnl -- LINE 2 -- valid real kinds (comma seperated list) + dnl -- LINE 1 -- valid integer kinds (comma separated list) + dnl -- LINE 2 -- valid real kinds (comma separated list) dnl -- LINE 3 -- max decimal precision for reals dnl -- LINE 4 -- number of valid integer kinds dnl -- LINE 5 -- number of valid real kinds diff --git a/release_docs/HISTORY-1_0-1_8_0_rc3.txt b/release_docs/HISTORY-1_0-1_8_0_rc3.txt index f23a2c5..f54ba64 100644 --- a/release_docs/HISTORY-1_0-1_8_0_rc3.txt +++ b/release_docs/HISTORY-1_0-1_8_0_rc3.txt @@ -103,9 +103,9 @@ New Features -JML 2006/6/21 - If both shared and static libraries are installed, now both will be tested during 'make install'. -MAM 2006/06/21 - - Added support to explicity enable stream_vfd or shared libraries + - Added support to explicitly enable stream_vfd or shared libraries when using parallel via the '--enable-stream_vfd' and - '--enable-shared' options, respectively. If not explicity defined, + '--enable-shared' options, respectively. If not explicitly defined, These settings default to enabled when parallel is not used, and disabled when parallel is used. -MAM 2006/06/17 - Remove the flexible parallel code and the --enable-fphdf5 @@ -169,7 +169,7 @@ New Features variable $FC, not $F9X. F9X still works, but is depreciated. The output of make may be different. This should be only a cosmetic effect. - make depened (or make dep) is no longer recognized, since automake + make depend (or make dep) is no longer recognized, since automake handles dependency tracking. Some new configure options exist. --enable-dependency-tracking and --disable-dependency-tracking are used to control automake's @@ -306,11 +306,11 @@ New Features - Added managements of collective IO supports for chunking storage inside parallel HDF5 1) Implemented One IO with collective mode for all chunks in the - application by building one MPI derived datatype accross all + application by building one MPI derived datatype across all chunks. 2) Implemented the decision-making support to do collective IO inside MPI-IO per chunk. - 3) Added the decision-making support to do one IO accross all chunks + 3) Added the decision-making support to do one IO across all chunks or to do multiple IOs with each IO per chunk. 4) Added the support to handle the case some processes won't do any IOs in collectively. @@ -319,7 +319,7 @@ New Features contributions to IOs, a special macro is added to change collective IO mode to independent IO mode inside HDF5 library. - Currently we find that MPICH at Linux and vender MPI-IO package at NCSA + Currently we find that MPICH at Linux and vendor MPI-IO package at NCSA Altix cannot handle this case. "hdf5_mpi_special_collective_io_works=${hdf5_mpi_special_collective_io_works='no'}" @@ -469,7 +469,7 @@ New Features object's object header. QAK 2003/10/06 - Added new H5Fget_freespace() routine to query the free space in a given file. QAK 2003/10/06 - - Added backward compatability with v1.6 for new Error API. SLU - + - Added backward compatibility with v1.6 for new Error API. SLU - 2003/09/24 - Changed 'objno' field in H5G_stat_t structure from 'unsigned long[2]' to 'haddr_t'. QAK - 2003/08/08 @@ -496,7 +496,7 @@ New Features - Add options of using atomicity and file-sync to test_mpio_1wMr. AKC - 2003/11/13 - Added parallel test, test_mpio_1wMr, which tests if the - underlaying parallel I/O system is conforming to the POSIX + underlying parallel I/O system is conforming to the POSIX write/read requirement. AKC - 2003/11/12 Fortran Library: @@ -606,7 +606,7 @@ New Features and H5LTdtype_to_text. H5LTtext_to_dtype creates a HDF5 data type given a text description; H5LTdtype_to_text converts a data type to text description. Only DDL definition is supported as text - desciption now. SLU - 2006/05/17 + description now. SLU - 2006/05/17 - Added Packet Table API for creating tables with less overhead than H5TB API. Added C++ wrapper for Packet Tables. See documentation. JML - 2004/03/28 @@ -639,7 +639,7 @@ Support for new platforms, languages and compilers. - Added support for HPUX11.23 for both 32 and 64-bit; HDF5 C++ - Added support for 64-bit Windows with Visual Studio .NET and 2005. SJW - 2007/06/25 - - Added suport for HPUX11.23 for both 32 and 64-bit; HDF5 C++ + - Added support for HPUX11.23 for both 32 and 64-bit; HDF5 C++ shared library is not supported with +DD64 flag. EIP - 2006/06/22 - Added support for VAX floating numbers for Alpha Open VMS 7.3.2 @@ -655,7 +655,7 @@ Support for new platforms, languages and compilers. - HDF5 Fortran APIs are supported on Mac OSX with IBM XL Fortran compiler version 8.1. This is a default compiler. - HDF5 Fortran APIs are supported on MAC OSX with Absoft F95 compiler - version 8.2; set F9X environment varibale to f95, for example + version 8.2; set F9X environment variable to f95, for example setenv F9X f95 Use --disable-shared --enable-static configure flags when Absoft compiler is used. @@ -700,7 +700,7 @@ Bug Fixes since HDF5-1.6.0 release - Re-open the file - Delete one of the attributes on the object - Add a smaller attribute to the object - - Delete the smaller atttribute on the object + - Delete the smaller attribute on the object - Add a larger attribute on the object After this, the number of header messages stored for the object @@ -855,7 +855,7 @@ Bug Fixes since HDF5-1.6.0 release - Fixed dtypes "sw long double -> double" failure in QSC class machines. AKC - 2004/4/16 - Fixed problem with fletcher32 filter when converting data of different - endianess. PVN - 2004/03/10 + endianness. PVN - 2004/03/10 - Fixed problem with H5Tget_native_type() not handling opaque fields correctly. QAK - 2004/01/31 - Fixed several errors in B-tree deletion code which could cause a @@ -1271,7 +1271,7 @@ Known Problems sometimes. This can be verified with t_mpi test under testpar. * On IRIX6.5, when C compiler version >7.4, the complicate MPI derived data type code will work. However, the user should be aware to enlarge MPI_TYPE_MAX environment - variable to some certian value in order to use collective irregular selection code. + variable to some certain value in order to use collective irregular selection code. For example, the current parallel HDF5 test needs to enlarge MPI_TYPE_MAX to 200,000 to make the test pass. * The dataset created or rewritten with the v1.6.3 library or after can't @@ -1428,7 +1428,7 @@ Known Problems exit $RETURN_VALUE - You get the HDF make files and test scipts to execute your filter script + You get the HDF make files and test scripts to execute your filter script by setting the environment variable "RUNSERIAL" to the full path of the script prior to running configure for parallel builds. Remember to "unsetenv RUNSERIAL" before running configure for a serial build. @@ -2098,7 +2098,7 @@ Bug Fixes since HDF5-1.6.5 Release - Re-open the file. - Delete one of the attributes on the object. - Add a smaller attribute to the object. - - Delete the smaller atttribute on the object. + - Delete the smaller attribute on the object. - Add a larger attribute on the object. After this, the number of header messages stored for the object @@ -2399,11 +2399,11 @@ Known Problems For 32-bit: You may do this, under csh: - setenv LD_LIBRARYN32_PATH .......(existing pathes):[full path of HDF5 + setenv LD_LIBRARYN32_PATH .......(existing paths):[full path of HDF5 directory/test/.libs]:[full path of HDF5 directory/src/.libs] For 64-bit: - setenv LD_LIBRARY64_PATH ......(existing pathes):[full path of HDF5 + setenv LD_LIBRARY64_PATH ......(existing paths):[full path of HDF5 directory/test/.libs]:[full path of HDF5 directory/src/.libs] NOTE: This problem ONLY affects the testing of the HDF5 library when you @@ -2723,7 +2723,7 @@ Bug Fixes since HDF5-1.6.4 Release ------- - Fixed collective IO in chunking-storage. HDF5 may have called the wrong routine when the shape of the dataspace in the file and in - the bufferred chunk were different. This bug was fixed to make sure + the buffered chunk were different. This bug was fixed to make sure the correct routine is called. KY - 2005/10/19 - Fixed core dump when closing root groups opened through two different file handles that operate on the same actual file. QAK - 2005/10/02 @@ -2744,7 +2744,7 @@ Bug Fixes since HDF5-1.6.4 Release together all have the same "file close degree". QAK - 2005/07/19 - More bug fixes on holding open files that are mounted and have IDs open. QAK - 2005/07/14 - - Dataset sieve buffer cache was inadvertantly disabled; it has been + - Dataset sieve buffer cache was inadvertently disabled; it has been re-enabled. QAK - 2005/07/08 - Don't unmount child files until the parent file actually closes. (Previously, if an object was holding open a file, the child files @@ -3024,7 +3024,7 @@ Known Problems to intel forum. * When testing parallel HDF5 with the C compiler version MIPSpro 7.4.3 at IRIX - 6.5, set enviroment variable MPI_TYPE_MAX to be a bigger number, for example + 6.5, set environment variable MPI_TYPE_MAX to be a bigger number, for example 120000, in order to pass the complicated collective IO tests inside parallel HDF5 library. This is not a problem inside parallel HDF5 library. You can always set a bigger number in your system. @@ -4448,7 +4448,7 @@ New Features verbosity via the '-v' option. The default is less verbose than before. AKC - 2004/01/23 - Added parallel test, test_mpio_1wMr, which tests if the - underlaying parallel I/O system is conforming to the POSIX + underlying parallel I/O system is conforming to the POSIX write/read requirement. This version includes options of using atomicity and file-sync. AKC - 2003/11/27 @@ -4821,7 +4821,7 @@ Known Problems Testing random sw long double -> double conversions If -ieee is used, the converted doubles spread over the range 0.0 to 1**-308. If -ieee is not used, the converted double values are mostly 0.0 but - occassionaly as 1**-308. This has been reported to the system staff. + occasionally as 1**-308. This has been reported to the system staff. All other tests have passed. * On AIX 5.1 when 64-bit parallel C Library is built with zlib configured in, @@ -5187,7 +5187,7 @@ Linux 2.4 IA64 Intel y n y n y y y Known Problems ============== * Fortran subroutine h5pget_driver_f doesn't return a correct driver information. - The fix willl be available in the 1.6.2 release. + The fix will be available in the 1.6.2 release. * There are two h5dump xml tests(h5dump --xml thlink.h5 and h5dump --xml tmany.h5) failed on windows xp with .NET for debug and debug dll. Release and Release dll work fine. @@ -5368,7 +5368,7 @@ Library: * The first version of szip compression support were implemented. User should have static szlib library installed. Using function H5Pset_szip to pass the szip parameters to the HDF5 library. - More detailed decription of the process will be followed. + More detailed description of the process will be followed. KY-2003/04/01 * Added Fletcher32 checksum as a filter in pipeline. It only works in chunked dataset. SLU - 2003/2/11 @@ -5416,15 +5416,15 @@ Library: be in different locations in the same file. The old functions H5Glink and H5Gmove remain valid. SLU - 2002/04/26 * Fill-value's behaviors for contiguous dataset have been redefined. - Basicly, dataset won't allocate space until it's necessary. Full details + Basically, dataset won't allocate space until it's necessary. Full details are available at http://hdf.ncsa.uiuc.edu/RFC/Fill_Value, at this moment. SLU - 2002/04/11 * Added new routine "H5Dfill" to fill a selection with a particular value in memory. QAK - 2002/04/09 * Improved performance of "regular" hyperslab I/O when using MPI-IO and the - datatype conversion is unneccessary. QAK - 2002/04/02 + datatype conversion is unnecessary. QAK - 2002/04/02 * Improved performance of single hyperslab I/O when datatype conversion is - unneccessary. QAK - 2002/04/02 + unnecessary. QAK - 2002/04/02 * Added new "H5Sget_select_type" API function to determine which type of selection is defined for a dataspace ("all", "none", "hyperslab" or "point"). QAK - 2002/02/07 @@ -5438,7 +5438,7 @@ Library: compatibility is enabled. * Added a new file access property, file close degree, to control file close behavior. It has four values, H5F_CLOSE_WEAK, H5F_CLOSE_SEMI, - H5F_CLOSE_STRONG, and H5F_CLOSE_DEFAULT. Two correspont functions + H5F_CLOSE_STRONG, and H5F_CLOSE_DEFAULT. Two correspond functions H5Pset_fclose_degree and H5Pget_fclose_degree are also provided. Two new functions H5Fget_obj_count and H5Fget_obj_ids are offerted to assist this new feature. For full details, please refer to the reference @@ -5458,7 +5458,7 @@ Library: * Improved regular hyperslab I/O by about a factor of 6 or so. * Modified the Pablo build procedure to permit building of the instrumented library to link either with the Trace libraries as before or with the - Pablo Performance Caputure Facility. + Pablo Performance Capture Facility. * Added new F90 APIs for generic properties, new filters, and time/space allocation properties. * C++ API: @@ -5536,7 +5536,7 @@ Library * Fixed data corruption problem which could occur when fill values were written to a contiguously stored dataset in parallel. QAK - 2002/08/27 * Fixed VL memory leak when data is overwritten. The heap objects holding - old data are freed. If the fill value writting time is set to + old data are freed. If the fill value writing time is set to H5D_FILL_TIME_NEVER, the library prohibits user to create VL type dataset. The library free all the heap objects storing VL type if there is nested VL type(a VL type contains another VL type). SLU - 2002/07/10 @@ -6157,7 +6157,7 @@ Known Problems * IRIX 6.5 fails to compile if configured with --enable-static-exec. - * The executables are always dynamic on Solaris 2.7 ans 2.8(64 and n32) + * The executables are always dynamic on Solaris 2.7 and 2.8(64 and n32) even if they are configured with --enable-static-exec. * The HDF5_MPI_OPT_TYPES optimization code in the parallel HDF5 will cause @@ -6175,7 +6175,7 @@ Known Problems compiler optimization causes errors in many HDF5 Library tests. Use -g -h zero flags to build HDF5 Library. - * On Cray SV1 10.0.1. 0 datatype convertion test fails. Please check HDF FTP site + * On Cray SV1 10.0.1. 0 datatype conversion test fails. Please check HDF FTP site if patch is available. We will try to provide one in the nearest future. * For configuration, building and testing with Intel and PGI compilers see @@ -6932,7 +6932,7 @@ Known Problems HDF5_MPI_OPT_TYPES to a non-zero value such as 1. * On IA64 systems one has to use -DIA64 compilation flag to compile - h4toh5 and h5toh4 utilites. After configuration step manually modify + h4toh5 and h5toh4 utilities. After configuration step manually modify Makefile in the tools/h4toh4 and tools/h5toh4 directories to add -DIA64 to the compilation flags. @@ -6998,7 +6998,7 @@ New Features and other version information in the source code. * Parallel HDF5 now runs on the HP V2500 and HP N4000 machines. * F90 API: - - Added aditional parameter "dims" to the h5dread_f/h5dwrite_f and + - Added additional parameter "dims" to the h5dread_f/h5dwrite_f and h5aread_f/h5awrite_f subroutines. This parameter is a 1-D array of size 7 and contains the sizes of the data buffer dimensions. This change enables portability between Windows and UNIX platforms. @@ -7012,8 +7012,8 @@ New Features * F90 static library is available on Windows platforms. See INSTALL_Windows_withF90.txt for details. * F90 APIs are available on HPUX 11.00 and 10.20 and IBM SP platforms. - * H5 <-> GIF convertor has been added. This is available under - tools/gifconv. The convertor supports the ability to create animated + * H5 <-> GIF converter has been added. This is available under + tools/gifconv. The converter supports the ability to create animated gifs as well. * Verified correct operation of library on Solaris 2.8 in both 64-bit and 32-bit compilation modes. See INSTALL document for instructions on @@ -7064,7 +7064,7 @@ Bug Fixes since HDF5-1.4.1 Release when --enable-hdf5v1_2 configure flag is enabled. * Added --enable-linux-lfs flag to allow more control over whether to enable or disable large file support on Linux. - * Fixed various bugs releated to SDS dimensional scale conversions in H4->H5 + * Fixed various bugs related to SDS dimensional scale conversions in H4->H5 converter. * Fixed a bug to correctly convert HDF4 objects with fill value into HDF5. * Fixed a bug of H5pubconf.h causing repeated definitions if it is included @@ -7345,7 +7345,7 @@ Bug Fixes since HDF5-1.4.0 Release causing errors during I/O * The RCSID string in H5public.h was causing the C++ compiling problem because when it was included multiple times, C++ did not like - multiple definitions of the same static variable. All occurance of + multiple definitions of the same static variable. All occurrence of RCSID definition are removed since we have not used it consistently before. @@ -8111,7 +8111,7 @@ Known Problems o SunOS 5.6 with C WorkShop Compilers 4.2: hyperslab selections will fail if library is compiled using optimization of any level. o TFLOPS: dsets test fails if compiled with optimization turned on. -o J90: tools fail to dispay data for the datasets with a compound datatype. +o J90: tools fail to display data for the datasets with a compound datatype. Platforms Tested ================ @@ -8379,7 +8379,7 @@ C. Changes Since Version 1.0.1 * New query functions for selections. * I/O operations bypass the stripmining loop and go directly to - storage for certain contiguous selections in the absense of type + storage for certain contiguous selections in the absence of type conversions. In other cases the stripmining buffers are used more effectively. diff --git a/release_docs/HISTORY-1_10.txt b/release_docs/HISTORY-1_10.txt index 127b821..4796ebb 100644 --- a/release_docs/HISTORY-1_10.txt +++ b/release_docs/HISTORY-1_10.txt @@ -161,7 +161,7 @@ New Features YES NO Default to STATIC NO YES Default to SHARED NO NO Default to SHARED - The defaults can be overriden by setting the config option + The defaults can be overridden by setting the config option HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC (ADB - 2020/07/09, HDFFV-11116) @@ -610,7 +610,7 @@ Bug Fixes since HDF5-1.10.6 release - h5diff fixed a command line parsing error. h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. - The macro H5_DBL_ABS_EQUAL was removed and a direct value comparision was used. + The macro H5_DBL_ABS_EQUAL was removed and a direct value comparison was used. (ADB - 2020/07/20, HDFFV-10897) @@ -618,7 +618,7 @@ Bug Fixes since HDF5-1.10.6 release h5diff would ignore all objects with a supplied path if the exclude-path argument is used. Adding the exclude-attribute argument will only eclude attributes, with the supplied path, - from comparision. + from comparison. (ADB - 2020/07/20, HDFFV-5935) @@ -1434,7 +1434,7 @@ Known Problems and thus does not suffer from the 32 bit size limitation described here. OpenMPI releases prior to v3.1.3 appear to have other datatype issues however, e.g. within a single parallel test (testphdf5) the subtests (cdsetr, eidsetr) - report data verfication errors before eventually aborting. + report data verification errors before eventually aborting. The most recent versions of OpenMPI (v3.1.3 or newer) have evidently resolved these isses and parallel HDF5 testing does not currently report errors though occasional hangs have been observed. @@ -1636,7 +1636,7 @@ New Features (DER - 2019/02/03, HDFFV-10696) - - Add ability to minimze dataset object headers. + - Add ability to minimize dataset object headers. Creation of many, very small datasets resulted in extensive file bloat due to extra space in the dataset object headers -- this space is @@ -2137,7 +2137,7 @@ Known Problems and thus does not suffer from the 32 bit size limitation described here. OpenMPI releases prior to v3.1.3 appear to have other datatype issues however, e.g. within a single parallel test (testphdf5) the subtests (cdsetr, eidsetr) - report data verfication errors before eventually aborting. + report data verification errors before eventually aborting. The most recent versions of OpenMPI (v3.1.3 or newer) have evidently resolved these isses and parallel HDF5 testing does not currently report errors though occasional hangs have been observed. @@ -2865,7 +2865,7 @@ Bug Fixes since HDF5-1.10.2 release When the function is called with the 'allgather' parameter set to a non-true value, the function will receive data from all MPI - ranks and gather it to the single rank specied by the 'root' + ranks and gather it to the single rank specified by the 'root' parameter. However, the bug in the function caused memory for the received data to be allocated on all MPI ranks, not just the singular rank specified as the receiver. In some circumstances, @@ -3431,11 +3431,11 @@ New Features file driver in HDF5 by previously having called H5Pset_fapl_mpio(). HDF5 parallel file operations which do not employ multiple ranks - e.g. specifiying MPI_COMM_SELF (whose MPI_Comm_size == 1) + e.g. specifying MPI_COMM_SELF (whose MPI_Comm_size == 1) as opposed to MPI_COMM_WORLD, will not be affected by this optimization. Conversely, parallel file operations on subgroups of MPI_COMM_WORLD are allowed to be run in parallel with each - subgroup operating as an independant collection of processes. + subgroup operating as an independent collection of processes. (RAW - 2017/10/10, HDFFV-10294) @@ -3574,7 +3574,7 @@ New Features (ADB - 2017/12/20) - The H5I_REFERENCE value in the H5I_type_t enum (defined in H5Ipublic.h) - has been marked as deprectated. + has been marked as deprecated. JNI code which refers to this value will be removed in a future major version of the library. The code will remain unchanged in the @@ -3895,7 +3895,7 @@ Bug Fixes since HDF5-1.10.1 release Use of this VFD feature flag was not in line with the documentation in the public H5FDpublic.h file. In particular, it was being used as a - proxy for determining if SWMR I/O is allowed. This is unecessary as we + proxy for determining if SWMR I/O is allowed. This is unnecessary as we already have a feature flag for this (H5FD_SUPPORTS_SWMR_IO). (DER - 2017/05/31, HDFFV-10214) @@ -4796,7 +4796,7 @@ Bug Fixes since HDF5-1.10.0-patch1 release files to ensure that the library fails gracefully in these cases: * Writing to a corrupted file that has an object message which is - incorrectly marked as sharable on disk results in a buffer overflow / + incorrectly marked as shareable on disk results in a buffer overflow / invalid write instead of a clean error message. * Decoding data from a corrupted file with a dataset encoded with the @@ -5258,7 +5258,7 @@ This release supports the following features: identically to the 1.8 library with the --with-default-api-version=v18 configure flag. This allows existing code to be compiled with the 1.10 library without requiring immediate changes to the application source - code. For addtional configuration options and other details, see + code. For additional configuration options and other details, see "API Compatibility Macros in HDF5" at https://www.hdfgroup.org/HDF5/doc/RM/APICompatMacros.html. @@ -5342,7 +5342,7 @@ This release supports the following features: Library -------- - - Concurrent Access to HDF5 Files - Single Writer/ Multple Reader (SWMR) + - Concurrent Access to HDF5 Files - Single Writer/ Multiple Reader (SWMR) The Single Writer/ Multiple Reader or SWMR feature enables users to read data concurrently while writing it. Communications between the @@ -5970,7 +5970,7 @@ This release supports the following features: identically to the 1.8 library with the --with-default-api-version=v18 configure flag. This allows existing code to be compiled with the 1.10 library without requiring immediate changes to the application source - code. For addtional configuration options and other details, see + code. For additional configuration options and other details, see "API Compatibility Macros in HDF5" at https://www.hdfgroup.org/HDF5/doc/RM/APICompatMacros.html. @@ -6054,7 +6054,7 @@ This release supports the following features: Library -------- - - Concurrent Access to HDF5 Files - Single Writer/ Multple Reader (SWMR) + - Concurrent Access to HDF5 Files - Single Writer/ Multiple Reader (SWMR) The Single Writer/ Multiple Reader or SWMR feature enables users to read data concurrently while writing it. Communications between the diff --git a/release_docs/HISTORY-1_10_0-1_12_0.txt b/release_docs/HISTORY-1_10_0-1_12_0.txt index 60516fc..a83e58d 100644 --- a/release_docs/HISTORY-1_10_0-1_12_0.txt +++ b/release_docs/HISTORY-1_10_0-1_12_0.txt @@ -810,7 +810,7 @@ Bug Fixes since HDF5-1.10.2 release When the function is called with the 'allgather' parameter set to a non-true value, the function will receive data from all MPI - ranks and gather it to the single rank specied by the 'root' + ranks and gather it to the single rank specified by the 'root' parameter. However, the bug in the function caused memory for the received data to be allocated on all MPI ranks, not just the singular rank specified as the receiver. In some circumstances, diff --git a/release_docs/HISTORY-1_13.txt b/release_docs/HISTORY-1_13.txt new file mode 100644 index 0000000..06be75b --- /dev/null +++ b/release_docs/HISTORY-1_13.txt @@ -0,0 +1,1758 @@ +HDF5 History +============ + +This file contains development history of the HDF5 1.13 releases from +the develop branch + +01. Release Information for hdf5-1.13.0 + +[Search on the string '%%%%' for section breaks of each release.] + +%%%%1.13.0%%%% + +HDF5 version 1.13.0 released on 2021-12-01 +================================================================================ + + +INTRODUCTION +============ + +This document describes the differences between this release and the previous +HDF5 release. It contains information on the platforms tested and known +problems in this release. For more details check the HISTORY*.txt files in the +HDF5 source. + +Note that documentation in the links below will be updated at the time of each +final release. + +Links to HDF5 documentation can be found on The HDF5 web page: + + https://portal.hdfgroup.org/display/HDF5/HDF5 + +The official HDF5 releases can be obtained from: + + https://www.hdfgroup.org/downloads/hdf5/ + +Changes from Release to Release and New Features in the HDF5-1.13.x release series +can be found at: + + https://portal.hdfgroup.org/display/HDF5/HDF5+Application+Developer%27s+Guide + +If you have any questions or comments, please send them to the HDF Help Desk: + + help@hdfgroup.org + + +CONTENTS +======== + +- New Features +- New platforms, languages and compilers tested +- Bug Fixes since HDF5-1.12.0 +- Platforms Tested +- Known Problems +- CMake vs. Autotools installations + + +New Features +============ + + Configuration: + ------------- + - Added new configure option to support building parallel tools. + See Tools below (autotools - CMake): + --enable-parallel-tools HDF5_BUILD_PARALLEL_TOOLS + + (RAW - 2021/10/25) + + - Added new configure options to enable dimension scales APIs (H5DS*) to + use new object references with the native VOL connector (aka native HDF5 + library). New references are always used for non-native terminal VOL + connectors (e.g., DAOS). + + Autotools --enable-dimension-scales-with-new-ref + CMake HDF5_DIMENSION_SCALES_NEW_REF=ON + + (EIP - 2021/10/25, HDFFV-11180) + + - Refactored the utils folder. + + Added subfolder test and moved the 'swmr_check_compat_vfd.c file' + from test into utils/test. Deleted the duplicate swmr_check_compat_vfd.c + file in hl/tools/h5watch folder. Also fixed vfd check options. + + (ADB - 2021/10/18) + + - Changed autotools and CMake configurations to derive both + compilation warnings-as-errors and warnings-only-warn configurations + from the same files, 'config/*/*error*'. Removed redundant files + 'config/*/*noerror*'. + + (DCY - 2021/09/29) + + - Added new option to control the build of High-Level tools + that default ON/enabled. + + Add configure options (autotools - CMake): + --enable-hltools HDF5_BUILD_HL_TOOLS + + Disabling this option prevents building the gif tool which + contains the following CVEs: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 + + (ADB - 2021/09/16, HDFFV-11266) + + - Adds C++ Autotools configuration file for Intel + + * Checks for icpc as the compiler + * Sets std=c++11 + * Copies most non-warning flags from intel-flags + + (DER - 2021/06/02) + + - Adds C++ Autotools configuration file for PGI + + * Checks for pgc++ as the compiler name (was: pgCC) + * Sets -std=c++11 + * Other options basically match new C options (below) + + (DER - 2021/06/02) + + - Updates PGI C options + + * -Minform set to warn (was: inform) to suppress spurious messages + * Sets -gopt -O2 as debug options + * Sets -O4 as 'high optimization' option + * Sets -O0 as 'no optimization' option + * Removes specific settings for PGI 9 and 10 + + (DER - 2021/06/02) + + - A C++11-compliant compiler is now required to build the C++ wrappers + + CMAKE_CXX_STANDARD is now set to 11 when building with CMake and + -std=c++11 is added when building with clang/gcc via the Autotools. + + (DER - 2021/05/27) + + - CMake will now run the shell script tests in test/ by default + + The test directory includes several shell script tests that previously + were not run by CMake. These are now run by default. TEST_SHELL_SCRIPTS + has been set to ON and SH_PROGRAM has been set to bash (some test + scripts use bash-isms). Platforms without bash (e.g., Windows) will + ignore the script tests. + + (DER - 2021/05/23) + + - Removed unused HDF5_ENABLE_HSIZET option from CMake + + This has been unused for some time and has no effect. + + (DER - 2021/05/23) + + - CMake no longer builds the C++ library by default + + HDF5_BUILD_CPP_LIB now defaults to OFF, which is in line with the + Autotools build defaults. + + (DER - 2021/04/20) + + - Removal of pre-VS2015 work-arounds + + HDF5 now requires Visual Studio 2015 or greater, so old work-around + code and definitions have been removed, including: + + * + * snprintf and vsnprintf + * llround, llroundf, lround, lroundf, round, roundf + * strtoll and strtoull + * va_copy + * struct timespec + + (DER - 2021/03/22) + + - Add CMake variable HDF5_LIB_INFIX + + This infix is added to all library names after 'hdf5'. + e.g. the infix '_openmpi' results in the library name 'libhdf5_openmpi.so' + This name is used in packages on debian based systems. + (see https://packages.debian.org/jessie/amd64/libhdf5-openmpi-8/filelist) + + (barcode - 2021/03/22) + + - On macOS, Universal Binaries can now be built, allowing native execution on + both Intel and Apple Silicon (ARM) based Macs. + + To do so, set CMAKE_OSX_ARCHITECTURES="x86_64;arm64" + + (SAM - 2021/02/07, github-311) + + - Added a configure-time option to control certain compiler warnings + diagnostics + + A new configure-time option was added that allows some compiler warnings + diagnostics to have the default operation. This is mainly intended for + library developers and currently only works for gcc 10 and above. The + diagnostics flags apply to C, C++ and Fortran compilers and will appear + in "H5 C Flags", H5 C++ Flags" and H5 Fortran Flags, respectively. They + will NOT be exported to h5cc, etc. + + The default is OFF, which will disable the warnings URL and color attributes + for the warnings output. ON will not add the flags and allow default behavior. + + Autotools: --enable-diags + + CMake: HDF5_ENABLE_BUILD_DIAGS + + (ADB - 2021/02/05, HDFFV-11213) + + - CMake option to build the HDF filter plugins project as an external project + + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. + + The option was refactored to use the CMake FetchContent process. This allows + more control over the filter targets, but required external project command + options to be moved to a CMake include file, HDF5PluginCache.cmake. Also + enabled the filter examples to be used as tests for operation of the + filter plugins. + + (ADB - 2020/12/10, OESS-98) + + - FreeBSD Autotools configuration now defaults to 'cc' and 'c++' compilers + + On FreeBSD, the autotools defaulted to 'gcc' as the C compiler and did + not process C++ options. Since FreeBSD 10, the default compiler has + been clang (via 'cc'). + + The default compilers have been set to 'cc' for C and 'c++' for C++, + which will pick up clang and clang++ respectively on FreeBSD 10+. + Additionally, clang options are now set correctly for both C and C++ + and g++ options will now be set if that compiler is being used (an + omission from the former functionality). + + (DER - 2020/11/28, HDFFV-11193) + + - Fixed POSIX problems when building w/ gcc on Solaris + + When building on Solaris using gcc, the POSIX symbols were not + being set correctly, which could lead to issues like clock_gettime() + not being found. + + The standard is now set to gnu99 when building with gcc on Solaris, + which allows POSIX things to be #defined and linked correctly. This + differs slightly from the gcc norm, where we set the standard to c99 + and manually set POSIX #define symbols. + + (DER - 2020/11/25, HDFFV-11191) + + - Added a configure-time option to consider certain compiler warnings + as errors + + A new configure-time option was added that converts some compiler warnings + to errors. This is mainly intended for library developers and currently + only works for gcc and clang. The warnings that are considered errors + will appear in the generated libhdf5.settings file. These warnings apply + to C and C++ code and will appear in "H5 C Flags" and H5 C++ Flags", + respectively. They will NOT be exported to h5cc, etc. + + The default is OFF. Building with this option may fail when compiling + on operating systems and with compiler versions not commonly used by + the library developers. Compilation may also fail when headers not + under the control of the library developers (e.g., mpi.h, hdfs.h) raise + warnings. + + Autotools: --enable-warnings-as-errors + + CMake: HDF5_ENABLE_WARNINGS_AS_ERRORS + + (DER - 2020/11/23, HDFFV-11189) + + - Autotools and CMake target added to produce doxygen generated documentation + + The default is OFF or disabled. + Autoconf option is '--enable-doxygen' + autotools make target is 'doxygen' and will build all doxygen targets + CMake configure option is 'HDF5_BUILD_DOC'. + CMake target is 'doxygen' for all available doxygen targets + CMake target is 'hdf5lib_doc' for the src subdirectory + + (ADB - 2020/11/03) + + - CMake option to use MSVC naming conventions with MinGW + + HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions + when using a MinGW toolchain + + (xan - 2020/10/30) + + - CMake option to statically link gcc libs with MinGW + + HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ + with the MinGW toolchain + + (xan - 2020/10/30) + + - CMake option to build the HDF filter plugins project as an external project + + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. + + The necessary options are (see the INSTALL_CMake.txt file): + HDF5_ENABLE_PLUGIN_SUPPORT + PLUGIN_TGZ_NAME or PLUGIN_GIT_URL + There are more options necessary for various filters and the plugin project + documents should be referenced. + + (ADB - 2020/09/27, OESS-98) + + - Added CMake option to format source files + + HDF5_ENABLE_FORMATTERS option will enable creation of targets using the + pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder + or tool folder. All sources can be formatted by executing the format target; + make format + + (ADB - 2020/08/24) + + - Add file locking configure and CMake options + + HDF5 1.10.0 introduced a file locking scheme, primarily to help + enforce SWMR setup. Formerly, the only user-level control of the scheme + was via the HDF5_USE_FILE_LOCKING environment variable. + + This change introduces configure-time options that control whether + or not file locking will be used and whether or not the library + ignores errors when locking has been disabled on the file system + (useful on some HPC Lustre installations). + + In both the Autotools and CMake, the settings have the effect of changing + the default property list settings (see the H5Pset/get_file_locking() + entry, below). + + The yes/no/best-effort file locking configure setting has also been + added to the libhdf5.settings file. + + Autotools: + + An --enable-file-locking=(yes|no|best-effort) option has been added. + + yes: Use file locking. + no: Do not use file locking. + best-effort: Use file locking and ignore "disabled" errors. + + CMake: + + Two self-explanatory options have been added: + + HDF5_USE_FILE_LOCKING + HDF5_IGNORE_DISABLED_FILE_LOCKS + + Setting both of these to ON is the equivalent to the Autotools' + best-effort setting. + + NOTE: + The precedence order of the various file locking control mechanisms is: + + 1) HDF5_USE_FILE_LOCKING environment variable (highest) + + 2) H5Pset_file_locking() + + 3) configure/CMake options (which set the property list defaults) + + 4) library defaults (currently best-effort) + + (DER - 2020/07/30, HDFFV-11092) + + - CMake option to link the generated Fortran MOD files into the include + directory. + + The Fortran generation of MOD files by a Fortran compile can produce + different binary files between SHARED and STATIC compiles with different + compilers and/or different platforms. Note that it has been found that + different versions of Fortran compilers will produce incompatible MOD + files. Currently, CMake will locate these MOD files in subfolders of + the include directory and add that path to the Fortran library target + in the CMake config file, which can be used by the CMake find library + process. For other build systems using the binary from a CMake install, + a new CMake configuration can be used to copy the pre-chosen version + of the Fortran MOD files into the install include directory. + + The default will depend on the configuration of + BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: + YES YES Default to SHARED + YES NO Default to STATIC + NO YES Default to SHARED + NO NO Default to SHARED + The defaults can be overridden by setting the config option + HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC + + (ADB - 2020/07/09, HDFFV-11116) + + - CMake option to use AEC (open source SZip) library instead of SZip + + The open source AEC library is a replacement library for SZip. In + order to use it for hdf5 the libaec CMake source was changed to add + "-fPIC" and exclude test files. Autotools does not build the + compression libraries within hdf5 builds. New option USE_LIBAEC is + required to compensate for the different files produced by AEC build. + + (ADB - 2020/04/22, OESS-65) + + - CMake ConfigureChecks.cmake file now uses CHECK_STRUCT_HAS_MEMBER + + Some handcrafted tests in HDFTests.c has been removed and the CMake + CHECK_STRUCT_HAS_MEMBER module has been used. + + (ADB - 2020/03/24, TRILAB-24) + + - Both build systems use same set of warnings flags + + GNU C, C++ and gfortran warnings flags were moved to files in a config + sub-folder named gnu-warnings. Flags that only are available for a specific + version of the compiler are in files named with that version. + Clang C warnings flags were moved to files in a config sub-folder + named clang-warnings. + Intel C, Fortran warnings flags were moved to files in a config sub-folder + named intel-warnings. + + There are flags in named "error-xxx" files with warnings that may + be promoted to errors. Some source files may still need fixes. + + There are also pairs of files named "developer-xxx" and "no-developer-xxx" + that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the + configure option:--enable-developer-warnings. + + In addition, CMake no longer applies these warnings for examples. + + (ADB - 2020/03/24, TRILAB-192) + + - Added test script for file size compare + + If CMake minimum version is at least 3.14, the fileCompareTest.cmake + script will compare file sizes. + + (ADB - 2020/02/24, HDFFV-11036) + + - Update CMake minimum version to 3.12 + + Updated CMake minimum version to 3.12 and added version checks + for Windows features. + + (ADB - 2020/02/05, TRILABS-142) + + - Fixed CMake include properties for Fortran libraries + + Corrected the library properties for Fortran to use the + correct path for the Fortran module files. + + (ADB - 2020/02/04, HDFFV-11012) + + - Added common warnings files for gnu and intel + + Added warnings files to use one common set of flags + during configure for both autotools and CMake build + systems. The initial implementation only affects a + general set of flags for gnu and intel compilers. + + (ADB - 2020/01/17) + + - Added new options to CMake for control of testing + + Added CMake options (default ON); + HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL + combined with: + HDF5_TEST_TOOLS + HDF5_TEST_EXAMPLES + HDF5_TEST_SWMR + HDF5_TEST_FORTRAN + HDF5_TEST_CPP + HDF5_TEST_JAVA + + (ADB - 2020/01/15, HDFFV-11001) + + - Added Clang sanitizers to CMake for analyzer support if compiler is clang. + + Added CMake code and files to execute the Clang sanitizers if + HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option + is set to one of the following: + Address + Memory + MemoryWithOrigins + Undefined + Thread + Leak + 'Address;Undefined' + + (ADB - 2019/12/12, TRILAB-135) + + - Update CMake for VS2019 support + + CMake added support for VS2019 in version 3.15. Changes to the CMake + generator setting required changes to scripts. Also updated version + references in CMake files as necessary. + + (ADB - 2019/11/18, HDFFV-10962) + + - Update CMake options to match new autotools options + + Add configure options (autotools - CMake): + enable-asserts HDF5_ENABLE_ASSERTS + enable-symbols HDF5_ENABLE_SYMBOLS + enable-profiling HDF5_ENABLE_PROFILING + enable-optimization HDF5_ENABLE_OPTIMIZATION + In addition NDEBUG is no longer forced defined and relies on the CMake + process. + + (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) + + + Library: + -------- + - Adds new file driver-level memory copy operation for + "ctl" callback and updates compact dataset I/O routines + to utilize it + + When accessing an HDF5 file with a file driver that uses + memory allocated in special ways (e.g., without standard + library's `malloc`), a crash could be observed when HDF5 + tries to perform `memcpy` operations on such a memory + region. + + These changes add a new H5FD_FEAT_MEMMANAGE VFD feature + flag, which, if specified as supported by a VFD, will + inform HDF5 that the VFD either uses special memory + management routines or wishes to perform memory management + in a specific way. Therefore, this flag instructs HDF5 to + ask the file driver to perform memory management for + certain operations. + + These changes also introduce a new "ctl" callback + operation identified by the H5FD_CTL__MEM_COPY op code. + This operation simply asks a VFD to perform a memory copy. + The arguments to this operation are passed to the "ctl" + callback's "input" parameter as a pointer to a struct + defined as: + + struct H5FD_ctl_memcpy_args_t { + void * dstbuf; /**< Destination buffer */ + hsize_t dst_off; /**< Offset within destination buffer */ + const void *srcbuf; /**< Source buffer */ + hsize_t src_off; /**< Offset within source buffer */ + size_t len; /**< Length of data to copy from source buffer */ + } H5FD_ctl_memcpy_args_t; + + Further, HDF5's compact dataset I/O routines were + identified as a problematic area that could cause a crash + for VFDs that make use of special memory management. Those + I/O routines were therefore updated to make use of this new + "ctl" callback operation in order to ask the underlying + file driver to correctly handle memory copies. + + (JTH - 2021/09/28) + + - Adds new "ctl" callback to VFD H5FD_class_t structure + with the following prototype: + + herr_t (*ctl)(H5FD_t *file, uint64_t op_code, + uint64_t flags, const void *input, + void **output); + + This newly-added "ctl" callback allows Virtual File + Drivers to intercept and handle arbitrary operations + identified by an operation code. Its parameters are + as follows: + + `file` [in] - A pointer to the file to be operated on + `op_code` [in] - The operation code identifying the + operation to be performed + `flags` [in] - Flags governing the behavior of the + operation performed (see H5FDpublic.h + for a list of valid flags) + `input` [in] - A pointer to arguments passed to the + VFD performing the operation + `output` [out] - A pointer for the receiving VFD to + use for output from the operation + + (JRM - 2021/08/16) + + - Change how the release part of version, in major.minor.release is checked + for compatibility + + The HDF5 library uses a function, H5check_version, to check that + the version defined in the header files, which is used to compile an + application is compatible with the version codified in the library, which + the application loads at runtime. This previously required an exact match + or the library would print a warning, dump the build settings and then + abort or continue. An environment variable controlled the logic. + + Now the function first checks that the library release version, in + major.minor.release, is not older than the version in the headers. + Secondly, if the release version is different, it checks if either + the library version or the header version is in the exception list, in + which case the release part of version, in major.minor.release, must + be exact. An environment variable still controls the logic. + + (ADB - 2021/07/27) + + - gcc warning suppression macros were moved out of H5public.h + + The HDF5 library uses a set of macros to suppress warnings on gcc. + These warnings were originally located in H5public.h so that the + multi VFD (which only uses public headers) could also make use of them + but internal macros should not be publicly exposed like this. + + These macros have now been moved to H5private.h. Pending future multi + VFD refactoring, the macros have been duplicated in H5FDmulti.c to + suppress the format string warnings there. + + (DER - 2021/06/03) + + - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX + + The size_hint value is ultimately stored in a uint32_t struct field, + so specifying a value larger than this on a 64-bit machine can cause + undefined behavior including crashing the system. + + The documentation for this API call was also incorrect, stating that + passing a negative value would cause the library to use a default + value. Instead, passing a "negative" value actually passes a very large + value, which is probably not what the user intends and can cause + crashes on 64-bit systems. + + The Doxygen documentation has been updated and passing values larger + than UINT32_MAX for size_hint will now produce a normal HDF5 error. + + (DER - 2021/04/29, HDFFV-11241) + + + - H5Pset_fapl_log() no longer crashes when passed an invalid fapl ID + + When passed an invalid fapl ID, H5Pset_fapl_log() would usually + segfault when attempting to free an uninitialized pointer in the error + handling code. This behavior is more common in release builds or + when the memory sanitization checks were not selected as a build + option. + + The pointer is now correctly initialized and the API call now + produces a normal HDF5 error when fed an invalid fapl ID. + + (DER - 2021/04/28, HDFFV-11240) + + - Fixes a segfault when H5Pset_mdc_log_options() is called multiple times + + The call incorrectly attempts to free an internal copy of the previous + log location string, which causes a segfault. This only happens + when the call is invoked multiple times on the same property list. + On the first call to a given fapl, the log location is set to NULL so + the segfault does not occur. + + The string is now handled properly and the segfault no longer occurs. + + (DER - 2021/04/27, HDFFV-11239) + + - HSYS_GOTO_ERROR now emits the results of GetLastError() on Windows + + HSYS_GOTO_ERROR is an internal macro that is used to produce error + messages when system calls fail. These strings include errno and the + the associated strerror() value, which are not particularly useful + when a Win32 API call fails. + + On Windows, this macro has been updated to include the result of + GetLastError(). When a system call fails on Windows, usually only + one of errno and GetLastError() will be useful, however we emit both + for the user to parse. The Windows error message is not emitted as + it would be awkward to free the FormatMessage() buffer given the + existing HDF5 error framework. Users will have to look up the error + codes in MSDN. + + The format string on Windows has been changed from: + + "%s, errno = %d, error message = '%s'" + + to: + + "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" + + for those inclined to parse it for error values. + + (DER - 2021/03/21) + + - File locking now works on Windows + + Since version 1.10.0, the HDF5 library has used a file locking scheme + to help enforce one reader at a time accessing an HDF5 file, which can + be helpful when setting up readers and writers to use the single- + writer/multiple-readers (SWMR) access pattern. + + In the past, this was only functional on POSIX systems where flock() or + fcntl() were present. Windows used a no-op stub that always succeeded. + + HDF5 now uses LockFileEx() and UnlockFileEx() to lock the file using the + same scheme as POSIX systems. We lock the entire file when we set up the + locks (by passing DWORDMAX as both size parameters to LockFileEx()). + + (DER - 2021/03/19, HDFFV-10191) + + - H5Epush_ret() now requires a trailing semicolon + + H5Epush_ret() is a function-like macro that has been changed to + contain a `do {} while(0)` loop. Consequently, a trailing semicolon + is now required to end the `while` statement. Previously, a trailing + semi would work, but was not mandatory. This change was made to allow + clang-format to correctly format the source code. + + (SAM - 2021/03/03) + + - Improved performance of H5Sget_select_elem_pointlist + + Modified library to cache the point after the last block of points + retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the + same function to retrieve the next block of points from the list can + proceed immediately without needing to iterate over the point list. + + (NAF - 2021/01/19) + + - Replaced H5E_ATOM with H5E_ID in H5Epubgen.h + + The term "atom" is archaic and not in line with current HDF5 library + terminology, which uses "ID" instead. "Atom" has mostly been purged + from the library internals and this change removes H5E_ATOM from + the H5Epubgen.h (exposed via H5Epublic.h) and replaces it with + H5E_ID. + + (DER - 2020/11/24, HDFFV-11190) + + - Add a new public function H5Ssel_iter_reset + + This function resets a dataspace selection iterator back to an + initial state so that it may be used for iteration once more. + This can be useful when needing to iterate over a selection + multiple times without having to repeatedly create/destroy + a selection iterator for that dataspace selection. + + (JTH - 2020/09/18) + + - Remove HDFS VFD stubs + + The original implementation of the HDFS VFD included non-functional + versions of the following public API calls when the HDFS VFD is + not built as a part of the HDF5 library: + + * H5FD_hdfs_init() + * H5Pget_fapl_hdfs() + * H5Pset_fapl_hdfs() + + They will remain present in HDF5 1.10 and HDF5 1.12 releases + for binary compatibility purposes but have been removed as of 1.14.0. + + Note that this has nothing to do with the real HDFS VFD API calls + that are fully functional when the HDFS VFD is configured and built. + + We simply changed: + + #ifdef LIBHDFS + + #else + + #endif + + to: + + #ifdef LIBHDFS + + #endif + + Which is how the other optional VFDs are handled. + + (DER - 2020/08/27) + + - Add Mirror VFD + + Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote + machine. Must be used in conjunction with the Splitter VFD. + + (JOS - 2020/03/13, TBD) + + - Add Splitter VFD + + Maintain separate R/W and W/O channels for "concurrent" file writes + to two files using a single HDF5 file handle. + + (JOS - 2020/03/13, TBD) + + - Refactored public exposure of haddr_t type in favor of "object tokens" + + To better accommodate HDF5 VOL connectors where "object addresses in a file" + may not make much sense, the following changes were made to the library: + + * Introduced new H5O_token_t "object token" type, which represents a + unique and permanent identifier for referencing an HDF5 object within + a container; these "object tokens" are meant to replace object addresses. + Along with the new type, a new H5Oopen_by_token API call was introduced + to open an object by a token, similar to how object addresses were + previously used with H5Oopen_by_addr. + + * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, + H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t + struct and H5L_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The original functions have been + renamed to version 1 functions and are deprecated in favor of the new version 2 + functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 + types and are now deprecated in favor of their version 2 counterparts. For each of + the functions and types, compatibility macros take place of the original symbols. + + * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, + H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t + struct and H5O_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The version 2 functions are now + deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t + types have been renamed to version 1 types and are now deprecated in favor of their + version 2 counterparts. For each, compatibility macros take place of the original + symbols. + + * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and + H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t + struct, which are used to retrieve the native HDF5 file format-specific information + about an object. This information (such as object header info and B-tree/heap info) + has been removed from the new H5O_info2_t struct so that the more generic + H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native + VOL connectors. + + * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare + two object tokens, convert an object token into a nicely-readable string format and + to convert an object token string back into a real object token, respectively. + + (DER, QAK, JTH - 2020/01/16) + + - Add new public function H5Sselect_adjust. + + This function shifts a dataspace selection by a specified logical offset + within the dataspace extent. This can be useful for VOL developers to + implement chunked datasets. + + (NAF - 2019/11/18) + + - Add new public function H5Sselect_project_intersection. + + This function computes the intersection between two dataspace selections + and projects that intersection into a third selection. This can be useful + for VOL developers to implement chunked or virtual datasets. + + (NAF - 2019/11/13, ID-148) + + - Add new public function H5VLget_file_type. + + This function returns a datatype equivalent to the supplied datatype but + with the location set to be in the file. This datatype can then be used + with H5Tconvert to convert data between file and in-memory representation. + This function is intended for use only by VOL connector developers. + + (NAF - 2019/11/08, ID-127) + + + Parallel Library: + ----------------- + - + + + Fortran Library: + ---------------- + - H5Fget_name_f fixed to handle correctly trailing whitespaces and + newly allocated buffers. + + (MSB - 2021/08/30, github-826,972) + + - Add wrappers for H5Pset/get_file_locking() API calls + + h5pget_file_locking_f() + h5pset_file_locking_f() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + C++ Library: + ------------ + - Add wrappers for H5Pset/get_file_locking() API calls + + FileAccPropList::setFileLocking() + FileAccPropList::getFileLocking() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + + Java Library: + ------------- + - Replaced HDF5AtomException with HDF5IdException + + Since H5E_ATOM changed to H5E_ID in the C library, the Java exception + that wraps the error category was also renamed. Its functionality + remains unchanged aside from the name. + + (See also the HDFFV-11190 note in the C library section) + + (DER - 2020/11/24, HDFFV-11190) + + - Added new H5S functions. + + H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, + H5Sselect_intersect_block, H5Sselect_project_intersection, + H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select + wrapper functions added. + + (ADB - 2020/10/27, HDFFV-10868) + + - Add wrappers for H5Pset/get_file_locking() API calls + + H5Pset_file_locking() + H5Pget_use_file_locking() + H5Pget_ignore_disabled_file_locking() + + Unlike the C++ and Fortran wrappers, there are separate getters for the + two file locking settings, each of which returns a boolean value. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + - Added ability to test java library with VOLs. + + Created a new CMake script that combines the java and vol test scripts. + + (ADB - 2020/02/03, HDFFV-10996) + + - Tests fail for non-English locales. + + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. + + (ADB - 2020/01/09, HDFFV-10995) + + + Tools: + ------ + - Added a new (unix ONLY) parallel meta tool 'h5dwalk', which utilizes the + mpifileutils (https://hpc.github.io/mpifileutils) open source utility + library to enable parallel execution of other HDF5 tools. + This approach can greatly enhance the serial hdf5 tool performance over large + collections of files by utilizing MPI parallelism to distribute an application + load over many independent MPI ranks and files. + + An introduction to the mpifileutils library and initial 'User Guide' for + the new 'h5dwalk" tool can be found at: + https://github.com/HDFGroup/hdf5doc/tree/master/RFCs/HDF5/tools/parallel_tools + + (RAW - 2021/10/25) + + - Refactored the perform tools and removed depends on test library. + + Moved the perf and h5perf tools from tools/test/perform to + tools/src/h5perf so that they can be installed. This required + that the test library dependency be removed by copying the + needed functions from h5test.c. + The standalone scripts and other perform tools remain in the + tools/test/perform folder. + + (ADB - 2021/08/10) + + - Removed partial long exceptions + + Some of the tools accepted shortened versions of the long options + (ex: --datas instead of --dataset). These were implemented inconsistently, + are difficult to maintian, and occasionally block useful long option + names. These partial long options have been removed from all the tools. + + (DER - 2021/08/03) + + - h5repack added help text for user-defined filters. + + Added help text line that states the valid values of the filter flag + for user-defined filters; + filter_flag: 1 is OPTIONAL or 0 is MANDATORY + + (ADB - 2021/01/14, HDFFV-11099) + + - Added h5delete tool + + Deleting HDF5 storage when using the VOL can be tricky when the VOL + does not create files. The h5delete tool is a simple wrapper around + the H5Fdelete() API call that uses the VOL specified in the + HDF5_VOL_CONNECTOR environment variable to delete a "file". If + the call to H5Fdelete() fails, the tool will attempt to use + the POSIX remove(3) call to remove the file. + + Note that the HDF5 library does currently have support for + H5Fdelete() in the native VOL connector. + + (DER - 2020/12/16) + + - h5repack added options to control how external links are handled. + + Currently h5repack preserves external links and cannot copy and merge + data from the external files. Two options, merge and prune, were added to + control how to merge data from an external link into the resulting file. + --merge Follow external soft link recursively and merge data. + --prune Do not follow external soft links and remove link. + --merge --prune Follow external link, merge data and remove dangling link. + + (ADB - 2020/08/05, HDFFV-9984) + + - h5repack was fixed to repack the reference attributes properly. + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. + + (KY -2020/02/07, HDFFV-11014) + + - h5diff was updated to use the new reference APIs. + + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. + + (ADB - 2019/12/19, HDFFV-10980) + + - h5dump and h5ls were updated to use the new reference APIs. + + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. + + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + + + High-Level APIs: + ---------------- + - added set/get for unsigned long long attributes + + The attribute writing high-level API has been expanded to include + public set/get functions for ULL attributes, analogously to the + existing set/get for other types. + + (AF - 2021/09/08) + + C Packet Table API: + ------------------- + - + + Internal header file: + --------------------- + - + + Documentation: + -------------- + - + + +New platforms, languages and compilers tested +============================================= + - Linux 5.13.14-200.fc34 #1 SMP x86_64 GNU/Linux Fedora34 + - Linux 5.11.0-34-generic #36-Ubuntu SMP x86_64 GNU/Linux Ubuntu 20.04 + - Linux 5.3.18-22-default #1 SMP x86_64 GNU/Linux SUSE15sp2 + - Linux-4.14.0-115.21.2 #1 SMP ppc64le GNU/Linux + - Linux-4.12.14-150.75-default #1 SMP x86_64 GNU/Linux + - macOS Apple M1 11.6 Darwin 20.6.0 arm64 + - macOS Big Sur 11.3.1 Darwin 20.4.0 x86_64 + - clang versions 11.0.1, 12.0.5 + - Visual Studio 2019 w/ clang 12.0.0 with MSVC-like command-line + + +Bug Fixes since HDF5-1.12.0 release +=================================== + Library + ------- + - Fixed an H5Pget_filter_by_id1/2() assert w/ out of range filter IDs + + Both H5Pget_filter_by_id1 and 2 did not range check the filter ID, which + could trip as assert in debug versions of the library. The library now + returns a normal HDF5 error when the filter ID is out of range. + + (DER - 2021/11/23, HDFFV-11286) + + - Fixed an issue with collective metadata reads being permanently disabled + after a dataset chunk lookup operation. This would usually cause a + mismatched MPI_Bcast and MPI_ERR_TRUNCATE issue in the library for + simple cases of H5Dcreate() -> H5Dwrite() -> H5Dcreate(). + + (JTH - 2021/11/08, HDFFV-11090) + + - Fixed cross platform incompatibility of references within variable length + types + + Reference types within variable length types previously could not be + read on a platform with different endianness from where they were + written. Fixed so cross platform portability is restored. + + (NAF - 2021/09/30) + + - Detection of simple data transform function "x" + + In the case of the simple data transform function "x" the (parallel) + library recognizes this is the same as not applying this data transform + function. This improves the I/O performance. In the case of the parallel + library, it also avoids breaking to independent I/O, which makes it + possible to apply a filter when writing or reading data to or from + the HDF5 file. + + (JWSB - 2021/09/13) + + - Fixed an invalid read and memory leak when parsing corrupt file space + info messages + + When the corrupt file from CVE-2020-10810 was parsed by the library, + the code that imports the version 0 file space info object header + message to the version 1 struct could read past the buffer read from + the disk, causing an invalid memory read. Not catching this error would + cause downstream errors that eventually resulted in a previously + allocated buffer to be unfreed when the library shut down. In builds + where the free lists are in use, this could result in an infinite loop + and SIGABRT when the library shuts down. + + We now track the buffer size and raise an error on attempts to read + past the end of it. + + (DER - 2021/08/12, HDFFV-11053) + + + - Fixed CVE-2018-14460 + + The tool h5repack produced a segfault when the rank in dataspace + message was corrupted, causing invalid read while decoding the + dimension sizes. + + The problem was fixed by ensuring that decoding the dimension sizes + and max values will not go beyond the end of the buffer. + + (BMR - 2021/05/12, HDFFV-11223) + + - Fixed CVE-2018-11206 + + The tool h5dump produced a segfault when the size of a fill value + message was corrupted and caused a buffer overflow. + + The problem was fixed by verifying the fill value's size + against the buffer size before attempting to access the buffer. + + (BMR - 2021/03/15, HDFFV-10480) + + - Fixed CVE-2018-14033 (same issue as CVE-2020-10811) + + The tool h5dump produced a segfault when the storage size message + was corrupted and caused a buffer overflow. + + The problem was fixed by verifying the storage size against the + buffer size before attempting to access the buffer. + + (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) + + - Remove underscores on header file guards + + Header file guards used a variety of underscores at the beginning of the define. + + Removed all leading (some trailing) underscores from header file guards. + + (ADB - 2021/03/03, #361) + + - Fixed a segmentation fault + + A segmentation fault occurred with a Mathworks corrupted file. + + A detection of accessing a null pointer was added to prevent the problem. + + (BMR - 2021/02/19, HDFFV-11150) + + - Fixed issue with MPI communicator and info object not being + copied into new FAPL retrieved from H5F_get_access_plist + + Added logic to copy the MPI communicator and info object into + the output FAPL. MPI communicator is retrieved from the VFD, while + the MPI info object is retrieved from the file's original FAPL. + + (JTH - 2021/02/15, HDFFV-11109) + + - Fixed problems with vlens and refs inside compound using + H5VLget_file_type() + + Modified library to properly ref count H5VL_object_t structs and only + consider file vlen and reference types to be equal if their files are + the same. + + (NAF - 2021/01/22) + + - Fixed CVE-2018-17432 + + The tool h5repack produced a segfault on a corrupted file which had + invalid rank for scalar or NULL datatype. + + The problem was fixed by modifying the dataspace encode and decode + functions to detect and report invalid rank. h5repack now fails + with an error message for the corrupted file. + + (BMR - 2020/10/26, HDFFV-10590) + + - Creation of dataset with optional filter + + When the combination of type, space, etc doesn't work for filter + and the filter is optional, it was supposed to be skipped but it was + not skipped and the creation failed. + + Allowed the creation of the dataset in such a situation. + + (BMR - 2020/08/13, HDFFV-10933) + + - Explicitly declared dlopen to use RTLD_LOCAL + + dlopen documentation states that if neither RTLD_GLOBAL nor + RTLD_LOCAL are specified, then the default behavior is unspecified. + The default on linux is usually RTLD_LOCAL while macos will default + to RTLD_GLOBAL. + + (ADB - 2020/08/12, HDFFV-11127) + + - H5Sset_extent_none() sets the dataspace class to H5S_NO_CLASS which + causes asserts/errors when passed to other dataspace API calls. + + H5S_NO_CLASS is an internal class value that should not have been + exposed via a public API call. + + In debug builds of the library, this can cause assert() function to + trip. In non-debug builds, it will produce normal library errors. + + The new library behavior is for H5Sset_extent_none() to convert + the dataspace into one of type H5S_NULL, which is better handled + by the library and easier for developers to reason about. + + (DER - 2020/07/27, HDFFV-11027) + + - Fixed issues CVE-2018-13870 and CVE-2018-13869 + + When a buffer overflow occurred because a name length was corrupted + and became very large, h5dump crashed on memory access violation. + + A check for reading pass the end of the buffer was added to multiple + locations to prevent the crashes and h5dump now simply fails with an + error message when this error condition occurs. + + (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) + + - Fixed the segmentation fault when reading attributes with multiple threads + + It was reported that the reading of attributes with variable length string + datatype will crash with segmentation fault particularly when the number of + threads is high (>16 threads). The problem was due to the file pointer that + was set in the variable length string datatype for the attribute. That file + pointer was already closed when the attribute was accessed. + + The problem was fixed by setting the file pointer to the current opened file pointer + when the attribute was accessed. Similar patch up was done before when reading + dataset with variable length string datatype. + + (VC - 2020/07/13, HDFFV-11080) + + - Fixed CVE-2020-10810 + + The tool h5clear produced a segfault during an error recovery in + the superblock decoding. An internal pointer was reset to prevent + further accessing when it is not assigned with a value. + + (BMR - 2020/06/29, HDFFV-11053) + + - Fixed CVE-2018-17435 + + The tool h52gif produced a segfault when the size of an attribute + message was corrupted and caused a buffer overflow. + + The problem was fixed by verifying the attribute message's size + against the buffer size before accessing the buffer. h52gif was + also fixed to display the failure instead of silently exiting + after the segfault was eliminated. + + (BMR - 2020/06/19, HDFFV-10591) + + + Java Library + ------------ + - JNI utility function does not handle new references. + + The JNI utility function for converting reference data to string did + not use the new APIs. In addition to fixing that function, added new + java tests for using the new APIs. + + (ADB - 2021/02/16, HDFFV-11212) + + - The H5FArray.java class, in which virtually the entire execution time + is spent using the HDFNativeData method that converts from an array + of bytes to an array of the destination Java type. + + 1. Convert the entire byte array into a 1-d array of the desired type, + rather than performing 1 conversion per row; + 2. Use the Java Arrays method copyOfRange to grab the section of the + array from (1) that is desired to be inserted into the destination array. + + (PGT,ADB - 2020/12/13, HDFFV-10865) + + - Added ability to test java library with VOLs. + + Created a new CMake script that combines the java and vol test scripts. + + (ADB - 2020/02/03, HDFFV-10996) + + - Tests fail for non-English locales. + + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. + + (ADB - 2020/01/09, HDFFV-10995) + + + Configuration + ------------- + - Corrected path searched by CMake find_package command + + The install path for cmake find_package files had been changed to use + "share/cmake" + for all platforms. However the trailing "hdf5" directory was not removed. + This "hdf5" additional directory has been removed. + + (ADB - 2021/09/27) + + - Corrected pkg-config compile script + + It was discovered that the position of the "$@" argument for the command + in the compile script may fail on some platforms and configurations. The + position of the "$@"command argument was moved before the pkg-config sub command. + + (ADB - 2021/08/30) + + - Fixed CMake C++ compiler flags + + A recent refactoring of the C++ configure files accidentally removed the + file that executed the enable_language command for C++ needed by the + HDFCXXCompilerFlags.cmake file. Also updated the intel warnings files, + including adding support for windows platforms. + + (ADB - 2021/08/10) + + - Better support for libaec (open-source Szip library) in CMake + + Implemented better support for libaec 1.0.5 (or later) library. This version + of libaec contains improvements for better integration with HDF5. Furthermore, + the variable USE_LIBAEC_STATIC has been introduced to allow to make use of + static version of libaec library. Use libaec_DIR or libaec_ROOT to set + the location in which libaec can be found. + + Be aware, the Szip library of libaec 1.0.4 depends on another library within + libaec library. This dependency is not specified in the current CMake + configuration which means that one can not use the static Szip library of + libaec 1.0.4 when building HDF5. This has been resolved in libaec 1.0.5. + + (JWSB - 2021/06/22) + + - Refactor CMake configure for Fortran + + The Fortran configure tests for KINDs reused a single output file that was + read to form the Integer and Real Kinds defines. However, if config was run + more then once, the CMake completed variable prevented the tests from executing + again and the last value saved in the file was used to create the define. + Creating separate files for each KIND solved the issue. + + In addition the test for H5_PAC_C_MAX_REAL_PRECISION was not pulling in + defines for proper operation and did not define H5_PAC_C_MAX_REAL_PRECISION + correctly for a zero value. This was fixed by supplying the required defines. + In addition it was moved from the Fortran specific HDF5UseFortran.camke file + to the C centric ConfigureChecks.cmake file. + + (ADB - 2021/06/03) + + - Move emscripten flag to compile flags + + The emscripten flag, -O0, was removed from target_link_libraries command + to the correct target_compile_options command. + + (ADB - 2021/04/26 HDFFV-11083) + + - Remove arbitrary warning flag groups from CMake builds + + The arbitrary groups were created to reduce the quantity of warnings being + reported that overwhelmed testing report systems. Considerable work has + been accomplished to reduce the warning count and these arbitrary groups + are no longer needed. + Also the default for all warnings, HDF5_ENABLE_ALL_WARNINGS, is now ON. + + Visual Studio warnings C4100, C4706, and C4127 have been moved to + developer warnings, HDF5_ENABLE_DEV_WARNINGS, and are disabled for normal builds. + + (ADB - 2021/03/22, HDFFV-11228) + + - Reclassify CMake messages, to allow new modes and --log-level option + + CMake message commands have a mode argument. By default, STATUS mode + was chosen for any non-error message. CMake version 3.15 added additional + modes, NOTICE, VERBOSE, DEBUG and TRACE. All message commands with a mode + of STATUS were reviewed and most were reclassified as VERBOSE. The new + mode was protected by a check for a CMake version of at least 3.15. If CMake + version 3.17 or above is used, the user can use the command line option + of "--log-level" to further restrict which message commands are displayed. + + (ADB - 2021/01/11, HDFFV-11144) + + - Fixes Autotools determination of the stat struct having an st_blocks field + + A missing parenthesis in an autoconf macro prevented building the test + code used to determine if the stat struct contains the st_blocks field. + Now that the test functions correctly, the H5_HAVE_STAT_ST_BLOCKS #define + found in H5pubconf.h will be defined correctly on both the Autotools and + CMake. This #define is only used in the tests and does not affect the + HDF5 C library. + + (DER - 2021/01/07, HDFFV-11201) + + - Add missing ENV variable line to hdfoptions.cmake file + + Using the build options to use system SZIP/ZLIB libraries need to also + specify the library root directory. Setting the {library}_ROOT ENV + variable was added to the hdfoptions.cmake file. + + (ADB - 2020/10/19 HDFFV-11108) + + + Tools + ----- + - Changed how h5dump and h5ls identify long double. + + Long double support is not consistent across platforms. Tools will always + identify long double as 128-bit [little/big]-endian float nn-bit precision. + New test file created for datasets with attributes for float, double and + long double. In addition any unknown integer or float datatype will now + also show the number of bits for precision. + These files are also used in the java tests. + + (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) + + - Fixed tools argument parsing. + + Tools parsing used the length of the option from the long array to match + the option from the command line. This incorrectly matched a shorter long + name option that happened to be a subset of another long option. + Changed to match whole names. + + (ADB - 2021/01/19, HDFFV-11106) + + - The tools library was updated by standardizing the error stack process. + + General sequence is: + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + h5tools_init(); + ... process the command-line (check for error-stack enable) ... + h5tools_error_report(); + ... (do work) ... + h5diff_exit(ret); + + (ADB - 2020/07/20, HDFFV-11066) + + - h5diff fixed a command line parsing error. + + h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. + The macro H5_DBL_ABS_EQUAL was removed and a direct value comparison was used. + + (ADB - 2020/07/20, HDFFV-10897) + + - h5diff added a command line option to ignore attributes. + + h5diff would ignore all objects with a supplied path if the exclude-path argument is used. + Adding the exclude-attribute argument will only exclude attributes, with the supplied path, + from comparison. + + (ADB - 2020/07/20, HDFFV-5935) + + - h5diff added another level to the verbose argument to print filenames. + + Added verbose level 3 that is level 2 plus the filenames. The levels are: + 0 : Identical to '-v' or '--verbose' + 1 : All level 0 information plus one-line attribute status summary + 2 : All level 1 information plus extended attribute status report + 3 : All level 2 information plus file names + + (ADB - 2020/07/20, HDFFV-1005) + + - h5repack was fixed to repack the reference attributes properly. + + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. + + (KY -2020/02/10, HDFFV-11014) + + - h5diff was updated to use the new reference APIs. + + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. + + (ADB - 2019/12/19, HDFFV-10980) + + - h5dump and h5ls were updated to use the new reference APIs. + + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. + + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + + + Performance + ------------- + - + + + Fortran API + ----------- + - Corrected INTERFACE INTENT(IN) to INTENT(OUT) for buf_size in h5fget_file_image_f. + + (MSB - 2020/02/18, HDFFV-11029) + + + High-Level Library + ------------------ + - Fixed HL_test_packet, test for packet table vlen of vlen. + + Incorrect length assignment. + + (ADB - 2021/10/14) + + + Fortran High-Level APIs + ----------------------- + - + + + Documentation + ------------- + - + + + F90 APIs + -------- + - + + + C++ APIs + -------- + - Added DataSet::operator= + + Some compilers complain if the copy constructor is given explicitly + but the assignment operator is implicitly set to default. + + (2021/05/19) + + + Testing + ------- + - Stopped java/test/junit.sh.in installing libs for testing under ${prefix} + + Lib files needed are now copied to a subdirectory in the java/test + directory, and on Macs the loader path for libhdf5.xxxs.so is changed + in the temporary copy of libhdf5_java.dylib. + + (LRK, 2020/07/02, HDFFV-11063) + + +Platforms Tested +=================== + + Linux 5.13.14-200.fc34 GNU gcc (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + #1 SMP x86_64 GNU/Linux GNU Fortran (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + Fedora34 clang version 12.0.1 (Fedora 12.0.1-1.fc34) + (cmake and autotools) + + Linux 5.11.0-34-generic GNU gcc (GCC) 9.3.0-17ubuntu1 + #36-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 9.3.0-17ubuntu1 + Ubuntu 20.04 Ubuntu clang version 10.0.0-4 + (cmake and autotools) + + Linux 5.8.0-63-generic GNU gcc (GCC) 10.3.0-1ubuntu1 + #71-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 10.3.0-1ubuntu1 + Ubuntu20.10 Ubuntu clang version 11.0.0-2 + (cmake and autotools) + + Linux 5.3.18-22-default GNU gcc (SUSE Linux) 7.5.0 + #1 SMP x86_64 GNU/Linux GNU Fortran (SUSE Linux) 7.5.0 + SUSE15sp2 clang version 7.0.1 (tags/RELEASE_701/final 349238) + (cmake and autotools) + + Linux-4.14.0-115.21.2 spectrum-mpi/rolling-release + #1 SMP ppc64le GNU/Linux clang 8.0.1, 11.0.1 + (lassen) GCC 7.3.1 + XL 16.1.1.2 + (cmake) + + Linux-4.12.14-150.75-default cray-mpich/7.7.10 + #1 SMP x86_64 GNU/Linux GCC 7.3.0, 8.2.0 + (cori) Intel (R) Version 19.0.3.199 + (cmake) + + Linux-4.12.14-197.86-default cray-mpich/7.7.6 + # 1SMP x86_64 GNU/Linux GCC 7.3.0, 9.3.0, 10.2.0 + (mutrino) Intel (R) Version 17.0.4, 18.0.5, 19.1.3 + (cmake) + + Linux 3.10.0-1160.36.2.el7.ppc64 gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + #1 SMP ppc64be GNU/Linux g++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + Power8 (echidna) GNU Fortran (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + + Linux 3.10.0-1160.24.1.el7 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos7 Version 4.8.5 20150623 (Red Hat 4.8.5-4) + (jelly/kituo/moohan) Version 4.9.3, Version 5.3.0, Version 6.3.0, + Version 7.2.0, Version 8.3.0, Version 9.1.0 + Intel(R) C (icc), C++ (icpc), Fortran (icc) + compilers: + Version 17.0.0.098 Build 20160721 + GNU C (gcc) and C++ (g++) 4.8.5 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + Intel(R) C (icc) and C++ (icpc) 17.0.0.098 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + MPICH 3.1.4 compiled with GCC 4.9.3 + MPICH 3.3 compiled with GCC 7.2.0 + OpenMPI 2.1.6 compiled with icc 18.0.1 + OpenMPI 3.1.3 and 4.0.0 compiled with GCC 7.2.0 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Linux-3.10.0-1127.0.0.1chaos openmpi-4.0.0 + #1 SMP x86_64 GNU/Linux clang 6.0.0, 11.0.1 + (quartz) GCC 7.3.0, 8.1.0 + Intel 16.0.4, 18.0.2, 19.0.4 + + macOS Apple M1 11.6 Apple clang version 12.0.5 (clang-1205.0.22.11) + Darwin 20.6.0 arm64 gfortran GNU Fortran (Homebrew GCC 11.2.0) 11.1.0 + (macmini-m1) Intel icc/icpc/ifort version 2021.3.0 202106092021.3.0 20210609 + + macOS Big Sur 11.3.1 Apple clang version 12.0.5 (clang-1205.0.22.9) + Darwin 20.4.0 x86_64 gfortran GNU Fortran (Homebrew GCC 10.2.0_3) 10.2.0 + (bigsur-1) Intel icc/icpc/ifort version 2021.2.0 20210228 + + macOS High Sierra 10.13.6 Apple LLVM version 10.0.0 (clang-1000.10.44.4) + 64-bit gfortran GNU Fortran (GCC) 6.3.0 + (bear) Intel icc/icpc/ifort version 19.0.4.233 20190416 + + macOS Sierra 10.12.6 Apple LLVM version 9.0.0 (clang-900.39.2) + 64-bit gfortran GNU Fortran (GCC) 7.4.0 + (kite) Intel icc/icpc/ifort version 17.0.2 + + Mac OS X El Capitan 10.11.6 Apple clang version 7.3.0 from Xcode 7.3 + 64-bit gfortran GNU Fortran (GCC) 5.2.0 + (osx1011test) Intel icc/icpc/ifort version 16.0.2 + + + Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos6 Version 4.4.7 20120313 + (platypus) Version 4.9.3, 5.3.0, 6.2.0 + MPICH 3.1.4 compiled with GCC 4.9.3 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Windows 10 x64 Visual Studio 2015 w/ Intel C/C++/Fortran 18 (cmake) + Visual Studio 2017 w/ Intel C/C++/Fortran 19 (cmake) + Visual Studio 2019 w/ clang 12.0.0 + with MSVC-like command-line (C/C++ only - cmake) + Visual Studio 2019 w/ Intel Fortran 19 (cmake) + Visual Studio 2019 w/ MSMPI 10.1 (C only - cmake) + + +Known Problems +============== + Setting a variable-length dataset fill value will leak the memory allocated + for the p field of the hvl_t struct. A fix is in progress for this. + HDFFV-10840 + + CMake files do not behave correctly with paths containing spaces. + Do not use spaces in paths because the required escaping for handling spaces + results in very complex and fragile build files. + ADB - 2019/05/07 + + At present, metadata cache images may not be generated by parallel + applications. Parallel applications can read files with metadata cache + images, but since this is a collective operation, a deadlock is possible + if one or more processes do not participate. + + CPP ptable test fails on both VS2017 and VS2019 with Intel compiler, JIRA + issue: HDFFV-10628. This test will pass with VS2015 with Intel compiler. + + The subsetting option in ph5diff currently will fail and should be avoided. + The subsetting option works correctly in serial h5diff. + + Known problems in previous releases can be found in the HISTORY*.txt files + in the HDF5 source. Please report any new problems found to + help@hdfgroup.org. + + +CMake vs. Autotools installations +================================= +While both build systems produce similar results, there are differences. +Each system produces the same set of folders on linux (only CMake works +on standard Windows); bin, include, lib and share. Autotools places the +COPYING and RELEASE.txt file in the root folder, CMake places them in +the share folder. + +The bin folder contains the tools and the build scripts. Additionally, CMake +creates dynamic versions of the tools with the suffix "-shared". Autotools +installs one set of tools depending on the "--enable-shared" configuration +option. + build scripts + ------------- + Autotools: h5c++, h5cc, h5fc + CMake: h5c++, h5cc, h5hlc++, h5hlcc + +The include folder holds the header files and the fortran mod files. CMake +places the fortran mod files into separate shared and static subfolders, +while Autotools places one set of mod files into the include folder. Because +CMake produces a tools library, the header files for tools will appear in +the include folder. + +The lib folder contains the library files, and CMake adds the pkgconfig +subfolder with the hdf5*.pc files used by the bin/build scripts created by +the CMake build. CMake separates the C interface code from the fortran code by +creating C-stub libraries for each Fortran library. In addition, only CMake +installs the tools library. The names of the szip libraries are different +between the build systems. + +The share folder will have the most differences because CMake builds include +a number of CMake specific files for support of CMake's find_package and support +for the HDF5 Examples CMake project. + +The issues with the gif tool are: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 +These CVE issues have not yet been addressed and can be avoided by not building +the gif tool. Disable building the High-Level tools with these options: + autotools: --disable-hltools + cmake: HDF5_BUILD_HL_TOOLS=OFF diff --git a/release_docs/HISTORY-1_8.txt b/release_docs/HISTORY-1_8.txt index da3d849..1e31010 100644 --- a/release_docs/HISTORY-1_8.txt +++ b/release_docs/HISTORY-1_8.txt @@ -1213,7 +1213,7 @@ New Features H5PLreplace - Replace the path at the specified index. H5PLinsert - Insert a plugin path at the specified index, moving other paths after the index. - H5PLremove - Remove the plugin path at the specifed index and + H5PLremove - Remove the plugin path at the specified index and compacting the list. H5PLget - Query the plugin path at the specified index. H5PLsize - Query the size of the current list of plugin paths. @@ -1251,7 +1251,7 @@ New Features // Returns a copy of the creation property list of a datatype. DataType::getCreatePlist() - C API H5Tget_create_plist() - // Opens an object within a location, regarless its type. + // Opens an object within a location, regardless its type. H5Location::openObjId() - C API H5Oopen() H5Location::openObjId() - C API H5Oopen() @@ -1730,7 +1730,7 @@ Bug Fixes since HDF5-1.8.17 files to ensure that the library fails gracefully in these cases: * Writing to a corrupted file that has an object message which is - incorrectly marked as sharable on disk results in a buffer overflow / + incorrectly marked as shareable on disk results in a buffer overflow / invalid write instead of a clean error message. * Decoding data from a corrupted file with a dataset encoded with the @@ -4622,7 +4622,7 @@ Bug Fixes since HDF5-1.8.13 Removed the module name from the package list of Fortran modules because that module was never generated. This was only an issue for Fortran - compliers that are not F2003 compatible. + compilers that are not F2003 compatible. (ADB - 2014/10/16, HDFFV-8932) @@ -4632,7 +4632,7 @@ Bug Fixes since HDF5-1.8.13 change was made which required the renaming of several public symbols in H5Ppublic.h. - The shared libary interface version number ("soname") has been increased + The shared library interface version number ("soname") has been increased on account of these changes. For a full list of the changed symbols, see the interface compatibility report, which is available as a link off of the 'HDF5 Software Changes from Release to Release' document: @@ -5891,7 +5891,7 @@ Known Problems * The data conversion test dt_arith.c fails in "long double" to integer conversion on Ubuntu 11.10 (3.0.0.13 kernel) with GCC 4.6.1 if the library - is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernal + is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernel (3.2.2 on Fedora) doesn't have the problem. Users should lower the optimization level (-O1 or -O0) by defining CFLAGS in the command line of "configure" like: @@ -6746,7 +6746,7 @@ Bug Fixes since HDF5-1.8.10 - Fixed a bug in the core VFD that causes failures when opening files > 2 GB. Fixes HDFFV-8124 and HDFFV-8158 (DER 2013/01/07) - - Fixed a bug where unintialized memory was read during variable-length + - Fixed a bug where uninitialized memory was read during variable-length type conversion. This caused segfaults in netCDF. Fixes HDFFV-8159. (DER 2013/03/30) - Removed the H5Pset/get_dxpl_multi functions from the library. The @@ -7094,7 +7094,7 @@ Known Problems * The data conversion test dt_arith.c fails in "long double" to integer conversion on Ubuntu 11.10 (3.0.0.13 kernel) with GCC 4.6.1 if the library - is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernal + is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernel (3.2.2 on Fedora) doesn't have the problem. Users should lower the optimization level (-O1 or -O0) by defining CFLAGS in the command line of "configure" like: @@ -7718,8 +7718,8 @@ Known Problems (MSC - 2012/04/18) * The data conversion test dt_arith.c fails in "long double" to integer - conversion on Ubuntu 11.10 (3.0.0.13 kernal) with GCC 4.6.1 if the library - is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernal + conversion on Ubuntu 11.10 (3.0.0.13 kernel) with GCC 4.6.1 if the library + is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernel (3.2.2 on Fedora) doesn't have the problem. Users should lower the optimization level (-O1 or -O0) by defining CFLAGS in the command line of "configure" like: @@ -8317,8 +8317,8 @@ Known Problems (MSC - 2012/04/18) * The data conversion test dt_arith.c fails in "long double" to integer - conversion on Ubuntu 11.10 (3.0.0.13 kernal) with GCC 4.6.1 if the library - is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernal + conversion on Ubuntu 11.10 (3.0.0.13 kernel) with GCC 4.6.1 if the library + is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernel (3.2.2 on Fedora) doesn't have the problem. Users should lower the optimization level (-O1 or -O0) by defining CFLAGS in the command line of "configure" like: @@ -8696,7 +8696,7 @@ Bug Fixes since HDF5-1.8.8 ------ - h5ltget_attribute_string_f: The h5ltget_attribute_string_f used to return the C NULL character in the returned character buffer. The returned - charactor buffer now does not return the C NULL character; the buffer + character buffer now does not return the C NULL character; the buffer is blank-padded if needed. (MSB - 2012/03/23) @@ -8799,7 +8799,7 @@ The following platforms and compilers have been tested for this release. Mac OS X Snow Leopard 10.6.8 i686-apple-darwin10-gcc-4.2.1 (GCC) 4.2.1 (gcc) Darwin Kernel Version 10.8.0 i686-apple-darwin10-g++-4.2.1 (GCC) 4.2.1 (g++) Intel 64-bit (Apple Inc. build 5666) (dot 3) - (fred) GNU Fortan (GCC) 4.6.1 (gfortran) + (fred) GNU Fortran (GCC) 4.6.1 (gfortran) Intel C (icc), Fortran (ifort), C++ (icpc) 12.1.0.038 Build 20110811 @@ -8976,8 +8976,8 @@ Known Problems (MSC - 2012/04/18) * The data conversion test dt_arith.c fails in "long double" to integer - conversion on Ubuntu 11.10 (3.0.0.13 kernal) with GCC 4.6.1 if the library - is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernal + conversion on Ubuntu 11.10 (3.0.0.13 kernel) with GCC 4.6.1 if the library + is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernel (3.2.2 on Fedora) doesn't have the problem. Users should lower the optimization level (-O1 or -O0) by defining CFLAGS in the command line of "configure" like: @@ -9394,7 +9394,7 @@ Bug Fixes since HDF5-1.8.7 Tools ----- - h5diff: fixed segfault over non-comparable attribute with different - dimention or rank, along with '-c' option to display details. + dimension or rank, along with '-c' option to display details. (JKM - 2011/10/24 - HDFFV-7770) - Fixed h5diff to display all the comparable objects and attributes regardless of detecting non-comparables. (JKM - 2011/09/16 - HDFFV-7693) @@ -9433,7 +9433,7 @@ Bug Fixes since HDF5-1.8.7 (-d, -p and --use-system-epsilon) are used together. (JKM - 2011/07/07 - HDFFV-7600) - Fixed h5dump so that it displays the first line of each element in correct - position for multiple dimention array types. Before this fix, + position for multiple dimension array types. Before this fix, the first line of each element in an array was displayed after the last line of previous element without moving to the next line (+indentation). @@ -10024,7 +10024,7 @@ Bug Fixes since HDF5-1.8.6 ibm-aix. (AKC - 2011/4/14) - Shared C libraries are no longer disabled on Mac when Fortran is enabled. Shared Fortran libraries are still not supported on Mac, - so configure will disable them by default, but this is overrideable + so configure will disable them by default, but this is overridable with the new --enable-unsupported configure option. The configure summary has been updated to reflect the fact that the shared-ness of the C++/Fortran wrapper libraries may not align with the C library. @@ -10666,7 +10666,7 @@ Bug Fixes since HDF5-1.8.5 - Added a new set of unit tests that are run during 'make check' to verify the behavior of the metadata accumulator. (MAM - 2010/10/15) - Modified library to always cache symbol table information. Libraries - from version 1.6.3 and earler have a bug which causes them to require + from version 1.6.3 and earlier have a bug which causes them to require this information for some operations. (NAF - 2010/09/21 - Bug 1864) - Fixed a bug where the library could generate an assertion/core dump when a file that had been created with H5Pset_libver_bounds(fapl, @@ -11826,7 +11826,7 @@ Bug Fixes since HDF5-1.8.3 created files exhibiting this problem should now be readable. NAF - 2009/10/27 - 1483 - Fixed error in library's internal caching mechanisms which could cause - an assertion failure (and attendent core dump) when encountering an + an assertion failure (and attendant core dump) when encountering an unusually formatted file. (QAK - 2009/10/13) - Fixed incorrect return value for H5Pget_preserve. AKC - 2009/10/08 - 1628 - Fixed an assertion failure that occurred when H5Ocopy was called on a @@ -12350,7 +12350,7 @@ Bug Fixes since HDF5-1.8.2 can only be successfully opened with write access. NAF - 2009/03/23 - 1189 - Removed the long_long #define and replaced all instances with "long long". This caused problems with third party products. All - currently supported compliers support the type. ADB - 2009/03/05 + currently supported compilers support the type. ADB - 2009/03/05 - Fixed various bugs that could prevent the fill value from being written in certain rare cases. NAF - 2009/02/26 - 1469 - Fixed a bug that prevented more than one dataset chunk from being cached @@ -13488,7 +13488,7 @@ Bug Fixes since HDF5-1.8.0 - The h5tset(get)_fields subroutines were missing the parameter to specify a sign position; fixed. (EIP - 2008/05/23) - Many APIs were fixed to work with the 8-byte integers in Fortran vs. - 4-byte integers in C. This change is trasparent to user applications. + 4-byte integers in C. This change is transparent to user applications. C++ API ------ @@ -13882,7 +13882,7 @@ Known Problems exit $RETURN_VALUE - You get the HDF5 make files and test scipts to execute your filter script + You get the HDF5 make files and test scripts to execute your filter script by setting the environment variable "RUNSERIAL" to the full path of the script prior to running configure for parallel builds. Remember to "unsetenv RUNSERIAL" before running configure for a serial build. @@ -14419,7 +14419,7 @@ Known Problems exit $RETURN_VALUE - You get the HDF5 make files and test scipts to execute your filter script + You get the HDF5 make files and test scripts to execute your filter script by setting the environment variable "RUNSERIAL" to the full path of the script prior to running configure for parallel builds. Remember to "unsetenv RUNSERIAL" before running configure for a serial build. diff --git a/release_docs/HISTORY-1_8_0-1_10_0.txt b/release_docs/HISTORY-1_8_0-1_10_0.txt index 616aa24..575d070 100644 --- a/release_docs/HISTORY-1_8_0-1_10_0.txt +++ b/release_docs/HISTORY-1_8_0-1_10_0.txt @@ -211,7 +211,7 @@ New Features Parallel Library: ----------------- - - Add H5Pget_mpio_no_collective_cause() function that retrive reasons + - Add H5Pget_mpio_no_collective_cause() function that retrieve reasons why the collective I/O was broken during read/write IO access. (JKM - 2012/08/30 HDFFV-8143) - Special Collective IO (IO when some processes do not contribute to the @@ -331,10 +331,10 @@ New Features - h5repack: Improved performance for big chunked datasets (size > 128MB) when used with layout (-l) or compression (-f) option. It would perform much better prior to the improvement, - especially for cases that chunk dimentions looks like + especially for cases that chunk dimensions looks like "1024x5x1" (compare to "1x5x1024"). When bigger numbers are toward front and smaller number is toward back in chunk - dimentions. HDFFV-7862 (JKM - 2012/03/01) + dimensions. HDFFV-7862 (JKM - 2012/03/01) - h5dump: Added new option --no-compact-subset. This option will not interpret the '[' character as starting the compact form of subsetting. This is useful when the "h5dump error: unable to @@ -564,7 +564,7 @@ Bug Fixes since HDF5-1.8.0 release - Fixed rare corruption bugs that could occur when using the new object header format. (NAF - 2012/3/15 - HDFFV-7879) - Creating a dataset in a read-only file caused seg fault when the file - is closed. It's fixed. The attemp to create a dataset will fail + is closed. It's fixed. The attempt to create a dataset will fail with the error stack indicating the file is read-only. (SLU - 2012/1/25. Issue 7756) - Fixed a seg fault that could occur when shrinking a dataset with chunks @@ -700,7 +700,7 @@ Bug Fixes since HDF5-1.8.0 release (NAF - 2009/03/23 - 1189) - Removed the long_long #define and replaced all instances with "long long". This caused problems with third party products. All - currently supported compliers support the type. (ADB - 2009/03/05) + currently supported compilers support the type. (ADB - 2009/03/05) - Fixed various bugs that could prevent the fill value from being written in certain rare cases. (NAF - 2009/02/26 - 1469) - Fixed a bug that prevented more than one dataset chunk from being cached @@ -995,7 +995,7 @@ Bug Fixes since HDF5-1.8.0 release - h5ls: fixed segfault when access region reference data in an attribute. HDFFV-7838 (JKM 2011/12/29) - h5diff: fixed segfault over non-comparable attribute with different - dimention or rank, along with '-c' option to display details. + dimension or rank, along with '-c' option to display details. HDFFV-7770 (JKM 2011/10/24) - Fixed h5diff to display all the comparable object and attribute regardless of non-comparables. HDFFV-7693 (JKM 2011/09/16) @@ -1025,7 +1025,7 @@ Bug Fixes since HDF5-1.8.0 release when mutually exclusive options (-d, -p and --use-system-epsilon) are used together. HDFFV-7600 (JKM 2011/07/07) - Fixed h5dump to display the first line of each element into correct - position for multiple dimention array type. + position for multiple dimension array type. Before this fix, the first line of each element in array were displayed after the last line of previous element without moving to the next line (+indentation). @@ -1077,7 +1077,7 @@ Bug Fixes since HDF5-1.8.0 release as source file. (XCAO 2011/3/8) - Fixed h5dump for skipping some values for long array type dataset on Windows. This issue only occurred on Windows due to the different - return behavior from _vsnprintf() funtion. Bug#2161 (JKM 2011/3/3) + return behavior from _vsnprintf() function. Bug#2161 (JKM 2011/3/3) - Fixed h5dump for skipping array indices every certain number when the array type dataset is relatively big. The certain number varies according to the size of array. Bug#2092 (JKM 2011/2/15). @@ -1202,7 +1202,7 @@ Bug Fixes since HDF5-1.8.0 release ------ - Lite: The h5ltget_attribute_string_f used to return the C NULL character in the - returned character buffer. The returned Fortran charactor buffer now does + returned character buffer. The returned Fortran character buffer now does not return the C NULL character. (MSB - 2012/3/23) - Lite: The h5ltget_dataset_info_f function (gets information about a dataset) was not correctly returning the dimension array. (PVN - 2009/3/23) @@ -1262,7 +1262,7 @@ Bug Fixes since HDF5-1.8.0 release The strings command of Mac inspects library files and older versions of strings may not know newer library format, resulting in errors. Fixed by sending the library file as stdin to the strings - coommand to avoid this problem. (AKC - 2013/03/08 HDFFV-8305) + command to avoid this problem. (AKC - 2013/03/08 HDFFV-8305) - Fixed a typo in the ERROR macro in test/testhdf5.h. It segmentation faulted when used before. (AKC - 2013/02/12 HDFFV-8267) @@ -1535,8 +1535,8 @@ Known Problems of GCC. (Issue 8017. SLU - 2012/6/12) * The data conversion test dt_arith.c fails in "long double" to integer - conversion on Ubuntu 11.10 (3.0.0.13 kernal) with GCC 4.6.1 if the library - is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernal + conversion on Ubuntu 11.10 (3.0.0.13 kernel) with GCC 4.6.1 if the library + is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernel (3.2.2 on Fedora) doesn't have the problem. Users should lower down the optimization level (-O1 or -O0) by defining CFLAGS in the command line of "configure" like: @@ -1719,7 +1719,7 @@ Known Problems exit $RETURN_VALUE - You get the HDF make files and test scipts to execute your filter script + You get the HDF make files and test scripts to execute your filter script by setting the environment variable "RUNSERIAL" to the full path of the script prior to running configure for parallel builds. Remember to "unsetenv RUNSERIAL" before running configure for a serial build. diff --git a/release_docs/INSTALL_Cygwin.txt b/release_docs/INSTALL_Cygwin.txt index eebffba..34d3e5a 100644 --- a/release_docs/INSTALL_Cygwin.txt +++ b/release_docs/INSTALL_Cygwin.txt @@ -2,7 +2,7 @@ HDF5 Build and Install Instructions for Cygwin ************************************************************************ -This document is a instruction on how to build, test and install HDF5 libary on +This document is a instruction on how to build, test and install HDF5 library on Cygwin. See detailed information in hdf5/INSTALL. NOTE: hdf5 can be built with CMake, see the INSTALL_CMake.txt file for more guidance. @@ -66,7 +66,7 @@ Preconditions: 2.2.2 Szip The HDF5 library has a predefined compression filter that uses the extended-Rice lossless compression algorithm for chunked - datatsets. For more information on Szip compression, license terms, + datasets. For more information on Szip compression, license terms, and obtaining the Szip source code, see: https://portal.hdfgroup.org/display/HDF5/Szip+Compression+in+HDF+Products diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index c21217f..160675f 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1,4 +1,4 @@ -HDF5 version 1.13.1-1 currently under development +HDF5 version 1.13.2-1 currently under development ================================================================================ @@ -36,7 +36,7 @@ CONTENTS - New Features - Support for new platforms and languages -- Bug Fixes since HDF5-1.12.0 +- Bug Fixes since HDF5-1.13.0 - Platforms Tested - Known Problems - CMake vs. Autotools installations @@ -47,1379 +47,216 @@ New Features Configuration: ------------- - - Added new configure option to support building parallel tools. - See Tools below (autotools - CMake): - --enable-parallel-tools HDF5_BUILD_PARALLEL_TOOLS + - CPack will now generate RPM/DEB packages. - (RAW - 2021/10/25) + Enabled the RPM and DEB CPack generators on linux. In addition to + generating STGZ and TGZ packages, CPack will try to package the + library for RPM and DEB packages. This is the initial attempt and + may change as issues are resolved. - - Added new configure options to enable dimension scales APIs (H5DS*) to - use new object references with the native VOL connector (aka native HDF5 - library). New references are always used for non-native terminal VOL - connectors (e.g., DAOS). + (ADB - 2022/01/27) - Autotools --enable-dimension-scales-with-new-ref - CMake HDF5_DIMENSION_SCALES_NEW_REF=ON + - Added new option to the h5cc scripts produced by CMake. - (EIP - 2021/10/25, HDFFV-11180) + Add -showconfig option to h5cc scripts that cat the + libhdf5-settings to the standard output. - - Refactored the utils folder. + (ADB - 2022/01/25) - Added subfolder test and moved the 'swmr_check_compat_vfd.c file' - from test into utils/test. Deleted the duplicate swmr_check_compat_vfd.c - file in hl/tools/h5watch folder. Also fixed vfd check options. - - (ADB - 2021/10/18) - - - Changed autotools and CMake configurations to derive both - compilation warnings-as-errors and warnings-only-warn configurations - from the same files, 'config/*/*error*'. Removed redundant files - 'config/*/*noerror*'. - - (DCY - 2021/09/29) - - - Added new option to control the build of High-Level tools - that default ON/enabled. - - Add configure options (autotools - CMake): - --enable-hltools HDF5_BUILD_HL_TOOLS - - Disabling this option prevents building the gif tool which - contains the following CVEs: - HDFFV-10592 CVE-2018-17433 - HDFFV-10593 CVE-2018-17436 - HDFFV-11048 CVE-2020-10809 - - (ADB - 2021/09/16, HDFFV-11266) - - - Adds C++ Autotools configuration file for Intel - - * Checks for icpc as the compiler - * Sets std=c++11 - * Copies most non-warning flags from intel-flags - - (DER - 2021/06/02) - - - Adds C++ Autotools configuration file for PGI - - * Checks for pgc++ as the compiler name (was: pgCC) - * Sets -std=c++11 - * Other options basically match new C options (below) - - (DER - 2021/06/02) - - - Updates PGI C options - - * -Minform set to warn (was: inform) to suppress spurious messages - * Sets -gopt -O2 as debug options - * Sets -O4 as 'high optimization' option - * Sets -O0 as 'no optimization' option - * Removes specific settings for PGI 9 and 10 - - (DER - 2021/06/02) - - - A C++11-compliant compiler is now required to build the C++ wrappers - - CMAKE_CXX_STANDARD is now set to 11 when building with CMake and - -std=c++11 is added when building with clang/gcc via the Autotools. - - (DER - 2021/05/27) - - - CMake will now run the shell script tests in test/ by default + - CMake will now run the PowerShell script tests in test/ by default + on Windows. The test directory includes several shell script tests that previously - were not run by CMake. These are now run by default. TEST_SHELL_SCRIPTS - has been set to ON and SH_PROGRAM has been set to bash (some test - scripts use bash-isms). Platforms without bash (e.g., Windows) will - ignore the script tests. - - (DER - 2021/05/23) - - - Removed unused HDF5_ENABLE_HSIZET option from CMake - - This has been unused for some time and has no effect. - - (DER - 2021/05/23) - - - CMake no longer builds the C++ library by default - - HDF5_BUILD_CPP_LIB now defaults to OFF, which is in line with the - Autotools build defaults. - - (DER - 2021/04/20) - - - Removal of pre-VS2015 work-arounds - - HDF5 now requires Visual Studio 2015 or greater, so old work-around - code and definitions have been removed, including: - - * - * snprintf and vsnprintf - * llround, llroundf, lround, lroundf, round, roundf - * strtoll and strtoull - * va_copy - * struct timespec - - (DER - 2021/03/22) - - - Add CMake variable HDF5_LIB_INFIX - - This infix is added to all library names after 'hdf5'. - e.g. the infix '_openmpi' results in the library name 'libhdf5_openmpi.so' - This name is used in packages on debian based systems. - (see https://packages.debian.org/jessie/amd64/libhdf5-openmpi-8/filelist) - - (barcode - 2021/03/22) - - - On macOS, Universal Binaries can now be built, allowing native execution on - both Intel and Apple Silicon (ARM) based Macs. - - To do so, set CMAKE_OSX_ARCHITECTURES="x86_64;arm64" - - (SAM - 2021/02/07, github-311) - - - Added a configure-time option to control certain compiler warnings - diagnostics - - A new configure-time option was added that allows some compiler warnings - diagnostics to have the default operation. This is mainly intended for - library developers and currently only works for gcc 10 and above. The - diagnostics flags apply to C, C++ and Fortran compilers and will appear - in "H5 C Flags", H5 C++ Flags" and H5 Fortran Flags, respectively. They - will NOT be exported to h5cc, etc. - - The default is OFF, which will disable the warnings URL and color attributes - for the warnings output. ON will not add the flags and allow default behavior. - - Autotools: --enable-diags - - CMake: HDF5_ENABLE_BUILD_DIAGS - - (ADB - 2021/02/05, HDFFV-11213) - - - CMake option to build the HDF filter plugins project as an external project - - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. - - The option was refactored to use the CMake FetchContent process. This allows - more control over the filter targets, but required external project command - options to be moved to a CMake include file, HDF5PluginCache.cmake. Also - enabled the filter examples to be used as tests for operation of the - filter plugins. - - (ADB - 2020/12/10, OESS-98) - - - FreeBSD Autotools configuration now defaults to 'cc' and 'c++' compilers - - On FreeBSD, the autotools defaulted to 'gcc' as the C compiler and did - not process C++ options. Since FreeBSD 10, the default compiler has - been clang (via 'cc'). - - The default compilers have been set to 'cc' for C and 'c++' for C++, - which will pick up clang and clang++ respectively on FreeBSD 10+. - Additionally, clang options are now set correctly for both C and C++ - and g++ options will now be set if that compiler is being used (an - omission from the former functionality). - - (DER - 2020/11/28, HDFFV-11193) - - - Fixed POSIX problems when building w/ gcc on Solaris - - When building on Solaris using gcc, the POSIX symbols were not - being set correctly, which could lead to issues like clock_gettime() - not being found. - - The standard is now set to gnu99 when building with gcc on Solaris, - which allows POSIX things to be #defined and linked correctly. This - differs slightly from the gcc norm, where we set the standard to c99 - and manually set POSIX #define symbols. - - (DER - 2020/11/25, HDFFV-11191) - - - Added a configure-time option to consider certain compiler warnings - as errors - - A new configure-time option was added that converts some compiler warnings - to errors. This is mainly intended for library developers and currently - only works for gcc and clang. The warnings that are considered errors - will appear in the generated libhdf5.settings file. These warnings apply - to C and C++ code and will appear in "H5 C Flags" and H5 C++ Flags", - respectively. They will NOT be exported to h5cc, etc. - - The default is OFF. Building with this option may fail when compiling - on operating systems and with compiler versions not commonly used by - the library developers. Compilation may also fail when headers not - under the control of the library developers (e.g., mpi.h, hdfs.h) raise - warnings. - - Autotools: --enable-warnings-as-errors - - CMake: HDF5_ENABLE_WARNINGS_AS_ERRORS - - (DER - 2020/11/23, HDFFV-11189) - - - Autotools and CMake target added to produce doxygen generated documentation - - The default is OFF or disabled. - Autoconf option is '--enable-doxygen' - autotools make target is 'doxygen' and will build all doxygen targets - CMake configure option is 'HDF5_BUILD_DOC'. - CMake target is 'doxygen' for all available doxygen targets - CMake target is 'hdf5lib_doc' for the src subdirectory - - (ADB - 2020/11/03) - - - CMake option to use MSVC naming conventions with MinGW - - HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions - when using a MinGW toolchain - - (xan - 2020/10/30) - - - CMake option to statically link gcc libs with MinGW - - HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ - with the MinGW toolchain - - (xan - 2020/10/30) - - - CMake option to build the HDF filter plugins project as an external project - - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. - - The necessary options are (see the INSTALL_CMake.txt file): - HDF5_ENABLE_PLUGIN_SUPPORT - PLUGIN_TGZ_NAME or PLUGIN_GIT_URL - There are more options necessary for various filters and the plugin project - documents should be referenced. - - (ADB - 2020/09/27, OESS-98) - - - Added CMake option to format source files - - HDF5_ENABLE_FORMATTERS option will enable creation of targets using the - pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder - or tool folder. All sources can be formatted by executing the format target; - make format - - (ADB - 2020/08/24) - - - Add file locking configure and CMake options - - HDF5 1.10.0 introduced a file locking scheme, primarily to help - enforce SWMR setup. Formerly, the only user-level control of the scheme - was via the HDF5_USE_FILE_LOCKING environment variable. - - This change introduces configure-time options that control whether - or not file locking will be used and whether or not the library - ignores errors when locking has been disabled on the file system - (useful on some HPC Lustre installations). - - In both the Autotools and CMake, the settings have the effect of changing - the default property list settings (see the H5Pset/get_file_locking() - entry, below). - - The yes/no/best-effort file locking configure setting has also been - added to the libhdf5.settings file. - - Autotools: - - An --enable-file-locking=(yes|no|best-effort) option has been added. - - yes: Use file locking. - no: Do not use file locking. - best-effort: Use file locking and ignore "disabled" errors. - - CMake: - - Two self-explanatory options have been added: - - HDF5_USE_FILE_LOCKING - HDF5_IGNORE_DISABLED_FILE_LOCKS - - Setting both of these to ON is the equivalent to the Autotools' - best-effort setting. - - NOTE: - The precedence order of the various file locking control mechanisms is: - - 1) HDF5_USE_FILE_LOCKING environment variable (highest) - - 2) H5Pset_file_locking() - - 3) configure/CMake options (which set the property list defaults) - - 4) library defaults (currently best-effort) - - (DER - 2020/07/30, HDFFV-11092) - - - CMake option to link the generated Fortran MOD files into the include - directory. - - The Fortran generation of MOD files by a Fortran compile can produce - different binary files between SHARED and STATIC compiles with different - compilers and/or different platforms. Note that it has been found that - different versions of Fortran compilers will produce incompatible MOD - files. Currently, CMake will locate these MOD files in subfolders of - the include directory and add that path to the Fortran library target - in the CMake config file, which can be used by the CMake find library - process. For other build systems using the binary from a CMake install, - a new CMake configuration can be used to copy the pre-chosen version - of the Fortran MOD files into the install include directory. - - The default will depend on the configuration of - BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: - YES YES Default to SHARED - YES NO Default to STATIC - NO YES Default to SHARED - NO NO Default to SHARED - The defaults can be overridden by setting the config option - HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC - - (ADB - 2020/07/09, HDFFV-11116) - - - CMake option to use AEC (open source SZip) library instead of SZip - - The open source AEC library is a replacement library for SZip. In - order to use it for hdf5 the libaec CMake source was changed to add - "-fPIC" and exclude test files. Autotools does not build the - compression libraries within hdf5 builds. New option USE_LIBAEC is - required to compensate for the different files produced by AEC build. - - (ADB - 2020/04/22, OESS-65) - - - CMake ConfigureChecks.cmake file now uses CHECK_STRUCT_HAS_MEMBER - - Some handcrafted tests in HDFTests.c has been removed and the CMake - CHECK_STRUCT_HAS_MEMBER module has been used. + were not run by CMake on Windows. These are now run by default. + If TEST_SHELL_SCRIPTS is ON and PWSH is found, the PowerShell scripts + will execute. Similar to the bash scripts on unix platforms. - (ADB - 2020/03/24, TRILAB-24) - - - Both build systems use same set of warnings flags - - GNU C, C++ and gfortran warnings flags were moved to files in a config - sub-folder named gnu-warnings. Flags that only are available for a specific - version of the compiler are in files named with that version. - Clang C warnings flags were moved to files in a config sub-folder - named clang-warnings. - Intel C, Fortran warnings flags were moved to files in a config sub-folder - named intel-warnings. - - There are flags in named "error-xxx" files with warnings that may - be promoted to errors. Some source files may still need fixes. - - There are also pairs of files named "developer-xxx" and "no-developer-xxx" - that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the - configure option:--enable-developer-warnings. - - In addition, CMake no longer applies these warnings for examples. - - (ADB - 2020/03/24, TRILAB-192) - - - Added test script for file size compare - - If CMake minimum version is at least 3.14, the fileCompareTest.cmake - script will compare file sizes. - - (ADB - 2020/02/24, HDFFV-11036) - - - Update CMake minimum version to 3.12 - - Updated CMake minimum version to 3.12 and added version checks - for Windows features. - - (ADB - 2020/02/05, TRILABS-142) - - - Fixed CMake include properties for Fortran libraries - - Corrected the library properties for Fortran to use the - correct path for the Fortran module files. - - (ADB - 2020/02/04, HDFFV-11012) - - - Added common warnings files for gnu and intel - - Added warnings files to use one common set of flags - during configure for both autotools and CMake build - systems. The initial implementation only affects a - general set of flags for gnu and intel compilers. - - (ADB - 2020/01/17) - - - Added new options to CMake for control of testing - - Added CMake options (default ON); - HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL - combined with: - HDF5_TEST_TOOLS - HDF5_TEST_EXAMPLES - HDF5_TEST_SWMR - HDF5_TEST_FORTRAN - HDF5_TEST_CPP - HDF5_TEST_JAVA - - (ADB - 2020/01/15, HDFFV-11001) - - - Added Clang sanitizers to CMake for analyzer support if compiler is clang. - - Added CMake code and files to execute the Clang sanitizers if - HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option - is set to one of the following: - Address - Memory - MemoryWithOrigins - Undefined - Thread - Leak - 'Address;Undefined' - - (ADB - 2019/12/12, TRILAB-135) - - - Update CMake for VS2019 support - - CMake added support for VS2019 in version 3.15. Changes to the CMake - generator setting required changes to scripts. Also updated version - references in CMake files as necessary. - - (ADB - 2019/11/18, HDFFV-10962) - - - Update CMake options to match new autotools options - - Add configure options (autotools - CMake): - enable-asserts HDF5_ENABLE_ASSERTS - enable-symbols HDF5_ENABLE_SYMBOLS - enable-profiling HDF5_ENABLE_PROFILING - enable-optimization HDF5_ENABLE_OPTIMIZATION - In addition NDEBUG is no longer forced defined and relies on the CMake - process. - - (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) + (ADB - 2021/11/23) Library: -------- - - Change how the release part of version, in major.minor.release is checked - for compatibility - - The HDF5 library uses a function, H5check_version, to check that - the version defined in the header files, which is used to compile an - application is compatible with the version codified in the library, which - the application loads at runtime. This previously required an exact match - or the library would print a warning, dump the build settings and then - abort or continue. An environment variable controlled the logic. - - Now the function first checks that the library release version, in - major.minor.release, is not older than the version in the headers. - Secondly, if the release version is different, it checks if either - the library version or the header version is in the exception list, in - which case the release part of version, in major.minor.release, must - be exact. An environment variable still controls the logic. - - (ADB - 2021/07/27) - - - gcc warning suppression macros were moved out of H5public.h - - The HDF5 library uses a set of macros to suppress warnings on gcc. - These warnings were originally located in H5public.h so that the - multi VFD (which only uses public headers) could also make use of them - but internal macros should not be publicly exposed like this. - - These macros have now been moved to H5private.h. Pending future multi - VFD refactoring, the macros have been duplicated in H5FDmulti.c to - suppress the format string warnings there. - - (DER - 2021/06/03) - - - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX - - The size_hint value is ultimately stored in a uint32_t struct field, - so specifying a value larger than this on a 64-bit machine can cause - undefined behavior including crashing the system. - - The documentation for this API call was also incorrect, stating that - passing a negative value would cause the library to use a default - value. Instead, passing a "negative" value actually passes a very large - value, which is probably not what the user intends and can cause - crashes on 64-bit systems. - - The Doxygen documentation has been updated and passing values larger - than UINT32_MAX for size_hint will now produce a normal HDF5 error. - - (DER - 2021/04/29, HDFFV-11241) - - - - H5Pset_fapl_log() no longer crashes when passed an invalid fapl ID - - When passed an invalid fapl ID, H5Pset_fapl_log() would usually - segfault when attempting to free an uninitialized pointer in the error - handling code. This behavior is more common in release builds or - when the memory sanitization checks were not selected as a build - option. - - The pointer is now correctly initialized and the API call now - produces a normal HDF5 error when fed an invalid fapl ID. - - (DER - 2021/04/28, HDFFV-11240) - - - Fixes a segfault when H5Pset_mdc_log_options() is called multiple times - - The call incorrectly attempts to free an internal copy of the previous - log location string, which causes a segfault. This only happens - when the call is invoked multiple times on the same property list. - On the first call to a given fapl, the log location is set to NULL so - the segfault does not occur. - - The string is now handled properly and the segfault no longer occurs. - - (DER - 2021/04/27, HDFFV-11239) - - - HSYS_GOTO_ERROR now emits the results of GetLastError() on Windows + - Add a new public function, H5ESget_requests() - HSYS_GOTO_ERROR is an internal macro that is used to produce error - messages when system calls fail. These strings include errno and the - the associated strerror() value, which are not particularly useful - when a Win32 API call fails. + This function allows the user to retrieve request pointers from an event + set. It is intended for use primarily by VOL plug in developers. - On Windows, this macro has been updated to include the result of - GetLastError(). When a system call fails on Windows, usually only - one of errno and GetLastError() will be useful, however we emit both - for the user to parse. The Windows error message is not emitted as - it would be awkward to free the FormatMessage() buffer given the - existing HDF5 error framework. Users will have to look up the error - codes in MSDN. + (NAF - 2022/01/11) - The format string on Windows has been changed from: - "%s, errno = %d, error message = '%s'" - - to: - - "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" - - for those inclined to parse it for error values. - - (DER - 2021/03/21) - - - File locking now works on Windows - - Since version 1.10.0, the HDF5 library has used a file locking scheme - to help enforce one reader at a time accessing an HDF5 file, which can - be helpful when setting up readers and writers to use the single- - writer/multiple-readers (SWMR) access pattern. - - In the past, this was only functional on POSIX systems where flock() or - fcntl() were present. Windows used a no-op stub that always succeeded. - - HDF5 now uses LockFileEx() and UnlockFileEx() to lock the file using the - same scheme as POSIX systems. We lock the entire file when we set up the - locks (by passing DWORDMAX as both size parameters to LockFileEx()). - - (DER - 2021/03/19, HDFFV-10191) - - - H5Epush_ret() now requires a trailing semicolon - - H5Epush_ret() is a function-like macro that has been changed to - contain a `do {} while(0)` loop. Consequently, a trailing semicolon - is now required to end the `while` statement. Previously, a trailing - semi would work, but was not mandatory. This change was made to allow - clang-format to correctly format the source code. - - (SAM - 2021/03/03) - - - Improved performance of H5Sget_select_elem_pointlist - - Modified library to cache the point after the last block of points - retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the - same function to retrieve the next block of points from the list can - proceed immediately without needing to iterate over the point list. - - (NAF - 2021/01/19) - - - Replaced H5E_ATOM with H5E_ID in H5Epubgen.h - - The term "atom" is archaic and not in line with current HDF5 library - terminology, which uses "ID" instead. "Atom" has mostly been purged - from the library internals and this change removes H5E_ATOM from - the H5Epubgen.h (exposed via H5Epublic.h) and replaces it with - H5E_ID. - - (DER - 2020/11/24, HDFFV-11190) - - - Add a new public function H5Ssel_iter_reset - - This function resets a dataspace selection iterator back to an - initial state so that it may be used for iteration once more. - This can be useful when needing to iterate over a selection - multiple times without having to repeatedly create/destroy - a selection iterator for that dataspace selection. - - (JTH - 2020/09/18) - - - Remove HDFS VFD stubs - - The original implementation of the HDFS VFD included non-functional - versions of the following public API calls when the HDFS VFD is - not built as a part of the HDF5 library: - - * H5FD_hdfs_init() - * H5Pget_fapl_hdfs() - * H5Pset_fapl_hdfs() - - They will remain present in HDF5 1.10 and HDF5 1.12 releases - for binary compatibility purposes but have been removed as of 1.14.0. - - Note that this has nothing to do with the real HDFS VFD API calls - that are fully functional when the HDFS VFD is configured and built. - - We simply changed: - - #ifdef LIBHDFS - - #else - - #endif - - to: - - #ifdef LIBHDFS - - #endif - - Which is how the other optional VFDs are handled. - - (DER - 2020/08/27) - - - Add Mirror VFD - - Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote - machine. Must be used in conjunction with the Splitter VFD. - - (JOS - 2020/03/13, TBD) - - - Add Splitter VFD - - Maintain separate R/W and W/O channels for "concurrent" file writes - to two files using a single HDF5 file handle. - - (JOS - 2020/03/13, TBD) - - - Refactored public exposure of haddr_t type in favor of "object tokens" - - To better accommodate HDF5 VOL connectors where "object addresses in a file" - may not make much sense, the following changes were made to the library: - - * Introduced new H5O_token_t "object token" type, which represents a - unique and permanent identifier for referencing an HDF5 object within - a container; these "object tokens" are meant to replace object addresses. - Along with the new type, a new H5Oopen_by_token API call was introduced - to open an object by a token, similar to how object addresses were - previously used with H5Oopen_by_addr. - - * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, - H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t - struct and H5L_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The original functions have been - renamed to version 1 functions and are deprecated in favor of the new version 2 - functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 - types and are now deprecated in favor of their version 2 counterparts. For each of - the functions and types, compatibility macros take place of the original symbols. - - * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, - H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t - struct and H5O_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The version 2 functions are now - deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t - types have been renamed to version 1 types and are now deprecated in favor of their - version 2 counterparts. For each, compatibility macros take place of the original - symbols. - - * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and - H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t - struct, which are used to retrieve the native HDF5 file format-specific information - about an object. This information (such as object header info and B-tree/heap info) - has been removed from the new H5O_info2_t struct so that the more generic - H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native - VOL connectors. - - * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare - two object tokens, convert an object token into a nicely-readable string format and - to convert an object token string back into a real object token, respectively. - - (DER, QAK, JTH - 2020/01/16) - - - Add new public function H5Sselect_adjust. - - This function shifts a dataspace selection by a specified logical offset - within the dataspace extent. This can be useful for VOL developers to - implement chunked datasets. + Parallel Library: + ----------------- + - Several improvements to parallel compression feature, including: - (NAF - 2019/11/18) + * Improved support for collective I/O (for both writes and reads) - - Add new public function H5Sselect_project_intersection. + * Significant reduction of memory usage for the feature as a whole - This function computes the intersection between two dataspace selections - and projects that intersection into a third selection. This can be useful - for VOL developers to implement chunked or virtual datasets. + * Reduction of copying of application data buffers passed to H5Dwrite - (NAF - 2019/11/13, ID-148) + * Addition of support for incremental file space allocation for filtered + datasets created in parallel. Incremental file space allocation is the + default for these types of datasets (early file space allocation is + also still supported), while early file space allocation is still the + default (and only supported allocation time) for unfiltered datasets + created in parallel. Incremental file space allocation should help with + parallel HDF5 applications that wish to use fill values on filtered + datasets, but would typically avoid doing so since dataset creation in + parallel would often take an excessive amount of time. Since these + datasets previously used early file space allocation, HDF5 would + allocate space for and write fill values to every chunk in the dataset + at creation time, leading to noticeable overhead. Instead, with + incremental file space allocation, allocation of file space for chunks + and writing of fill values to those chunks will be delayed until each + individual chunk is initially written to. - - Add new public function H5VLget_file_type. + * Addition of support for HDF5's "don't filter partial edge chunks" flag + (https://portal.hdfgroup.org/display/HDF5/H5P_SET_CHUNK_OPTS) - This function returns a datatype equivalent to the supplied datatype but - with the location set to be in the file. This datatype can then be used - with H5Tconvert to convert data between file and in-memory representation. - This function is intended for use only by VOL connector developers. + * Addition of proper support for HDF5 fill values with the feature - (NAF - 2019/11/08, ID-127) + * Addition of 'H5_HAVE_PARALLEL_FILTERED_WRITES' macro to H5pubconf.h + so HDF5 applications can determine at compile-time whether the feature + is available + * Addition of simple examples (ph5_filtered_writes.c and + ph5_filtered_writes_no_sel.c) under examples directory to demonstrate + usage of the feature - Parallel Library: - ----------------- - - + * Improved coverage of regression testing for the feature + (JTH - 2022/2/23) Fortran Library: ---------------- - - H5Fget_name_f fixed to handle correctly trailing whitespaces and - newly allocated buffers. - - (MSB - 2021/08/30, github-826,972) - - - Add wrappers for H5Pset/get_file_locking() API calls - - h5pget_file_locking_f() - h5pset_file_locking_f() - - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + - - (DER - 2020/07/30, HDFFV-11092) C++ Library: ------------ - - Add wrappers for H5Pset/get_file_locking() API calls - - FileAccPropList::setFileLocking() - FileAccPropList::getFileLocking() - - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. - - (DER - 2020/07/30, HDFFV-11092) + - Java Library: ------------- - - Replaced HDF5AtomException with HDF5IdException - - Since H5E_ATOM changed to H5E_ID in the C library, the Java exception - that wraps the error category was also renamed. Its functionality - remains unchanged aside from the name. - - (See also the HDFFV-11190 note in the C library section) - - (DER - 2020/11/24, HDFFV-11190) - - - Added new H5S functions. - - H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, - H5Sselect_intersect_block, H5Sselect_project_intersection, - H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select - wrapper functions added. - - (ADB - 2020/10/27, HDFFV-10868) - - - Add wrappers for H5Pset/get_file_locking() API calls - - H5Pset_file_locking() - H5Pget_use_file_locking() - H5Pget_ignore_disabled_file_locking() - - Unlike the C++ and Fortran wrappers, there are separate getters for the - two file locking settings, each of which returns a boolean value. - - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. - - (DER - 2020/07/30, HDFFV-11092) - - - Added ability to test java library with VOLs. - - Created a new CMake script that combines the java and vol test scripts. - - (ADB - 2020/02/03, HDFFV-10996) - - - Tests fail for non-English locales. - - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. - - (ADB - 2020/01/09, HDFFV-10995) + - Tools: ------ - - Refactored the perform tools and removed depends on test library. - - Moved the perf and h5perf tools from tools/test/perform to - tools/src/h5perf so that they can be installed. This required - that the test library dependency be removed by copying the - needed functions from h5test.c. - The standalone scripts and other perform tools remain in the - tools/test/perform folder. - - (ADB - 2021/08/10) - - - Removed partial long exceptions - - Some of the tools accepted shortened versions of the long options - (ex: --datas instead of --dataset). These were implemented inconsistently, - are difficult to maintian, and occasionally block useful long option - names. These partial long options have been removed from all the tools. - - (DER - 2021/08/03) - - - h5repack added help text for user-defined filters. - - Added help text line that states the valid values of the filter flag - for user-defined filters; - filter_flag: 1 is OPTIONAL or 0 is MANDATORY - - (ADB - 2021/01/14, HDFFV-11099) - - - Added h5delete tool - - Deleting HDF5 storage when using the VOL can be tricky when the VOL - does not create files. The h5delete tool is a simple wrapper around - the H5Fdelete() API call that uses the VOL specified in the - HDF5_VOL_CONNECTOR environment variable to delete a "file". If - the call to H5Fdelete() fails, the tool will attempt to use - the POSIX remove(3) call to remove the file. - - Note that the HDF5 library does currently have support for - H5Fdelete() in the native VOL connector. - - (DER - 2020/12/16) - - - h5repack added options to control how external links are handled. - - Currently h5repack preserves external links and cannot copy and merge - data from the external files. Two options, merge and prune, were added to - control how to merge data from an external link into the resulting file. - --merge Follow external soft link recursively and merge data. - --prune Do not follow external soft links and remove link. - --merge --prune Follow external link, merge data and remove dangling link. - - (ADB - 2020/08/05, HDFFV-9984) - - - h5repack was fixed to repack the reference attributes properly. - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. - - (KY -2020/02/07, HDFFV-11014) - - - h5diff was updated to use the new reference APIs. - - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. - - (ADB - 2019/12/19, HDFFV-10980) - - - h5dump and h5ls were updated to use the new reference APIs. - - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. - - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + - High-Level APIs: ---------------- - - added set/get for unsigned long long attributes - - The attribute writing high-level API has been expanded to include - public set/get functions for ULL attributes, analogously to the - existing set/get for other types. + - - (AF - 2021/09/08) C Packet Table API: ------------------- - + Internal header file: --------------------- - + Documentation: -------------- - + Support for new platforms, languages and compilers ================================================== - + Bug Fixes since HDF5-1.12.0 release =================================== Library ------- - - Fixed an H5Pget_filter_by_id1/2() assert w/ out of range filter IDs - - Both H5Pget_filter_by_id1 and 2 did not range check the filter ID, which - could trip as assert in debug versions of the library. The library now - returns a normal HDF5 error when the filter ID is out of range. - - (DER - 2021/11/23, HDFFV-11286) - - - Fixed an issue with collective metadata reads being permanently disabled - after a dataset chunk lookup operation. This would usually cause a - mismatched MPI_Bcast and MPI_ERR_TRUNCATE issue in the library for - simple cases of H5Dcreate() -> H5Dwrite() -> H5Dcreate(). - - (JTH - 2021/11/08, HDFFV-11090) - - - Fixed cross platform incompatibility of references within variable length - types - - Reference types within variable length types previously could not be - read on a platform with different endianness from where they were - written. Fixed so cross platform portability is restored. - - (NAF - 2021/09/30) - - - Detection of simple data transform function "x" - - In the case of the simple data transform function "x" the (parallel) - library recognizes this is the same as not applying this data transform - function. This improves the I/O performance. In the case of the parallel - library, it also avoids breaking to independent I/O, which makes it - possible to apply a filter when writing or reading data to or from - the HDF5 file. - - (JWSB - 2021/09/13) - - - Fixed an invalid read and memory leak when parsing corrupt file space - info messages - - When the corrupt file from CVE-2020-10810 was parsed by the library, - the code that imports the version 0 file space info object header - message to the version 1 struct could read past the buffer read from - the disk, causing an invalid memory read. Not catching this error would - cause downstream errors that eventually resulted in a previously - allocated buffer to be unfreed when the library shut down. In builds - where the free lists are in use, this could result in an infinite loop - and SIGABRT when the library shuts down. - - We now track the buffer size and raise an error on attempts to read - past the end of it. - - (DER - 2021/08/12, HDFFV-11053) - - - - Fixed CVE-2018-14460 - - The tool h5repack produced a segfault when the rank in dataspace - message was corrupted, causing invalid read while decoding the - dimension sizes. - - The problem was fixed by ensuring that decoding the dimension sizes - and max values will not go beyong the end of the buffer. - - (BMR - 2021/05/12, HDFFV-11223) - - - Fixed CVE-2018-11206 - - The tool h5dump produced a segfault when the size of a fill value - message was corrupted and caused a buffer overflow. - - The problem was fixed by verifying the fill value's size - against the buffer size before attempting to access the buffer. - - (BMR - 2021/03/15, HDFFV-10480) - - - Fixed CVE-2018-14033 (same issue as CVE-2020-10811) - - The tool h5dump produced a segfault when the storage size message - was corrupted and caused a buffer overflow. - - The problem was fixed by verifying the storage size against the - buffer size before attempting to access the buffer. - - (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) - - - Remove underscores on header file guards - - Header file guards used a variety of underscores at the beginning of the define. - - Removed all leading (some trailing) underscores from header file guards. - - (ADB - 2021/03/03, #361) - - - Fixed a segmentation fault - - A segmentation fault occurred with a Mathworks corrupted file. - - A detection of accessing a null pointer was added to prevent the problem. - - (BMR - 2021/02/19, HDFFV-11150) - - - Fixed issue with MPI communicator and info object not being - copied into new FAPL retrieved from H5F_get_access_plist - - Added logic to copy the MPI communicator and info object into - the output FAPL. MPI communicator is retrieved from the VFD, while - the MPI info object is retrieved from the file's original FAPL. - - (JTH - 2021/02/15, HDFFV-11109) - - - Fixed problems with vlens and refs inside compound using - H5VLget_file_type() - - Modified library to properly ref count H5VL_object_t structs and only - consider file vlen and reference types to be equal if their files are - the same. - - (NAF - 2021/01/22) - - - Fixed CVE-2018-17432 - - The tool h5repack produced a segfault on a corrupted file which had - invalid rank for scalar or NULL datatype. - - The problem was fixed by modifying the dataspace encode and decode - functions to detect and report invalid rank. h5repack now fails - with an error message for the corrupted file. - - (BMR - 2020/10/26, HDFFV-10590) - - - Creation of dataset with optional filter - - When the combination of type, space, etc doesn't work for filter - and the filter is optional, it was supposed to be skipped but it was - not skipped and the creation failed. - - Allowed the creation of the dataset in such a situation. - - (BMR - 2020/08/13, HDFFV-10933) - - - Explicitly declared dlopen to use RTLD_LOCAL - - dlopen documentation states that if neither RTLD_GLOBAL nor - RTLD_LOCAL are specified, then the default behavior is unspecified. - The default on linux is usually RTLD_LOCAL while macos will default - to RTLD_GLOBAL. - - (ADB - 2020/08/12, HDFFV-11127) - - - H5Sset_extent_none() sets the dataspace class to H5S_NO_CLASS which - causes asserts/errors when passed to other dataspace API calls. - - H5S_NO_CLASS is an internal class value that should not have been - exposed via a public API call. - - In debug builds of the library, this can cause assert() function to - trip. In non-debug builds, it will produce normal library errors. - - The new library behavior is for H5Sset_extent_none() to convert - the dataspace into one of type H5S_NULL, which is better handled - by the library and easier for developers to reason about. - - (DER - 2020/07/27, HDFFV-11027) - - - Fixed issues CVE-2018-13870 and CVE-2018-13869 - - When a buffer overflow occurred because a name length was corrupted - and became very large, h5dump crashed on memory access violation. - - A check for reading pass the end of the buffer was added to multiple - locations to prevent the crashes and h5dump now simply fails with an - error message when this error condition occurs. - - (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) - - - Fixed the segmentation fault when reading attributes with multiple threads - - It was reported that the reading of attributes with variable length string - datatype will crash with segmentation fault particularly when the number of - threads is high (>16 threads). The problem was due to the file pointer that - was set in the variable length string datatype for the attribute. That file - pointer was already closed when the attribute was accessed. - - The problem was fixed by setting the file pointer to the current opened file pointer - when the attribute was accessed. Similar patch up was done before when reading - dataset with variable length string datatype. - - (VC - 2020/07/13, HDFFV-11080) - - - Fixed CVE-2020-10810 - - The tool h5clear produced a segfault during an error recovery in - the superblock decoding. An internal pointer was reset to prevent - further accessing when it is not assigned with a value. - - (BMR - 2020/06/29, HDFFV-11053) - - - Fixed CVE-2018-17435 - - The tool h52gif produced a segfault when the size of an attribute - message was corrupted and caused a buffer overflow. - - The problem was fixed by verifying the attribute message's size - against the buffer size before accessing the buffer. h52gif was - also fixed to display the failure instead of silently exiting - after the segfault was eliminated. - - (BMR - 2020/06/19, HDFFV-10591) + - Fixed a metadata cache bug when resizing a pinned/protected cache entry + + When resizing a pinned/protected cache entry, the metadata + cache code previously would wait until after resizing the + entry to attempt to log the newly-dirtied entry. This would + cause H5C_resize_entry to mark the entry as dirty and make + H5AC_resize_entry think that it doesn't need to add the + newly-dirtied entry to the dirty entries skiplist. + + Thus, a subsequent H5AC__log_moved_entry would think it + needs to allocate a new entry for insertion into the dirty + entry skip list, since the entry doesn't exist on that list. + This causes an assertion failure, as the code to allocate a + new entry assumes that the entry is not dirty. + + (JRM - 2022/02/28) + + - Issue #1436 identified a problem with the H5_VERS_RELEASE check in the + H5check_version function. + + Investigating the original fix, #812, we discovered some inconsistencies + with a new block added to check H5_VERS_RELEASE for incompatibilities. + This new block was not using the new warning text dealing with the + H5_VERS_RELEASE check and would cause the warning to be duplicated. + + By removing the H5_VERS_RELEASE argument in the first check for + H5_VERS_MAJOR and H5_VERS_MINOR, the second check would only check + the H5_VERS_RELEASE for incompatible release versions. This adheres + to the statement that except for the develop branch, all release versions + in a major.minor maintenance branch should be compatible. The prerequisite + is that an application will not use any APIs not present in all release versions. + + (ADB - 2022/02/24, #1438) + + - Unified handling of collective metadata reads to correctly fix old bugs + + Due to MPI-related issues occurring in HDF5 from mismanagement of the + status of collective metadata reads, they were forced to be disabled + during chunked dataset raw data I/O in the HDF5 1.10.5 release. This + wouldn't generally have affected application performance because HDF5 + already disables collective metadata reads during chunk lookup, since + it is generally unlikely that the same chunks will be read by all MPI + ranks in the I/O operation. However, this was only a partial solution + that wasn't granular enough. + + This change now unifies the handling of the file-global flag and the + API context-level flag for collective metadata reads in order to + simplify querying of the true status of collective metadata reads. Thus, + collective metadata reads are once again enabled for chunked dataset + raw data I/O, but manually controlled at places where some processing + occurs on MPI rank 0 only and would cause issues when collective + metadata reads are enabled. + + (JTH - 2021/11/16, HDFFV-10501/HDFFV-10562) + + - Fixed several potential MPI deadlocks in library failure conditions + + In the parallel library, there were several places where MPI rank 0 + could end up skipping past collective MPI operations when some failure + occurs in rank 0-specific processing. This would lead to deadlocks + where rank 0 completes an operation while other ranks wait in the + collective operation. These places have been rewritten to have rank 0 + push an error and try to cleanup after the failure, then continue to + participate in the collective operation to the best of its ability. + + (JTH - 2021/11/09) Java Library ------------ - - JNI utility function does not handle new references. - - The JNI utility function for converting reference data to string did - not use the new APIs. In addition to fixing that function, added new - java tests for using the new APIs. - - (ADB - 2021/02/16, HDFFV-11212) - - - The H5FArray.java class, in which virtually the entire execution time - is spent using the HDFNativeData method that converts from an array - of bytes to an array of the destination Java type. - - 1. Convert the entire byte array into a 1-d array of the desired type, - rather than performing 1 conversion per row; - 2. Use the Java Arrays method copyOfRange to grab the section of the - array from (1) that is desired to be inserted into the destination array. - - (PGT,ADB - 2020/12/13, HDFFV-10865) - - - Added ability to test java library with VOLs. - - Created a new CMake script that combines the java and vol test scripts. - - (ADB - 2020/02/03, HDFFV-10996) - - - Tests fail for non-English locales. - - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. - - (ADB - 2020/01/09, HDFFV-10995) + - Configuration ------------- - - Corrected path searched by CMake find_package command - - The install path for cmake find_package files had been changed to use - "share/cmake" - for all platforms. However the trailing "hdf5" directory was not removed. - This "hdf5" additional directory has been removed. - - (ADB - 2021/09/27) - - - Corrected pkg-config compile script - - It was discovered that the position of the "$@" argument for the command - in the compile script may fail on some platforms and configurations. The - position of the "$@"command argument was moved before the pkg-config sub command. - - (ADB - 2021/08/30) - - - Fixed CMake C++ compiler flags - - A recent refactoring of the C++ configure files accidently removed the - file that executed the enable_language command for C++ needed by the - HDFCXXCompilerFlags.cmake file. Also updated the intel warnings files, - including adding support for windows platforms. - - (ADB - 2021/08/10) - - - Better support for libaec (open-source Szip library) in CMake - - Implemented better support for libaec 1.0.5 (or later) library. This version - of libaec contains improvements for better integration with HDF5. Furthermore, - the variable USE_LIBAEC_STATIC has been introduced to allow to make use of - static version of libaec library. Use libaec_DIR or libaec_ROOT to set - the location in which libaec can be found. - - Be aware, the Szip library of libaec 1.0.4 depends on another library within - libaec library. This dependency is not specified in the current CMake - configuration which means that one can not use the static Szip library of - libaec 1.0.4 when building HDF5. This has been resolved in libaec 1.0.5. - - (JWSB - 2021/06/22) - - - Refactor CMake configure for Fortran - - The Fortran configure tests for KINDs reused a single output file that was - read to form the Integer and Real Kinds defines. However, if config was run - more then once, the CMake completed variable prevented the tests from executing - again and the last value saved in the file was used to create the define. - Creating separate files for each KIND solved the issue. - - In addition the test for H5_PAC_C_MAX_REAL_PRECISION was not pulling in - defines for proper operation and did not define H5_PAC_C_MAX_REAL_PRECISION - correctly for a zero value. This was fixed by supplying the required defines. - In addition it was moved from the Fortran specific HDF5UseFortran.camke file - to the C centric ConfigureChecks.cmake file. - - (ADB - 2021/06/03) - - - Move emscripten flag to compile flags - - The emscripten flag, -O0, was removed from target_link_libraries command - to the correct target_compile_options command. - - (ADB - 2021/04/26 HDFFV-11083) - - - Remove arbitrary warning flag groups from CMake builds - - The arbitrary groups were created to reduce the quantity of warnings being - reported that overwhelmed testing report systems. Considerable work has - been accomplished to reduce the warning count and these arbitrary groups - are no longer needed. - Also the default for all warnings, HDF5_ENABLE_ALL_WARNINGS, is now ON. - - Visual Studio warnings C4100, C4706, and C4127 have been moved to - developer warnings, HDF5_ENABLE_DEV_WARNINGS, and are disabled for normal builds. - - (ADB - 2021/03/22, HDFFV-11228) - - - Reclassify CMake messages, to allow new modes and --log-level option - - CMake message commands have a mode argument. By default, STATUS mode - was chosen for any non-error message. CMake version 3.15 added additional - modes, NOTICE, VERBOSE, DEBUG and TRACE. All message commands with a mode - of STATUS were reviewed and most were reclassified as VERBOSE. The new - mode was protected by a check for a CMake version of at least 3.15. If CMake - version 3.17 or above is used, the user can use the command line option - of "--log-level" to further restrict which message commands are displayed. - - (ADB - 2021/01/11, HDFFV-11144) - - - Fixes Autotools determination of the stat struct having an st_blocks field - - A missing parenthesis in an autoconf macro prevented building the test - code used to determine if the stat struct contains the st_blocks field. - Now that the test functions correctly, the H5_HAVE_STAT_ST_BLOCKS #define - found in H5pubconf.h will be defined correctly on both the Autotools and - CMake. This #define is only used in the tests and does not affect the - HDF5 C library. - - (DER - 2021/01/07, HDFFV-11201) - - - Add missing ENV variable line to hdfoptions.cmake file - - Using the build options to use system SZIP/ZLIB libraries need to also - specify the library root directory. Setting the {library}_ROOT ENV - variable was added to the hdfoptions.cmake file. - - (ADB - 2020/10/19 HDFFV-11108) + - Tools ----- - - Changed how h5dump and h5ls identify long double. - - Long double support is not consistent across platforms. Tools will always - identify long double as 128-bit [little/big]-endian float nn-bit precision. - New test file created for datasets with attributes for float, double and - long double. In addition any unknown integer or float datatype will now - also show the number of bits for precision. - These files are also used in the java tests. - - (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) - - - Fixed tools argument parsing. - - Tools parsing used the length of the option from the long array to match - the option from the command line. This incorrectly matched a shorter long - name option that happened to be a subset of another long option. - Changed to match whole names. - - (ADB - 2021/01/19, HDFFV-11106) - - - The tools library was updated by standardizing the error stack process. - - General sequence is: - h5tools_setprogname(PROGRAMNAME); - h5tools_setstatus(EXIT_SUCCESS); - h5tools_init(); - ... process the command-line (check for error-stack enable) ... - h5tools_error_report(); - ... (do work) ... - h5diff_exit(ret); - - (ADB - 2020/07/20, HDFFV-11066) - - - h5diff fixed a command line parsing error. - - h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. - The macro H5_DBL_ABS_EQUAL was removed and a direct value comparision was used. - - (ADB - 2020/07/20, HDFFV-10897) - - - h5diff added a command line option to ignore attributes. - - h5diff would ignore all objects with a supplied path if the exclude-path argument is used. - Adding the exclude-attribute argument will only exclude attributes, with the supplied path, - from comparison. - - (ADB - 2020/07/20, HDFFV-5935) - - - h5diff added another level to the verbose argument to print filenames. - - Added verbose level 3 that is level 2 plus the filenames. The levels are: - 0 : Identical to '-v' or '--verbose' - 1 : All level 0 information plus one-line attribute status summary - 2 : All level 1 information plus extended attribute status report - 3 : All level 2 information plus file names - - (ADB - 2020/07/20, HDFFV-1005) - - - h5repack was fixed to repack the reference attributes properly. - - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. - - (KY -2020/02/10, HDFFV-11014) - - - h5diff was updated to use the new reference APIs. - - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. - - (ADB - 2019/12/19, HDFFV-10980) - - - h5dump and h5ls were updated to use the new reference APIs. - - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. - - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + - Performance @@ -1429,18 +266,12 @@ Bug Fixes since HDF5-1.12.0 release Fortran API ----------- - - Corrected INTERFACE INTENT(IN) to INTENT(OUT) for buf_size in h5fget_file_image_f. - - (MSB - 2020/02/18, HDFFV-11029) + - High-Level Library ------------------ - - Fixed HL_test_packet, test for packet table vlen of vlen. - - Incorrect length assignment. - - (ADB - 2021/10/14) + - Fortran High-Level APIs @@ -1460,23 +291,7 @@ Bug Fixes since HDF5-1.12.0 release C++ APIs -------- - - Added DataSet::operator= - - Some compilers complain if the copy constructor is given explicitly - but the assignment operator is implicitly set to default. - - (2021/05/19) - - - Testing - ------- - - Stopped java/test/junit.sh.in installing libs for testing under ${prefix} - - Lib files needed are now copied to a subdirectory in the java/test - directory, and on Macs the loader path for libhdf5.xxxs.so is changed - in the temporary copy of libhdf5_java.dylib. - - (LRK, 2020/07/02, HDFFV-11063) + - Platforms Tested diff --git a/release_docs/USING_CMake_Examples.txt b/release_docs/USING_CMake_Examples.txt index a12a952..920705b 100644 --- a/release_docs/USING_CMake_Examples.txt +++ b/release_docs/USING_CMake_Examples.txt @@ -20,15 +20,14 @@ Notes: This short instruction is written for users who want to quickly I. Preconditions ======================================================================== - 1. We suggest you obtain the latest CMake for windows from the Kitware + 1. We suggest you obtain the latest CMake for your platform from the Kitware web site. The HDF5 1.13.x product requires a minimum CMake version of 3.12. If you are using VS2019, the minimum version is 3.15. 2. You have installed the HDF5 library built with CMake, by executing the HDF Install Utility (the *.msi file in the binary package for - Windows or the *.sh on Linux). If you are using a Windows platform, - you can obtain a pre-built Windows binary from The HDF Group's website - at www.hdfgroup.org. + Windows or the *.sh on Linux). You can obtain pre-built binaries + from The HDF Group's website at www.hdfgroup.org. diff --git a/release_docs/USING_HDF5_CMake.txt b/release_docs/USING_HDF5_CMake.txt index 792c719..abe1906 100644 --- a/release_docs/USING_HDF5_CMake.txt +++ b/release_docs/USING_HDF5_CMake.txt @@ -29,27 +29,30 @@ Notes: This short instruction is written for users who want to quickly of using a ctest script for building and testing. See INSTALL_CMake.txt for more information. + 4. See https://cmake.org/cmake/help/latest/command/find_package.html + for more information on the CMake "Config Mode Search Procedure". ======================================================================== I. Preconditions ======================================================================== - 1. We suggest you obtain the latest CMake for windows from the Kitware + 1. We suggest you obtain the latest CMake for your platform from the Kitware web site. The HDF5 1.13.x product requires a minimum CMake version - of 3.12. + of 3.12. If you are using VS2019, the minimum version is 3.15. 2. You have installed the HDF5 library built with CMake, by executing the HDF Install Utility (the *.msi file in the binary package for - Windows). You can obtain pre-built binaries from The HDF Group's website at - www.hdfgroup.org. + Windows or the *.sh on Linux). You can obtain pre-built binaries + from The HDF Group's website at www.hdfgroup.org. - 3. Set the environment variable HDF5_DIR to the installed location of - the config files for HDF5. + 3. Set the HDF5_ROOT CMake variable, -DHDF5_ROOT= + or environment variable, set(ENV{HDF5_ROOT} "") + to the installed location of HDF5. On Windows: - HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.13.x/cmake + HDF5_ROOT=C:/Program Files/HDF_Group/HDF5/1.13.x/ On unix: - HDF5_DIR=/HDF_Group/HDF5/1.13.x/cmake + HDF5_ROOT=/HDF_Group/HDF5/1.13.x/ If you are using shared libraries, you may need to add to the path environment variable. Set the path environment variable to the diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 33d31fb..dcb74a9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -241,6 +241,7 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDmpi.c ${HDF5_SRC_DIR}/H5FDmpio.c ${HDF5_SRC_DIR}/H5FDmulti.c + ${HDF5_SRC_DIR}/H5FDperform.c ${HDF5_SRC_DIR}/H5FDros3.c ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.c diff --git a/src/H5.c b/src/H5.c index 69c81ea..1585dd3 100644 --- a/src/H5.c +++ b/src/H5.c @@ -29,9 +29,12 @@ #include "H5Lprivate.h" /* Links */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5PLprivate.h" /* Plugins */ #include "H5SLprivate.h" /* Skip lists */ #include "H5Tprivate.h" /* Datatypes */ +#include "H5FDsec2.h" /* for H5FD_sec2_init() */ + /****************/ /* Local Macros */ /****************/ @@ -63,16 +66,13 @@ static int H5__mpi_delete_cb(MPI_Comm comm, int keyval, void *attr_val, int *fla /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ -/* Library incompatible release versions */ -const unsigned VERS_RELEASE_EXCEPTIONS[] = {0}; -const unsigned VERS_RELEASE_EXCEPTIONS_SIZE = 0; +/* Library incompatible release versions, develop releases are incompatible by design */ +const unsigned VERS_RELEASE_EXCEPTIONS[] = {0, 1, 2}; +const unsigned VERS_RELEASE_EXCEPTIONS_SIZE = 3; /* statically initialize block for pthread_once call used in initializing */ /* the first global mutex */ @@ -101,32 +101,30 @@ static H5_atclose_node_t *H5_atclose_head = NULL; /* Declare a free list to manage the H5_atclose_node_t struct */ H5FL_DEFINE_STATIC(H5_atclose_node_t); -/*-------------------------------------------------------------------------- -NAME - H5__init_package -- Initialize interface-specific information -USAGE - herr_t H5__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. ---------------------------------------------------------------------------*/ -herr_t -H5__init_package(void) +/*------------------------------------------------------------------------- + * Function: H5_default_vfd_init + * + * Purpose: Initialize the default VFD. + * + * Return: Success: non-negative + * Failure: negative + *------------------------------------------------------------------------- + */ +static herr_t +H5_default_vfd_init(void) { - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT - - /* Run the library initialization routine, if it hasn't already ran */ - if (!H5_INIT_GLOBAL && !H5_TERM_GLOBAL) { - if (H5_init_library() < 0) - HGOTO_ERROR(H5E_LIB, H5E_CANTINIT, FAIL, "unable to initialize library") - } /* end if */ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI(FAIL) + /* Load the hid_t for the default VFD for the side effect + * it has of initializing the default VFD. + */ + if (H5FD_sec2_init() == H5I_INVALID_HID) { + HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to load default VFD ID") + } done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5__init_package() */ +} /*-------------------------------------------------------------------------- * NAME @@ -145,15 +143,20 @@ done: herr_t H5_init_library(void) { + size_t i; herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI(FAIL) + + /* Run the library initialization routine, if it hasn't already run */ + if (H5_INIT_GLOBAL || H5_TERM_GLOBAL) + HGOTO_DONE(SUCCEED) + /* Set the 'library initialized' flag as early as possible, to avoid * possible re-entrancy. */ H5_INIT_GLOBAL = TRUE; - FUNC_ENTER_NOAPI(FAIL) - #ifdef H5_HAVE_PARALLEL { int mpi_initialized; @@ -257,28 +260,33 @@ H5_init_library(void) * The dataspace interface needs to be initialized so that future IDs for * dataspaces work. */ - if (H5E_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize error interface") - if (H5VL_init_phase1() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize vol interface") - if (H5P_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize property list interface") - if (H5T_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize datatype interface") - if (H5D_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize dataset interface") - if (H5AC_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize metadata caching interface") - if (H5L_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize link interface") - if (H5FS_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize FS interface") - if (H5S_init() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize dataspace interface") - + /* clang-format off */ + struct { + herr_t (*func)(void); + const char *descr; + } initializer[] = { + {H5E_init, "error"} + , {H5VL_init_phase1, "VOL"} + , {H5SL_init, "skip lists"} + , {H5FD_init, "VFD"} + , {H5_default_vfd_init, "default VFD"} + , {H5P_init_phase1, "property list"} + , {H5AC_init, "metadata caching"} + , {H5L_init, "link"} + , {H5S_init, "dataspace"} + , {H5PL_init, "plugins"} /* Finish initializing interfaces that depend on the interfaces above */ - if (H5VL_init_phase2() < 0) - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "unable to initialize vol interface") + , {H5P_init_phase2, "property list"} + , {H5VL_init_phase2, "VOL"} + }; + + for (i = 0; i < NELMTS(initializer); i++) { + if (initializer[i].func() < 0) { + HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, + "unable to initialize %s interface", initializer[i].descr) + } + } + /* clang-format on */ /* Debugging? */ H5__debug_mask("-all"); @@ -302,9 +310,11 @@ done: void H5_term_library(void) { - int pending, ntries = 0, n; - size_t at = 0; - char loop[1024]; + int pending, ntries = 0; + char loop[1024], *next = loop; + size_t i; + size_t nleft = sizeof(loop); + int nprinted; H5E_auto2_t func; #ifdef H5_HAVE_THREADSAFE @@ -348,108 +358,137 @@ H5_term_library(void) H5_atclose_head = NULL; } /* end if */ + /* clang-format off */ + /* * Terminate each interface. The termination functions return a positive * value if they do something that might affect some other interface in a * way that would necessitate some cleanup work in the other interface. */ -#define DOWN(F) \ - (((n = H5##F##_term_package()) && (at + 8) < sizeof loop) \ - ? (HDsprintf(loop + at, "%s%s", (at ? "," : ""), #F), at += HDstrlen(loop + at), n) \ - : ((n > 0 && (at + 5) < sizeof loop) ? (HDsprintf(loop + at, "..."), at += HDstrlen(loop + at), n) \ - : n)) - do { - pending = 0; - - /* Try to organize these so the "higher" level components get shut - * down before "lower" level components that they might rely on. -QAK - */ +#define TERMINATOR(module, wait) { \ + .func = H5##module##_term_package \ + , .name = #module \ + , .completed = false \ + , .await_prior = wait \ + } + /* + * Termination is ordered by the `terminator` table so the "higher" level + * packages are shut down before "lower" level packages that they + * rely on: + */ + struct { + int (*func)(void); /* function to terminate the module; returns 0 + * on success, >0 if termination was not + * completed and we should try to terminate + * some dependent modules, first. + */ + const char *name; /* name of the module */ + hbool_t completed; /* true iff this terminator was already + * completed + */ + const hbool_t await_prior; /* true iff all prior terminators in the + * list must complete before this + * terminator is attempted + */ + } terminator[] = { /* Close the event sets first, so that all asynchronous operations - * complete before anything else attempts to shut down. + * complete before anything else attempts to shut down. */ - pending += DOWN(ES); - - /* Close down the user-facing interfaces, after the event sets */ - if (pending == 0) { - /* Close the interfaces dependent on others */ - pending += DOWN(L); - - /* Close the "top" of various interfaces (IDs, etc) but don't shut - * down the whole interface yet, so that the object header messages - * get serialized correctly for entries in the metadata cache and the - * symbol table entry in the superblock gets serialized correctly, etc. - * all of which is performed in the 'F' shutdown. - */ - pending += DOWN(A_top); - pending += DOWN(D_top); - pending += DOWN(G_top); - pending += DOWN(M_top); - pending += DOWN(R_top); - pending += DOWN(S_top); - pending += DOWN(T_top); - } /* end if */ - + TERMINATOR(ES, false) + /* Do not attempt to close down package L until after event sets + * have finished closing down. + */ + , TERMINATOR(L, true) + /* Close the "top" of various interfaces (IDs, etc) but don't shut + * down the whole interface yet, so that the object header messages + * get serialized correctly for entries in the metadata cache and the + * symbol table entry in the superblock gets serialized correctly, etc. + * all of which is performed in the 'F' shutdown. + * + * The tops of packages A, D, G, M, S, T do not need to wait for L + * or previous packages to finish closing down. + */ + , TERMINATOR(A_top, false) + , TERMINATOR(D_top, false) + , TERMINATOR(G_top, false) + , TERMINATOR(M_top, false) + , TERMINATOR(S_top, false) + , TERMINATOR(T_top, false) /* Don't shut down the file code until objects in files are shut down */ - if (pending == 0) - pending += DOWN(F); - + , TERMINATOR(F, true) /* Don't shut down the property list code until all objects that might - * use property lists are shut down */ - if (pending == 0) - pending += DOWN(P); - + * use property lists are shut down + */ + , TERMINATOR(P, true) /* Wait to shut down the "bottom" of various interfaces until the - * files are closed, so pieces of the file can be serialized - * correctly. + * files are closed, so pieces of the file can be serialized + * correctly. + * + * Shut down the "bottom" of the attribute, dataset, group, + * reference, dataspace, and datatype interfaces, fully closing + * out the interfaces now. */ - if (pending == 0) { - /* Shut down the "bottom" of the attribute, dataset, group, - * reference, dataspace, and datatype interfaces, fully closing - * out the interfaces now. - */ - pending += DOWN(A); - pending += DOWN(D); - pending += DOWN(G); - pending += DOWN(M); - pending += DOWN(R); - pending += DOWN(S); - pending += DOWN(T); - } /* end if */ - - /* Don't shut down "low-level" components until "high-level" components - * have successfully shut down. This prevents property lists and IDs - * from being closed "out from underneath" of the high-level objects - * that depend on them. -QAK + , TERMINATOR(A, true) + , TERMINATOR(D, false) + , TERMINATOR(G, false) + , TERMINATOR(M, false) + , TERMINATOR(S, false) + , TERMINATOR(T, false) + /* Wait to shut down low-level packages like AC until after + * the preceding high-level packages have shut down. This prevents + * low-level objects from closing "out from underneath" their + * reliant high-level objects. */ - if (pending == 0) { - pending += DOWN(AC); - /* Shut down the "pluggable" interfaces, before the plugin framework */ - pending += DOWN(Z); - pending += DOWN(FD); - pending += DOWN(VL); - /* Don't shut down the plugin code until all "pluggable" interfaces (Z, FD, PL) are shut down */ - if (pending == 0) - pending += DOWN(PL); - /* Don't shut down the error code until other APIs which use it are shut down */ - if (pending == 0) - pending += DOWN(E); - /* Don't shut down the ID code until other APIs which use them are shut down */ - if (pending == 0) - pending += DOWN(I); - /* Don't shut down the skip list code until everything that uses it is down */ - if (pending == 0) - pending += DOWN(SL); - /* Don't shut down the free list code until everything that uses it is down */ - if (pending == 0) - pending += DOWN(FL); - /* Don't shut down the API context code until _everything_ else is down */ - if (pending == 0) - pending += DOWN(CX); - } /* end if */ + , TERMINATOR(AC, true) + /* Shut down the "pluggable" interfaces, before the plugin framework */ + , TERMINATOR(Z, false) + , TERMINATOR(FD, false) + , TERMINATOR(VL, false) + /* Don't shut down the plugin code until all "pluggable" interfaces + * (Z, FD, PL) are shut down + */ + , TERMINATOR(PL, true) + /* Shut down the following packages in strictly the order given + * by the table. + */ + , TERMINATOR(E, true) + , TERMINATOR(I, true) + , TERMINATOR(SL, true) + , TERMINATOR(FL, true) + , TERMINATOR(CX, true) + }; + + do { + pending = 0; + for (i = 0; i < NELMTS(terminator); i++) { + if (terminator[i].completed) + continue; + if (pending != 0 && terminator[i].await_prior) + break; + if (terminator[i].func() == 0) { + terminator[i].completed = true; + continue; + } + + /* log a package when its terminator needs to be retried */ + pending++; + nprinted = HDsnprintf(next, nleft, "%s%s", + (next != loop) ? "," : "", terminator[i].name); + if (nprinted < 0) + continue; + if ((size_t)nprinted >= nleft) + nprinted = HDsnprintf(next, nleft, "..."); + if (nprinted < 0 || (size_t)nprinted >= nleft) + continue; + nleft -= (size_t)nprinted; + next += nprinted; + } } while (pending && ntries++ < 100); + /* clang-format on */ + if (pending) { /* Only display the error message if the user is interested in them. */ if (func) { @@ -915,6 +954,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) static int checked = 0; /* If we've already checked the version info */ static unsigned int disable_version_check = 0; /* Set if the version check should be disabled */ static const char * version_mismatch_warning = VERSION_MISMATCH_WARNING; + static const char * release_mismatch_warning = RELEASE_MISMATCH_WARNING; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API_NOINIT_NOERR_NOFS @@ -935,10 +975,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) } /* H5_VERS_MAJOR and H5_VERS_MINOR must match */ - /* Cast relnum to int to avoid warning for unsigned < 0 comparison - * in first release versions */ - if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > (int)relnum) { - + if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum) { switch (disable_version_check) { case 0: HDfprintf(stderr, "%s%s", version_mismatch_warning, @@ -973,9 +1010,10 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) break; } /* end switch */ - } /* end if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > relnum) */ + } /* end if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum) */ /* H5_VERS_RELEASE should be compatible, we will only add checks for exceptions */ + /* Library develop release versions are incompatible by design */ if (H5_VERS_RELEASE != relnum) { for (unsigned i = 0; i < VERS_RELEASE_EXCEPTIONS_SIZE; i++) { /* Check for incompatible headers or incompatible library */ @@ -983,7 +1021,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) switch (disable_version_check) { case 0: HDfprintf( - stderr, "%s%s", version_mismatch_warning, + stderr, "%s%s", release_mismatch_warning, "You can, at your own risk, disable this warning by setting the environment\n" "variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'.\n" "Setting it to 2 or higher will suppress the warning messages totally.\n"); @@ -1002,7 +1040,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) "%s'HDF5_DISABLE_VERSION_CHECK' " "environment variable is set to %d, application will\n" "continue at your own risk.\n", - version_mismatch_warning, disable_version_check); + release_mismatch_warning, disable_version_check); /* Mention the versions we are referring to */ HDfprintf(stderr, "Headers are %u.%u.%u, library is %u.%u.%u\n", majnum, minnum, relnum, (unsigned)H5_VERS_MAJOR, (unsigned)H5_VERS_MINOR, diff --git a/src/H5A.c b/src/H5A.c index c986d1f..b5cc77a 100644 --- a/src/H5A.c +++ b/src/H5A.c @@ -1868,7 +1868,7 @@ done: if all attributes were processed. DESCRIPTION - This function interates over the attributes of dataset or group + This function iterates over the attributes of dataset or group specified with 'loc_id' & 'obj_name'. For each attribute of the object, the 'op_data' and some additional information (specified below) are passed to the 'op' function. The iteration begins with the '*idx' @@ -1956,7 +1956,7 @@ done: if all attributes were processed. DESCRIPTION - This function interates over the attributes of dataset or group + This function iterates over the attributes of dataset or group specified with 'loc_id' & 'obj_name'. For each attribute of the object, the 'op_data' and some additional information (specified below) are passed to the 'op' function. The iteration begins with the '*idx' @@ -2410,7 +2410,7 @@ done: htri_t H5Aexists(hid_t obj_id, const char *attr_name) { - hbool_t exists; /* Flag for attribute existance */ + hbool_t exists; /* Flag for attribute existence */ htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_API(FAIL) @@ -2526,7 +2526,7 @@ done: htri_t H5Aexists_by_name(hid_t loc_id, const char *obj_name, const char *attr_name, hid_t lapl_id) { - hbool_t exists; /* Flag for attribute existance */ + hbool_t exists; /* Flag for attribute existence */ htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_API(FAIL) diff --git a/src/H5AC.c b/src/H5AC.c index e89c22d..150cc39 100644 --- a/src/H5AC.c +++ b/src/H5AC.c @@ -70,9 +70,6 @@ static herr_t H5AC__verify_tag(const H5AC_class_t *type); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -143,29 +140,7 @@ H5AC_init(void) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5AC_init() */ - -/*------------------------------------------------------------------------- - * Function: H5AC__init_package - * - * Purpose: Initialize interface-specific information - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Thursday, July 18, 2002 - * - *------------------------------------------------------------------------- - */ -herr_t -H5AC__init_package(void) -{ - FUNC_ENTER_PACKAGE_NOERR + FUNC_ENTER_NOAPI_NOERR #ifdef H5_HAVE_PARALLEL /* check whether to enable strict collective function calling @@ -182,8 +157,8 @@ H5AC__init_package(void) } #endif /* H5_HAVE_PARALLEL */ - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5AC__init_package() */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_init() */ /*------------------------------------------------------------------------- * Function: H5AC_term_package @@ -204,10 +179,6 @@ H5AC_term_package(void) { FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) - /* Reset interface initialization flag */ - H5_PKG_INIT_VAR = FALSE; - FUNC_LEAVE_NOAPI(0) } /* end H5AC_term_package() */ @@ -339,7 +310,7 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co aux_ptr->sync_point_done = NULL; aux_ptr->p0_image_len = 0; - HDsprintf(prefix, "%d:", mpi_rank); + HDsnprintf(prefix, sizeof(prefix), "%d:", mpi_rank); if (mpi_rank == 0) { if (NULL == (aux_ptr->d_slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) @@ -1248,7 +1219,7 @@ done: * metadata cache flush. * * Initially, this means setting up the slist prior to the - * flush. We do this in a seperate call because + * flush. We do this in a separate call because * H5F__flush_phase2() make repeated calls to H5AC_flush(). * Handling this detail in separate calls allows us to avoid * the overhead of setting up and taking down the skip list @@ -1300,7 +1271,7 @@ done: * flush. * * Initially, this means taking down the slist after the - * flush. We do this in a seperate call because + * flush. We do this in a separate call because * H5F__flush_phase2() make repeated calls to H5AC_flush(). * Handling this detail in separate calls allows us to avoid * the overhead of setting up and taking down the skip list @@ -1489,21 +1460,82 @@ H5AC_resize_entry(void *thing, size_t new_size) cache_ptr = entry_ptr->cache_ptr; HDassert(cache_ptr); - /* Resize the entry */ - if (H5C_resize_entry(thing, new_size) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTRESIZE, FAIL, "can't resize entry") - #ifdef H5_HAVE_PARALLEL - { + /* Log the generation of dirty bytes of metadata iff: + * + * 1) The entry is clean on entry, and this resize will dirty it + * (i.e. the current and new sizes are different), and + * + * 2) This is a parallel computation -- which it is if the aux_ptr + * is non-null. + * + * A few points to note about this section of the code: + * + * 1) This call must occur before the call to H5C_resize_entry() since + * H5AC__log_dirtied_entry() expects the target entry to be clean + * on entry. + * + * 2) This code has some basic issues in terms of the number of bytes + * added to the dirty bytes count. + * + * First, it adds the initial entry size to aux_ptr->dirty_bytes, + * not the final size. Note that this code used to use the final + * size, but code to support this has been removed from + * H5AC__log_dirtied_entry() for reasons unknown since I wrote this + * code. + * + * As long as all ranks do the same thing here, this probably doesn't + * matter much, although it will delay initiation of sync points. + * + * A more interesting point is that this code will not increment + * aux_ptr->dirty_bytes if a dirty entry is resized. At first glance + * this seems major, as particularly with the older file formats, + * resizes can be quite large. However, this is probably not an + * issue either, since such resizes will be accompanied by large + * amounts of dirty metadata creation in other areas -- which will + * cause aux_ptr->dirty_bytes to be incremented. + * + * The bottom line is that this code is probably OK, but the above + * points should be kept in mind. + * + * One final observation: This comment is occasioned by a bug caused + * by moving the call to H5AC__log_dirtied_entry() after the call to + * H5C_resize_entry(), and then only calling H5AC__log_dirtied_entry() + * if entry_ptr->is_dirty was false. + * + * Since H5C_resize_entry() marks the target entry dirty unless there + * is not change in size, this had the effect of not calling + * H5AC__log_dirtied_entry() when it should be, and corrupting + * the cleaned and dirtied lists used by rank 0 in the parallel + * version of the metadata cache. + * + * The point here is that you should be very careful when working with + * this code, and not modify it unless you fully understand it. + * + * JRM -- 2/28/22 + */ + + if ((!entry_ptr->is_dirty) && (entry_ptr->size != new_size)) { + + /* the entry is clean, and will be marked dirty in the resize + * operation. + */ H5AC_aux_t *aux_ptr; aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); - if ((!entry_ptr->is_dirty) && (NULL != aux_ptr)) + + if (NULL != aux_ptr) { + if (H5AC__log_dirtied_entry(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "can't log dirtied entry") + } } #endif /* H5_HAVE_PARALLEL */ + /* Resize the entry */ + if (H5C_resize_entry(thing, new_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTRESIZE, FAIL, "can't resize entry") + done: /* If currently logging, generate a message */ if (cache_ptr != NULL && cache_ptr->log_info != NULL) @@ -1685,9 +1717,14 @@ H5AC_unprotect(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *thing, un if (H5AC__log_dirtied_entry((H5AC_info_t *)thing) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "can't log dirtied entry") - if (deleted && aux_ptr->mpi_rank == 0) - if (H5AC__log_deleted_entry((H5AC_info_t *)thing) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "H5AC__log_deleted_entry() failed") + if (deleted && aux_ptr->mpi_rank == 0) { + if (H5AC__log_deleted_entry((H5AC_info_t *)thing) < 0) { + /* If we fail to log the deleted entry, push an error but still + * participate in a possible sync point ahead + */ + HDONE_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "H5AC__log_deleted_entry() failed") + } + } } /* end if */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5ACdbg.c b/src/H5ACdbg.c index 2d24adb..5e24517 100644 --- a/src/H5ACdbg.c +++ b/src/H5ACdbg.c @@ -197,9 +197,9 @@ done: * If either the parent or the child is not in the metadata * cache, the function sets *fd_exists_ptr to FALSE. * - * If both are in the cache, the childs list of parents is + * If both are in the cache, the child's list of parents is * searched for the proposed parent. If the proposed parent - * is found in the childs parent list, the function sets + * is found in the child's parent list, the function sets * *fd_exists_ptr to TRUE. In all other non-error cases, * the function sets *fd_exists_ptr FALSE. * diff --git a/src/H5ACmpio.c b/src/H5ACmpio.c index 7d2ba25..7eaf751 100644 --- a/src/H5ACmpio.c +++ b/src/H5ACmpio.c @@ -304,8 +304,10 @@ H5AC__broadcast_candidate_list(H5AC_t *cache_ptr, unsigned *num_entries_ptr, had * are used to receiving from process 0, and also load it * into a buffer for transmission. */ - if (H5AC__copy_candidate_list_to_buffer(cache_ptr, &chk_num_entries, &haddr_buf_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate buffer.") + if (H5AC__copy_candidate_list_to_buffer(cache_ptr, &chk_num_entries, &haddr_buf_ptr) < 0) { + /* Push an error, but still participate in following MPI_Bcast */ + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate buffer.") + } HDassert(chk_num_entries == num_entries); HDassert(haddr_buf_ptr != NULL); @@ -428,18 +430,23 @@ H5AC__broadcast_clean_list(H5AC_t *cache_ptr) /* allocate a buffer to store the list of entry base addresses in */ buf_size = sizeof(haddr_t) * num_entries; - if (NULL == (addr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for addr buffer") - - /* Set up user data for callback */ - udata.aux_ptr = aux_ptr; - udata.addr_buf_ptr = addr_buf_ptr; - udata.u = 0; - - /* Free all the clean list entries, building the address list in the callback */ - /* (Callback also removes the matching entries from the dirtied list) */ - if (H5SL_free(aux_ptr->c_slist_ptr, H5AC__broadcast_clean_list_cb, &udata) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "Can't build address list for clean entries") + if (NULL == (addr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) { + /* Push an error, but still participate in following MPI_Bcast */ + HDONE_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for addr buffer") + } + else { + /* Set up user data for callback */ + udata.aux_ptr = aux_ptr; + udata.addr_buf_ptr = addr_buf_ptr; + udata.u = 0; + + /* Free all the clean list entries, building the address list in the callback */ + /* (Callback also removes the matching entries from the dirtied list) */ + if (H5SL_free(aux_ptr->c_slist_ptr, H5AC__broadcast_clean_list_cb, &udata) < 0) { + /* Push an error, but still participate in following MPI_Bcast */ + HDONE_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "Can't build address list for clean entries") + } + } /* Now broadcast the list of cleaned entries */ if (MPI_SUCCESS != @@ -1132,7 +1139,7 @@ done: * * While the list of candidate cache entries is prepared * elsewhere, this function is the main routine for distributing - * and applying the list. It must be run simultaniously on + * and applying the list. It must be run simultaneously on * all processes that have the relevant file open. To ensure * proper synchronization, there is a barrier at the beginning * of this function. @@ -1311,7 +1318,7 @@ done: * * This function is the main routine for handling this * notification procedure. It must be called - * simultaniously on all processes that have the relevant + * simultaneously on all processes that have the relevant * file open. To this end, it is called only during a * sync point, with a barrier prior to the call. * @@ -1448,8 +1455,10 @@ H5AC__receive_haddr_list(MPI_Comm mpi_comm, unsigned *num_entries_ptr, haddr_t * /* allocate buffers to store the list of entry base addresses in */ buf_size = sizeof(haddr_t) * num_entries; - if (NULL == (haddr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for haddr buffer") + if (NULL == (haddr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) { + /* Push an error, but still participate in following MPI_Bcast */ + HDONE_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for haddr buffer") + } /* Now receive the list of candidate entries */ if (MPI_SUCCESS != @@ -1800,10 +1809,14 @@ H5AC__rsp__dist_md_write__flush_to_min_clean(H5F_t *f) if (evictions_enabled) { /* construct candidate list -- process 0 only */ - if (aux_ptr->mpi_rank == 0) + if (aux_ptr->mpi_rank == 0) { + /* If constructing candidate list fails, push an error but still participate + * in collective operations during following candidate list propagation + */ if (H5AC__construct_candidate_list(cache_ptr, aux_ptr, H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate list.") + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate list.") + } /* propagate and apply candidate list -- all processes */ if (H5AC__propagate_and_apply_candidate_list(f) < 0) @@ -1899,15 +1912,21 @@ H5AC__rsp__p0_only__flush(H5F_t *f) aux_ptr->write_permitted = FALSE; /* Check for error on the write operation */ - if (result < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.") - - /* this code exists primarily for the test bed -- it allows us to - * enforce POSIX semantics on the server that pretends to be a - * file system in our parallel tests. - */ - if (aux_ptr->write_done) - (aux_ptr->write_done)(); + if (result < 0) { + /* If write operation fails, push an error but still participate + * in collective operations during following cache entry + * propagation + */ + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.") + } + else { + /* this code exists primarily for the test bed -- it allows us to + * enforce POSIX semantics on the server that pretends to be a + * file system in our parallel tests. + */ + if (aux_ptr->write_done) + (aux_ptr->write_done)(); + } } /* end if */ /* Propagate cleaned entries to other ranks. */ @@ -2019,15 +2038,21 @@ H5AC__rsp__p0_only__flush_to_min_clean(H5F_t *f) aux_ptr->write_permitted = FALSE; /* Check for error on the write operation */ - if (result < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_flush_to_min_clean() failed.") - - /* this call exists primarily for the test code -- it is used - * to enforce POSIX semantics on the process used to simulate - * reads and writes in t_cache.c. - */ - if (aux_ptr->write_done) - (aux_ptr->write_done)(); + if (result < 0) { + /* If write operation fails, push an error but still participate + * in collective operations during following cache entry + * propagation + */ + HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_flush_to_min_clean() failed.") + } + else { + /* this call exists primarily for the test code -- it is used + * to enforce POSIX semantics on the process used to simulate + * reads and writes in t_cache.c. + */ + if (aux_ptr->write_done) + (aux_ptr->write_done)(); + } } /* end if */ if (H5AC__propagate_flushed_and_still_clean_entries_list(f) < 0) @@ -2093,11 +2118,11 @@ H5AC__run_sync_point(H5F_t *f, int sync_point_op) (sync_point_op == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED)); #if H5AC_DEBUG_DIRTY_BYTES_CREATION - HDfprintf(stdout, "%d:H5AC_propagate...:%u: (u/uu/i/iu/r/ru) = %zu/%u/%zu/%u/%zu/%u\n", aux_ptr->mpi_rank, + HDfprintf(stdout, "%d:H5AC_propagate...:%u: (u/uu/i/iu/m/mu) = %zu/%u/%zu/%u/%zu/%u\n", aux_ptr->mpi_rank, aux_ptr->dirty_bytes_propagations, aux_ptr->unprotect_dirty_bytes, aux_ptr->unprotect_dirty_bytes_updates, aux_ptr->insert_dirty_bytes, - aux_ptr->insert_dirty_bytes_updates, aux_ptr->rename_dirty_bytes, - aux_ptr->rename_dirty_bytes_updates); + aux_ptr->insert_dirty_bytes_updates, aux_ptr->move_dirty_bytes, + aux_ptr->move_dirty_bytes_updates); #endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ /* clear collective access flag on half of the entries in the @@ -2161,8 +2186,8 @@ H5AC__run_sync_point(H5F_t *f, int sync_point_op) aux_ptr->unprotect_dirty_bytes_updates = 0; aux_ptr->insert_dirty_bytes = 0; aux_ptr->insert_dirty_bytes_updates = 0; - aux_ptr->rename_dirty_bytes = 0; - aux_ptr->rename_dirty_bytes_updates = 0; + aux_ptr->move_dirty_bytes = 0; + aux_ptr->move_dirty_bytes_updates = 0; #endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ done: diff --git a/src/H5ACpkg.h b/src/H5ACpkg.h index 521f85a..9b75034 100644 --- a/src/H5ACpkg.h +++ b/src/H5ACpkg.h @@ -346,7 +346,7 @@ H5FL_EXTERN(H5AC_aux_t); * * The following field supports the metadata cache image feature. * - * p0_image_len: unsiged integer containing the length of the metadata cache + * p0_image_len: unsigned integer containing the length of the metadata cache * image constructed by MPI process 0. This field should be 0 * if the value is unknown, or if cache image is not enabled. * diff --git a/src/H5ACproxy_entry.c b/src/H5ACproxy_entry.c index d310a2b..25e7e9a 100644 --- a/src/H5ACproxy_entry.c +++ b/src/H5ACproxy_entry.c @@ -422,7 +422,7 @@ H5AC_proxy_entry_dest(H5AC_proxy_entry_t *pentry) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity checks */ HDassert(pentry); @@ -434,7 +434,6 @@ H5AC_proxy_entry_dest(H5AC_proxy_entry_t *pentry) /* Free the proxy entry object */ pentry = H5FL_FREE(H5AC_proxy_entry_t, pentry); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5AC_proxy_entry_dest() */ diff --git a/src/H5ACpublic.h b/src/H5ACpublic.h index f8f4f28..c853794 100644 --- a/src/H5ACpublic.h +++ b/src/H5ACpublic.h @@ -76,7 +76,7 @@ extern "C" { * * *** DEPRECATED *** Use H5Fstart/stop logging functions instead * - * The trace file is a debuging feature that allow the capture of + * The trace file is a debugging feature that allow the capture of * top level metadata cache requests for purposes of debugging and/or * optimization. This field should normally be set to FALSE, as * trace file collection imposes considerable overhead. @@ -123,7 +123,7 @@ extern "C" { * H5C_incr__off ) && ( decr_mode == H5C_decr__off )). There * is no logical reason why this should be so, but it simplifies * implementation and testing, and I can't think of any reason - * why it would be desireable. If you can think of one, I'll + * why it would be desirable. If you can think of one, I'll * revisit the issue. * * set_initial_size: Boolean flag indicating whether the size of the @@ -396,7 +396,7 @@ extern "C" { * * When the sync point is reached (or when there is a user generated * flush), process zero flushes sufficient entries to bring it into - * complience with its min clean size (or flushes all dirty entries in + * compliance with its min clean size (or flushes all dirty entries in * the case of a user generated flush), broad casts the list of * entries just cleaned to all the other processes, and then exits * the sync point. @@ -576,7 +576,7 @@ typedef struct H5AC_cache_config_t { size_t min_size; /**< Lower bound (in bytes) on the range of values that the - * adaptive cache resize code can select as the mininum cache * size. */ + * adaptive cache resize code can select as the minimum cache * size. */ long int epoch_length; /**< Number of cache accesses between runs of the adaptive cache resize @@ -708,13 +708,13 @@ typedef struct H5AC_cache_config_t { * of bytes of dirty metadata created since the last synchronization exceeds * this limit.\n This field only applies to the parallel case. While it is * ignored elsewhere, it can still draw a value out of bounds error.\n It - * must be consistant across all caches on any given file.\n By default, + * must be consistent across all caches on any given file.\n By default, * this field is set to 256 KB. It shouldn't be more than half the current * max cache size times the min clean fraction. */ int metadata_write_strategy; /**< Desired metadata write strategy. The valid values for this field - * are:\n #H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY: Specifies tha only + * are:\n #H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY: Specifies the only * process zero is allowed to write dirty metadata to disk.\n * #H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED: Specifies that process zero * still makes the decisions as to what entries should be flushed, but the diff --git a/src/H5Adense.c b/src/H5Adense.c index b0d793c..fa51f9d 100644 --- a/src/H5Adense.c +++ b/src/H5Adense.c @@ -337,7 +337,7 @@ H5A__dense_open(H5F_t *f, const H5O_ainfo_t *ainfo, const char *name) H5HF_t * fheap = NULL; /* Fractal heap handle */ H5HF_t * shared_fheap = NULL; /* Fractal heap handle for shared header messages */ H5B2_t * bt2_name = NULL; /* v2 B-tree handle for name index */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ hbool_t attr_exists; /* Attribute exists in v2 B-tree */ H5A_t * ret_value = NULL; /* Return value */ @@ -356,7 +356,7 @@ H5A__dense_open(H5F_t *f, const H5O_ainfo_t *ainfo, const char *name) if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, NULL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ @@ -429,7 +429,7 @@ H5A__dense_insert(H5F_t *f, const H5O_ainfo_t *ainfo, H5A_t *attr) H5WB_t * wb = NULL; /* Wrapped buffer for attribute data */ uint8_t attr_buf[H5A_ATTR_BUF_SIZE]; /* Buffer for serializing message */ unsigned mesg_flags = 0; /* Flags for storing message */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -443,7 +443,7 @@ H5A__dense_insert(H5F_t *f, const H5O_ainfo_t *ainfo, H5A_t *attr) if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ htri_t shared_mesg; /* Should this message be stored in the Shared Message table? */ @@ -727,7 +727,7 @@ H5A__dense_write(H5F_t *f, const H5O_ainfo_t *ainfo, H5A_t *attr) H5HF_t * fheap = NULL; /* Fractal heap handle */ H5HF_t * shared_fheap = NULL; /* Fractal heap handle for shared header messages */ H5B2_t * bt2_name = NULL; /* v2 B-tree handle for name index */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -743,7 +743,7 @@ H5A__dense_write(H5F_t *f, const H5O_ainfo_t *ainfo, H5A_t *attr) if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ @@ -865,7 +865,7 @@ H5A__dense_rename(H5F_t *f, const H5O_ainfo_t *ainfo, const char *old_name, cons H5B2_t * bt2_name = NULL; /* v2 B-tree handle for name index */ H5B2_t * bt2_corder = NULL; /* v2 B-tree handle for creation order ndex */ H5A_t * attr_copy = NULL; /* Copy of attribute to rename */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ htri_t shared_mesg; /* Should this message be stored in the Shared Message table? */ hbool_t attr_exists; /* Attribute exists in v2 B-tree */ herr_t ret_value = SUCCEED; /* Return value */ @@ -882,7 +882,7 @@ H5A__dense_rename(H5F_t *f, const H5O_ainfo_t *ainfo, const char *old_name, cons if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ @@ -1181,7 +1181,7 @@ H5A__dense_iterate(H5F_t *f, hid_t loc_id, const H5O_ainfo_t *ainfo, H5_index_t /* Check on iteration order */ if (order == H5_ITER_NATIVE && H5F_addr_defined(bt2_addr)) { H5A_bt2_ud_it_t udata; /* User data for iterator callback */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ /* Open the fractal heap */ if (NULL == (fheap = H5HF_open(f, ainfo->fheap_addr))) @@ -1191,7 +1191,7 @@ H5A__dense_iterate(H5F_t *f, hid_t loc_id, const H5O_ainfo_t *ainfo, H5_index_t if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ @@ -1338,7 +1338,7 @@ H5A__dense_remove(H5F_t *f, const H5O_ainfo_t *ainfo, const char *name) H5HF_t * shared_fheap = NULL; /* Fractal heap handle for shared header messages */ H5B2_t * bt2_name = NULL; /* v2 B-tree handle for name index */ H5A_t * attr_copy = NULL; /* Copy of attribute to remove */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1356,7 +1356,7 @@ H5A__dense_remove(H5F_t *f, const H5O_ainfo_t *ainfo, const char *name) if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ @@ -1585,7 +1585,7 @@ H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type /* If there is an index defined for the field, use it */ if (H5F_addr_defined(bt2_addr)) { H5A_bt2_ud_rmbi_t udata; /* User data for v2 B-tree record removal */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ /* Open the fractal heap */ if (NULL == (fheap = H5HF_open(f, ainfo->fheap_addr))) @@ -1595,7 +1595,7 @@ H5A__dense_remove_by_idx(H5F_t *f, const H5O_ainfo_t *ainfo, H5_index_t idx_type if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ @@ -1675,7 +1675,7 @@ H5A__dense_exists(H5F_t *f, const H5O_ainfo_t *ainfo, const char *name, hbool_t H5HF_t * fheap = NULL; /* Fractal heap handle */ H5HF_t * shared_fheap = NULL; /* Fractal heap handle for shared header messages */ H5B2_t * bt2_name = NULL; /* v2 B-tree handle for name index */ - htri_t attr_sharable; /* Flag indicating attributes are sharable */ + htri_t attr_sharable; /* Flag indicating attributes are shareable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -1694,7 +1694,7 @@ H5A__dense_exists(H5F_t *f, const H5O_ainfo_t *ainfo, const char *name, hbool_t if ((attr_sharable = H5SM_type_shared(f, H5O_ATTR_ID)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't determine if attributes are shared") - /* Get handle for shared message heap, if attributes are sharable */ + /* Get handle for shared message heap, if attributes are shareable */ if (attr_sharable) { haddr_t shared_fheap_addr; /* Address of fractal heap to use */ diff --git a/src/H5Adeprec.c b/src/H5Adeprec.c index 8ae4e41..05e4eed 100644 --- a/src/H5Adeprec.c +++ b/src/H5Adeprec.c @@ -356,7 +356,7 @@ done: last operator if it was non-zero, or zero if all attributes were processed. DESCRIPTION - This function interates over the attributes of dataset or group + This function iterates over the attributes of dataset or group specified with 'loc_id'. For each attribute of the object, the 'op_data' and some additional information (specified below) are passed to the 'op' function. The iteration begins with the '*attr_number' diff --git a/src/H5Aint.c b/src/H5Aint.c index 300d686..b0b91d0 100644 --- a/src/H5Aint.c +++ b/src/H5Aint.c @@ -99,9 +99,6 @@ static herr_t H5A__iterate_common(hid_t loc_id, H5_index_t idx_type, H5_iter_ord /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Format version bounds for attribute */ const unsigned H5O_attr_ver_bounds[] = { H5O_ATTR_VERSION_1, /* H5F_LIBVER_EARLIEST */ @@ -139,9 +136,6 @@ static const H5I_class_t H5I_ATTR_CLS[1] = {{ (H5I_free_t)H5A__close_cb /* Callback routine for closing objects of this class */ }}; -/* Flag indicating "top" of interface has been initialized */ -static hbool_t H5A_top_package_initialize_s = FALSE; - /*------------------------------------------------------------------------- * Function: H5A_init * @@ -158,30 +152,6 @@ H5A_init(void) herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5A_init() */ - -/*-------------------------------------------------------------------------- -NAME - H5A__init_package -- Initialize interface-specific information -USAGE - herr_t H5A__init_package() - -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5A__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE /* * Create attribute ID type. @@ -189,12 +159,9 @@ H5A__init_package(void) if (H5I_register_type(H5I_ATTR_CLS) < 0) HGOTO_ERROR(H5E_ATTR, H5E_CANTINIT, FAIL, "unable to initialize interface") - /* Mark "top" of interface as initialized, too */ - H5A_top_package_initialize_s = TRUE; - done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5A__init_package() */ +} /* end H5A_init() */ /*-------------------------------------------------------------------------- NAME @@ -220,16 +187,10 @@ H5A_top_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5A_top_package_initialize_s) { - if (H5I_nmembers(H5I_ATTR) > 0) { - (void)H5I_clear_type(H5I_ATTR, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - - /* Mark closed */ - if (0 == n) - H5A_top_package_initialize_s = FALSE; - } /* end if */ + if (H5I_nmembers(H5I_ATTR) > 0) { + (void)H5I_clear_type(H5I_ATTR, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ FUNC_LEAVE_NOAPI(n) } /* H5A_top_term_package() */ @@ -260,18 +221,11 @@ H5A_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Sanity checks */ - HDassert(0 == H5I_nmembers(H5I_ATTR)); - HDassert(FALSE == H5A_top_package_initialize_s); - - /* Destroy the attribute object id group */ - n += (H5I_dec_type_ref(H5I_ATTR) > 0); + /* Sanity checks */ + HDassert(0 == H5I_nmembers(H5I_ATTR)); - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Destroy the attribute object id group */ + n += (H5I_dec_type_ref(H5I_ATTR) > 0); FUNC_LEAVE_NOAPI(n) } /* H5A_term_package() */ @@ -1148,7 +1102,7 @@ H5A__get_info(const H5A_t *attr, H5A_info_t *ainfo) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Check args */ HDassert(attr); @@ -1166,7 +1120,6 @@ H5A__get_info(const H5A_t *attr, H5A_info_t *ainfo) ainfo->corder = attr->shared->crt_idx; } /* end else */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5A__get_info() */ @@ -1383,14 +1336,13 @@ H5A_oloc(H5A_t *attr) { H5O_loc_t *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI(NULL) + FUNC_ENTER_NOAPI_NOERR HDassert(attr); /* Set return value */ ret_value = &(attr->oloc); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5A_oloc() */ @@ -1414,14 +1366,13 @@ H5A_nameof(H5A_t *attr) { H5G_name_t *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI(NULL) + FUNC_ENTER_NOAPI_NOERR HDassert(attr); /* Set return value */ ret_value = &(attr->path); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5A_nameof() */ @@ -1443,14 +1394,13 @@ H5A_type(const H5A_t *attr) { H5T_t *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI(NULL) + FUNC_ENTER_NOAPI_NOERR HDassert(attr); /* Set return value */ ret_value = attr->shared->dt; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5A_type() */ diff --git a/src/H5Apublic.h b/src/H5Apublic.h index b78ae05..4ac6a53 100644 --- a/src/H5Apublic.h +++ b/src/H5Apublic.h @@ -622,7 +622,7 @@ H5_DLL hid_t H5Aget_type(hid_t attr_id); * For example, if \p idx_type, \p order, and \p idx are set to * #H5_INDEX_NAME, #H5_ITER_INC, and 5, respectively, the attribute * in question is the fifth attribute from the beginning of the - * alpha-numeric index of attribute names. If \p order were set to + * alphanumeric index of attribute names. If \p order were set to * #H5_ITER_DEC, it would be the fifth attribute from the end of * the index. * @@ -677,7 +677,7 @@ H5_DLL herr_t H5Aiterate2(hid_t loc_id, H5_index_t idx_type, H5_iter_order_t ord * For example, if \p idx_type, \p order, and \p idx are set to * #H5_INDEX_NAME, #H5_ITER_INC, and 5, respectively, the attribute * in question is the fifth attribute from the beginning of the - * alpha-numeric index of attribute names. If \p order were set to + * alphanumeric index of attribute names. If \p order were set to * #H5_ITER_DEC, it would be the fifth attribute from the end of * the index. * diff --git a/src/H5B.c b/src/H5B.c index fa6284c..a9aa312 100644 --- a/src/H5B.c +++ b/src/H5B.c @@ -157,9 +157,6 @@ static H5B_t *H5B__copy(const H5B_t *old_bt); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declare a free list to manage the haddr_t sequence information */ H5FL_SEQ_DEFINE(haddr_t); diff --git a/src/H5B2.c b/src/H5B2.c index cc6a664..b64f253 100644 --- a/src/H5B2.c +++ b/src/H5B2.c @@ -61,9 +61,6 @@ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* v2 B-tree client ID to class mapping */ /* Remember to add client ID to H5B2_subid_t in H5B2private.h when adding a new diff --git a/src/H5C.c b/src/H5C.c index 802b4e3..470805b 100644 --- a/src/H5C.c +++ b/src/H5C.c @@ -153,19 +153,10 @@ static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t *entry); static herr_t H5C__serialize_ring(H5F_t *f, H5C_ring_t ring); static herr_t H5C__serialize_single_entry(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr); +static herr_t H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr); static herr_t H5C__verify_len_eoa(H5F_t *f, const H5C_class_t *type, haddr_t addr, size_t *len, hbool_t actual); -#if H5C_DO_SLIST_SANITY_CHECKS -static hbool_t H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr); -#endif /* H5C_DO_SLIST_SANITY_CHECKS */ - -#if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t H5C__validate_lru_list(H5C_t *cache_ptr); -static herr_t H5C__validate_pinned_entry_list(H5C_t *cache_ptr); -static herr_t H5C__validate_protected_entry_list(H5C_t *cache_ptr); -#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ - #ifndef NDEBUG static void H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t *entry, const H5C_cache_entry_t *base_entry); @@ -175,9 +166,6 @@ static void H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t *entry, /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declare a free list to manage the tag info struct */ H5FL_DEFINE(H5C_tag_info_t); @@ -397,7 +385,7 @@ H5C_create(size_t max_cache_size, size_t min_clean_size, int max_type_id, cache_ptr->slist_size = (size_t)0; /* slist_ring_len, slist_ring_size, and - * slist_ptr initialized above. + * slist_ptr initializaed above. */ #if H5C_DO_SANITY_CHECKS @@ -811,7 +799,7 @@ H5C_prep_for_file_close(H5F_t *f) * * 2) Since the FSM settle routines are only invoked once during * file close, invoking them now will prevent their invocation - * during a flush, and thus avoid any resulting entrie dirties, + * during a flush, and thus avoid any resulting entry dirties, * deletions, insertion, or moves during the flush. */ if (H5C__serialize_cache(f) < 0) @@ -890,6 +878,9 @@ H5C_dest(H5F_t *f) HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "Can't generate metadata cache image") } + /* Question: Is it possible for cache_ptr->slist be non-null at this + * point? If no, shouldn't this if statement be an assert? + */ if (cache_ptr->slist_ptr != NULL) { HDassert(cache_ptr->slist_len == 0); @@ -1178,10 +1169,10 @@ H5C_evict_or_refresh_all_entries_in_page(H5F_t *f, uint64_t page, uint32_t lengt * Note that this will be satisfied from the metadata * file via the VFD SWMR reade VFD. * - * For this reason, we don't nead to check for reads + * For this reason, we don't need to check for reads * past the EOA. Torn reads and checksums are also * not an issue, since pages in the metadata file - * are checksumed and re-tried if necessary in the + * are checksummed and re-tried if necessary in the * VFD SWMR reader VFD. */ if (NULL == (image_ptr = (uint8_t *)H5MM_malloc(image_len + H5C_IMAGE_EXTRA_SPACE))) @@ -1402,7 +1393,7 @@ H5C_expunge_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, unsigned flag HDassert(H5F_addr_defined(addr)); #if H5C_DO_EXTREME_SANITY_CHECKS - if (H5C__validate_lru_list(cache_ptr) < 0) + if (H5C_validate_lru_list(cache_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "LRU extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1437,7 +1428,7 @@ H5C_expunge_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, unsigned flag done: #if H5C_DO_EXTREME_SANITY_CHECKS - if (H5C__validate_lru_list(cache_ptr) < 0) + if (H5C_validate_lru_list(cache_ptr) < 0) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "LRU extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1544,8 +1535,8 @@ H5C_flush_cache(H5F_t *f, unsigned flags) #endif /* H5C_DO_SANITY_CHECKS */ #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1724,8 +1715,8 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u #if H5C_DO_EXTREME_SANITY_CHECKS /* no need to verify that entry is not already in the index as */ /* we already make that check below. */ - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1908,7 +1899,7 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u * Finally, we usually don't check to see if the cache is * oversized at the end of an unprotect. As a result, it is * possible to have a vastly oversized cache with no protected - * entries as long as all the protects preceed the unprotects. + * entries as long as all the protects precede the unprotects. * * Since items 1 and 2 are not changing any time soon, I see * no point in worrying about the third. @@ -1927,8 +1918,8 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u H5C__UPDATE_RP_FOR_INSERTION(cache_ptr, entry_ptr, FAIL) #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed just before done") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1942,23 +1933,32 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u #ifdef H5_HAVE_PARALLEL if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) - coll_access = H5CX_get_coll_metadata_read(); + coll_access = H5F_get_coll_metadata_reads(f); entry_ptr->coll_access = coll_access; if (coll_access) { H5C__INSERT_IN_COLL_LIST(cache_ptr, entry_ptr, FAIL) /* Make sure the size of the collective entries in the cache remain in check */ - if (cache_ptr->max_cache_size * 80 < cache_ptr->coll_list_size * 100) - if (H5C_clear_coll_entries(cache_ptr, TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear collective metadata entries") - } /* end if */ + if (H5P_USER_TRUE == H5F_COLL_MD_READ(f)) { + if (cache_ptr->max_cache_size * 80 < cache_ptr->coll_list_size * 100) { + if (H5C_clear_coll_entries(cache_ptr, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear collective metadata entries") + } /* end if */ + } /* end if */ + else { + if (cache_ptr->max_cache_size * 40 < cache_ptr->coll_list_size * 100) { + if (H5C_clear_coll_entries(cache_ptr, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear collective metadata entries") + } /* end if */ + } /* end else */ + } /* end if */ #endif done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2290,8 +2290,8 @@ H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type, haddr_t old_addr, hadd HDassert(H5F_addr_ne(old_addr, new_addr)); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2404,8 +2404,8 @@ H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type, haddr_t old_addr, hadd done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2451,8 +2451,7 @@ H5C_resize_entry(void *thing, size_t new_size) HGOTO_ERROR(H5E_CACHE, H5E_BADTYPE, FAIL, "Entry isn't pinned or protected??") #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || (H5C_validate_pinned_entry_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2548,8 +2547,7 @@ H5C_resize_entry(void *thing, size_t new_size) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || (H5C_validate_pinned_entry_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2589,8 +2587,8 @@ H5C_pin_protected_entry(void *thing) HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2604,8 +2602,8 @@ H5C_pin_protected_entry(void *thing) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2668,8 +2666,8 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign HDassert(H5F_addr_defined(addr)); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2688,7 +2686,7 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign #ifdef H5_HAVE_PARALLEL if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) - coll_access = H5CX_get_coll_metadata_read(); + coll_access = H5F_get_coll_metadata_reads(f); #endif /* H5_HAVE_PARALLEL */ /* first check to see if the target is in cache */ @@ -2747,9 +2745,14 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign H5MM_memcpy(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - if (0 == mpi_rank) - if (H5C__generate_image(f, cache_ptr, entry_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't generate entry's image") + if (0 == mpi_rank) { + if (H5C__generate_image(f, cache_ptr, entry_ptr) < 0) { + /* If image generation fails, push an error but + * still participate in the following MPI_Bcast + */ + HDONE_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't generate entry's image") + } + } } /* end if */ HDassert(entry_ptr->image_ptr); @@ -2897,7 +2900,7 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign * Finally, we usually don't check to see if the cache is * oversized at the end of an unprotect. As a result, it is * possible to have a vastly oversized cache with no protected - * entries as long as all the protects preceed the unprotects. + * entries as long as all the protects precede the unprotects. * * Since items 1, 2, and 3 are not changing any time soon, I * see no point in worrying about the fourth. @@ -3012,7 +3015,7 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign * * Also, if the min_clean_size requirement is not met, we * should also call H5C__make_space_in_cache() to bring us - * into complience. + * into compliance. */ if (cache_ptr->index_size >= cache_ptr->max_cache_size) @@ -3048,16 +3051,24 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign #ifdef H5_HAVE_PARALLEL /* Make sure the size of the collective entries in the cache remain in check */ - if (coll_access) - if (cache_ptr->max_cache_size * 80 < cache_ptr->coll_list_size * 100) - if (H5C_clear_coll_entries(cache_ptr, TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "can't clear collective metadata entries") -#endif /* H5_HAVE_PARALLEL */ + if (coll_access) { + if (H5P_USER_TRUE == H5F_COLL_MD_READ(f)) { + if (cache_ptr->max_cache_size * 80 < cache_ptr->coll_list_size * 100) + if (H5C_clear_coll_entries(cache_ptr, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "can't clear collective metadata entries") + } /* end if */ + else { + if (cache_ptr->max_cache_size * 40 < cache_ptr->coll_list_size * 100) + if (H5C_clear_coll_entries(cache_ptr, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "can't clear collective metadata entries") + } /* end else */ + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -3565,8 +3576,8 @@ H5C_unpin_entry(void *_entry_ptr) HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -3576,8 +3587,8 @@ H5C_unpin_entry(void *_entry_ptr) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -3744,8 +3755,8 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) was_clean = !(entry_ptr->is_dirty); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -4011,8 +4022,8 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -5336,7 +5347,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t *f, hbool_t write_permitte /* for now at least, don't bother to maintain the minimum clean size, * as the cache should now be less than its maximum size. Due to - * the vaguries of the cache size reduction algorthim, we may not + * the vaguries of the cache size reduction algorithm, we may not * reduce the size of the cache. * * If we do, we will calculate a new minimum clean size, which will @@ -6348,7 +6359,7 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) * If either of these happen, and one of the target * or proxy entries happens to be the next entry in * the hash bucket, we could either find ourselves - * either scanning a non-existant entry, scanning + * either scanning a non-existent entry, scanning * through a different bucket, or skipping an entry. * * Neither of these are good, so restart the @@ -6546,8 +6557,8 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(ring < H5C_RING_NTYPES); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -6806,11 +6817,11 @@ done: * be cleared and not flushed, and the call can't be part of a * sequence of flushes. * - * If the caller knows the address of the skip list node at - * which the target entry resides, it can avoid a lookup - * by supplying that address in the tgt_node_ptr parameter. - * If this parameter is NULL, the function will do a skip list - * search for the entry instead. + * If the caller knows the address of the skip list node at + * which the target entry resides, it can avoid a lookup + * by supplying that address in the tgt_node_ptr parameter. + * If this parameter is NULL, the function will do a skip list + * search for the entry instead. * * The function does nothing silently if there is no entry * at the supplied address, or if the entry found has the @@ -7611,22 +7622,17 @@ H5C__load_entry(H5F_t *f, #endif /* H5_HAVE_PARALLEL */ const H5C_class_t *type, haddr_t addr, void *udata) { - hbool_t dirty = FALSE; /* Flag indicating whether thing */ - /* was dirtied during deserialize */ - uint8_t * image = NULL; /* Buffer for disk image */ - void * thing = NULL; /* Pointer to thing loaded */ - H5C_cache_entry_t *entry = NULL; /* Alias for thing loaded, as */ - /* cache entry */ -#if 0 -size_t init_len; -#endif - size_t len; /* Size of image in file */ + hbool_t dirty = FALSE; /* Flag indicating whether thing was dirtied during deserialize */ + uint8_t * image = NULL; /* Buffer for disk image */ + void * thing = NULL; /* Pointer to thing loaded */ + H5C_cache_entry_t *entry = NULL; /* Alias for thing loaded, as cache entry */ + size_t len; /* Size of image in file */ #ifdef H5_HAVE_PARALLEL - int mpi_rank = 0; /* MPI process rank */ - MPI_Comm comm = MPI_COMM_NULL; /* File MPI Communicator */ - int mpi_code; /* MPI error code */ -#endif /* H5_HAVE_PARALLEL */ - void *ret_value = NULL; /* Return value */ + int mpi_rank = 0; /* MPI process rank */ + MPI_Comm comm = MPI_COMM_NULL; /* File MPI Communicator */ + int mpi_code; /* MPI error code */ +#endif /* H5_HAVE_PARALLEL */ + void *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -7746,15 +7752,11 @@ init_len = len; /* If the entry could be read speculatively and the length is still * changing, check for updating the actual size */ - if ((type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) && (len_changed)) { - + if ((type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) && len_changed) { /* Retrieve the actual length */ actual_len = len; - if (type->get_final_load_size(image, len, udata, &actual_len) < 0) { - - /* Transfer control to while() and count towards retries */ - continue; - } + if (type->get_final_load_size(image, len, udata, &actual_len) < 0) + continue; /* Transfer control to while() and count towards retries */ /* Check for the length changing */ if (actual_len != len) { @@ -7792,11 +7794,11 @@ init_len = len; * initial portion of the buffer. * * In addition to being of questionable utility, - * this optimization changed the invarient that + * this optimization changed the invariant that * that metadata is read and written atomically. * While this didn't cause immediate problems, * the page buffer in VFD SWMR depends on this - * invarient in its management of multi-page + * invariant in its management of multi-page * metadata entries. * * To repair this issue, I have reverted to @@ -7812,7 +7814,7 @@ init_len = len; * disable it in the VFD SWMR case. * * While this issue was detected in the global - * heap case, note that the super bloc, the + * heap case, note that the super block, the * local heap, and the fractal heap also use * speculative loads. * @@ -8338,7 +8340,7 @@ done: /*------------------------------------------------------------------------- * - * Function: H5C__validate_lru_list + * Function: H5C_validate_lru_list * * Purpose: Debugging function that scans the LRU list for errors. * @@ -8358,15 +8360,15 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t -H5C__validate_lru_list(H5C_t *cache_ptr) +herr_t +H5C_validate_lru_list(H5C_t *cache_ptr) { int32_t len = 0; size_t size = 0; H5C_cache_entry_t *entry_ptr = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI(FAIL) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); @@ -8406,7 +8408,7 @@ H5C__validate_lru_list(H5C_t *cache_ptr) entry_ptr = entry_ptr->next; } - if ((cache_ptr->LRU_list_len != len) || (cache_ptr->LRU_list_size != size)) + if ((cache_ptr->LRU_list_len != (uint32_t)len) || (cache_ptr->LRU_list_size != size)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 8 failed") done: @@ -8414,12 +8416,12 @@ done: HDassert(0); FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__validate_lru_list() */ +} /* H5C_validate_lru_list() */ #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /*------------------------------------------------------------------------- * - * Function: H5C__validate_pinned_entry_list + * Function: H5C_validate_pinned_entry_list * * Purpose: Debugging function that scans the pinned entry list for * errors. @@ -8439,15 +8441,15 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t -H5C__validate_pinned_entry_list(H5C_t *cache_ptr) +herr_t +H5C_validate_pinned_entry_list(H5C_t *cache_ptr) { int32_t len = 0; size_t size = 0; H5C_cache_entry_t *entry_ptr = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI(FAIL) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); @@ -8490,7 +8492,7 @@ H5C__validate_pinned_entry_list(H5C_t *cache_ptr) entry_ptr = entry_ptr->next; } - if ((cache_ptr->pel_len != len) || (cache_ptr->pel_size != size)) + if ((cache_ptr->pel_len != (uint32_t)len) || (cache_ptr->pel_size != size)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 9 failed") done: @@ -8498,12 +8500,12 @@ done: HDassert(0); FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__validate_pinned_entry_list() */ +} /* H5C_validate_pinned_entry_list() */ #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /*------------------------------------------------------------------------- * - * Function: H5C__validate_protected_entry_list + * Function: H5C_validate_protected_entry_list * * Purpose: Debugging function that scans the protected entry list for * errors. @@ -8523,15 +8525,15 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t -H5C__validate_protected_entry_list(H5C_t *cache_ptr) +herr_t +H5C_validate_protected_entry_list(H5C_t *cache_ptr) { int32_t len = 0; size_t size = 0; H5C_cache_entry_t *entry_ptr = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI(FAIL) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); @@ -8574,7 +8576,7 @@ H5C__validate_protected_entry_list(H5C_t *cache_ptr) entry_ptr = entry_ptr->next; } - if ((cache_ptr->pl_len != len) || (cache_ptr->pl_size != size)) + if ((cache_ptr->pl_len != (uint32_t)len) || (cache_ptr->pl_size != size)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 9 failed") done: @@ -8582,12 +8584,12 @@ done: HDassert(0); FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__validate_protected_entry_list() */ +} /* H5C_validate_protected_entry_list() */ #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /*------------------------------------------------------------------------- * - * Function: H5C__entry_in_skip_list + * Function: H5C_entry_in_skip_list * * Purpose: Debugging function that scans skip list to see if it * is in present. We need this, as it is possible for @@ -8605,8 +8607,8 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_SLIST_SANITY_CHECKS -static hbool_t -H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) +hbool_t +H5C_entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) { H5SL_node_t *node_ptr; hbool_t in_slist; @@ -8634,7 +8636,7 @@ H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) } return (in_slist); -} /* H5C__entry_in_skip_list() */ +} /* H5C_entry_in_skip_list() */ #endif /* H5C_DO_SLIST_SANITY_CHECKS */ /*------------------------------------------------------------------------- @@ -8703,7 +8705,7 @@ done: * To cork the object * Return error if the object is already corked * H5C__UNCORK: - * To uncork the obejct + * To uncork the object * Return error if the object is not corked * H5C__GET_CORKED: * To retrieve the cork status of an object in @@ -9124,8 +9126,8 @@ H5C__serialize_cache(H5F_t *f) #endif /* H5C_DO_SANITY_CHECKS */ #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -9134,7 +9136,7 @@ H5C__serialize_cache(H5F_t *f) * each entry in the cache to zero before we start the serialization. * This allows us to detect the case in which any entry is serialized * more than once (a performance issues), and more importantly, the - * case is which any flush depencency parent is serializes more than + * case is which any flush dependency parent is serializes more than * once (a correctness issue). */ { @@ -9291,8 +9293,8 @@ H5C__serialize_ring(H5F_t *f, H5C_ring_t ring) * are serialized correctly, it doesn't matter if we have to go back * and serialize an entry a second time. * - * These possible actions result in the following modfications to - * tha basic algorithm: + * These possible actions result in the following modifications to + * the basic algorithm: * * 1) In the event of an entry expunge, eviction or removal, we must * restart the scan as it is possible that the next entry in our @@ -9342,7 +9344,7 @@ H5C__serialize_ring(H5F_t *f, H5C_ring_t ring) * condition appears. * * Observe that either eviction or removal of entries as a result of - * a serialization is not a problem as long as the flush depencency + * a serialization is not a problem as long as the flush dependency * tree does not change beyond the removal of a leaf. */ while (!done) { @@ -9569,7 +9571,7 @@ done: * *------------------------------------------------------------------------- */ -herr_t +static herr_t H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) { haddr_t new_addr = HADDR_UNDEF; @@ -9578,7 +9580,7 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) unsigned serialize_flags = H5C__SERIALIZE_NO_FLAGS_SET; herr_t ret_value = SUCCEED; - FUNC_ENTER_PACKAGE + FUNC_ENTER_STATIC /* Sanity check */ HDassert(f); diff --git a/src/H5CS.c b/src/H5CS.c index d7cb6f1..0fabc0d 100644 --- a/src/H5CS.c +++ b/src/H5CS.c @@ -47,7 +47,7 @@ typedef struct H5CS_t { * each thread individually. The association of stacks to threads will * be handled by the pthread library. * - * In order for this macro to work, H5CS_get_my_stack() must be preceeded + * In order for this macro to work, H5CS_get_my_stack() must be preceded * by "H5CS_t *fstack =". */ static H5CS_t *H5CS__get_stack(void); @@ -127,7 +127,7 @@ H5CS__get_stack(void) herr_t H5CS_print_stack(const H5CS_t *fstack, FILE *stream) { - const int indent = 2; /* Indention level */ + const int indent = 2; /* Indentation level */ int i; /* Local index ariable */ /* Don't push this function on the function stack... :-) */ diff --git a/src/H5CX.c b/src/H5CX.c index 01bf435..c5bb8e4 100644 --- a/src/H5CX.c +++ b/src/H5CX.c @@ -50,7 +50,7 @@ * each thread individually. The association of contexts to threads will * be handled by the pthread library. * - * In order for this macro to work, H5CX_get_my_context() must be preceeded + * In order for this macro to work, H5CX_get_my_context() must be preceded * by "H5CX_node_t *ctx =". */ #define H5CX_get_my_context() H5CX__get_context() @@ -428,9 +428,6 @@ static H5CX_node_t *H5CX__pop_common(hbool_t update_dxpl_props); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*******************/ /* Local Variables */ /*******************/ @@ -463,18 +460,17 @@ H5FL_DEFINE_STATIC(H5CX_node_t); /* Declare a static free list to manage H5CX_state_t structs */ H5FL_DEFINE_STATIC(H5CX_state_t); -/*-------------------------------------------------------------------------- -NAME - H5CX__init_package -- Initialize interface-specific information -USAGE - herr_t H5CX__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. ---------------------------------------------------------------------------*/ +/*------------------------------------------------------------------------- + * Function: H5CX_init + * + * Purpose: Initialize the interface from some other layer. + * + * Return: Success: non-negative + * Failure: negative + *------------------------------------------------------------------------- + */ herr_t -H5CX__init_package(void) +H5CX_init(void) { H5P_genplist_t *dx_plist; /* Data transfer property list */ H5P_genplist_t *lc_plist; /* Link creation property list */ @@ -484,7 +480,7 @@ H5CX__init_package(void) H5P_genplist_t *fa_plist; /* File access property list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI(FAIL) /* Reset the "default DXPL cache" information */ HDmemset(&H5CX_def_dxpl_cache, 0, sizeof(H5CX_dxpl_cache_t)); @@ -649,10 +645,9 @@ H5CX__init_package(void) if (H5P_get(fa_plist, H5F_ACS_LIBVER_HIGH_BOUND_NAME, &H5CX_def_fapl_cache.high_bound) < 0) HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve dataset minimize flag") - done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5CX__init_package() */ +} /*------------------------------------------------------------------------- * Function: H5CX_term_package @@ -673,24 +668,20 @@ H5CX_term_package(void) { FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - H5CX_node_t *cnode; /* Context node */ + H5CX_node_t *cnode; /* Context node */ - /* Pop the top context node from the stack */ - /* (Can't check for errors, as rest of library is shut down) */ - cnode = H5CX__pop_common(FALSE); + /* Pop the top context node from the stack */ + /* (Can't check for errors, as rest of library is shut down) */ + cnode = H5CX__pop_common(FALSE); - /* Free the context node */ - /* (Allocated with HDmalloc() in H5CX_push_special() ) */ - HDfree(cnode); + /* Free the context node */ + /* (Allocated with HDmalloc() in H5CX_push_special() ) */ + HDfree(cnode); #ifndef H5_HAVE_THREADSAFE - H5CX_head_g = NULL; + H5CX_head_g = NULL; #endif /* H5_HAVE_THREADSAFE */ - H5_PKG_INIT_VAR = FALSE; - } /* end if */ - FUNC_LEAVE_NOAPI(0) } /* end H5CX_term_package() */ @@ -1227,7 +1218,7 @@ H5CX_set_libver_bounds(H5F_t *f) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -1241,7 +1232,6 @@ H5CX_set_libver_bounds(H5F_t *f) (*head)->ctx.low_bound_valid = TRUE; (*head)->ctx.high_bound_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_libver_bounds() */ @@ -1407,9 +1397,7 @@ H5CX_set_apl(hid_t *acspl_id, const H5P_libclass_t *libclass, /* If parallel is enabled and the file driver used is the MPI-IO * VFD, issue an MPI barrier for easier debugging if the API function - * calling this is supposed to be called collectively. Note that this - * happens only when the environment variable H5_COLL_BARRIER is set - * to non 0. + * calling this is supposed to be called collectively. */ if (H5_coll_api_sanity_check_g) { MPI_Comm mpi_comm; /* File communicator */ @@ -1435,7 +1423,7 @@ done: * Purpose: Sanity checks and sets up collective operations. * * Note: Should be called for all API routines that modify file - * file metadata but don't pass in an access property list. + * metadata but don't pass in an access property list. * * Return: Non-negative on success / Negative on failure * @@ -1466,9 +1454,7 @@ H5CX_set_loc(hid_t /* If parallel is enabled and the file driver used is the MPI-IO * VFD, issue an MPI barrier for easier debugging if the API function - * calling this is supposed to be called collectively. Note that this - * happens only when the environment variable H5_COLL_BARRIER is set - * to non 0. + * calling this is supposed to be called collectively. */ if (H5_coll_api_sanity_check_g) { MPI_Comm mpi_comm; /* File communicator */ @@ -1509,7 +1495,7 @@ H5CX_set_vol_wrap_ctx(void *vol_wrap_ctx) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -1521,7 +1507,6 @@ H5CX_set_vol_wrap_ctx(void *vol_wrap_ctx) /* Mark the value as valid */ (*head)->ctx.vol_wrap_ctx_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_vol_wrap_ctx() */ @@ -1543,7 +1528,7 @@ H5CX_set_vol_connector_prop(const H5VL_connector_prop_t *vol_connector_prop) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -1555,7 +1540,6 @@ H5CX_set_vol_connector_prop(const H5VL_connector_prop_t *vol_connector_prop) /* Mark the value as valid */ (*head)->ctx.vol_connector_prop_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_vol_connector_prop() */ @@ -1637,7 +1621,7 @@ H5CX_get_vol_wrap_ctx(void **vol_wrap_ctx) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ HDassert(vol_wrap_ctx); @@ -1651,7 +1635,6 @@ H5CX_get_vol_wrap_ctx(void **vol_wrap_ctx) else *vol_wrap_ctx = NULL; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_get_vol_wrap_ctx() */ @@ -1673,7 +1656,7 @@ H5CX_get_vol_connector_prop(H5VL_connector_prop_t *vol_connector_prop) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ HDassert(vol_connector_prop); @@ -1687,7 +1670,6 @@ H5CX_get_vol_connector_prop(H5VL_connector_prop_t *vol_connector_prop) else HDmemset(vol_connector_prop, 0, sizeof(H5VL_connector_prop_t)); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_get_vol_connector_prop() */ @@ -1803,7 +1785,7 @@ H5CX_get_mpi_coll_datatypes(MPI_Datatype *btype, MPI_Datatype *ftype) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ HDassert(btype); @@ -1815,7 +1797,6 @@ H5CX_get_mpi_coll_datatypes(MPI_Datatype *btype, MPI_Datatype *ftype) *btype = (*head)->ctx.btype; *ftype = (*head)->ctx.ftype; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_get_mpi_coll_datatypes() */ @@ -2977,7 +2958,7 @@ H5CX_set_mpi_coll_datatypes(MPI_Datatype btype, MPI_Datatype ftype) herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -2987,7 +2968,6 @@ H5CX_set_mpi_coll_datatypes(MPI_Datatype btype, MPI_Datatype ftype) (*head)->ctx.btype = btype; (*head)->ctx.ftype = ftype; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_mpi_coll_datatypes() */ @@ -3009,7 +2989,7 @@ H5CX_set_io_xfer_mode(H5FD_mpio_xfer_t io_xfer_mode) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3021,7 +3001,6 @@ H5CX_set_io_xfer_mode(H5FD_mpio_xfer_t io_xfer_mode) /* Mark the value as valid */ (*head)->ctx.io_xfer_mode_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_io_xfer_mode() */ @@ -3043,7 +3022,7 @@ H5CX_set_mpio_coll_opt(H5FD_mpio_collective_opt_t mpio_coll_opt) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3055,7 +3034,6 @@ H5CX_set_mpio_coll_opt(H5FD_mpio_collective_opt_t mpio_coll_opt) /* Mark the value as valid */ (*head)->ctx.mpio_coll_opt_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_mpio_coll_opt() */ @@ -3135,7 +3113,7 @@ H5CX_set_vlen_alloc_info(H5MM_allocate_t alloc_func, void *alloc_info, H5MM_free H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3150,7 +3128,6 @@ H5CX_set_vlen_alloc_info(H5MM_allocate_t alloc_func, void *alloc_info, H5MM_free /* Mark the value as valid */ (*head)->ctx.vl_alloc_info_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_vlen_alloc_info() */ @@ -3172,7 +3149,7 @@ H5CX_set_nlinks(size_t nlinks) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3184,7 +3161,6 @@ H5CX_set_nlinks(size_t nlinks) /* Mark the value as valid */ (*head)->ctx.nlinks_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_nlinks() */ diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index c500356..8ec1c59 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -160,6 +160,8 @@ H5_DLL herr_t H5CX_set_vlen_alloc_info(H5MM_allocate_t alloc_func, void *alloc_i /* "Setter" routines for LAPL properties cached in API context */ H5_DLL herr_t H5CX_set_nlinks(size_t nlinks); +H5_DLL herr_t H5CX_init(void); + /* "Setter" routines for cached DXPL properties that must be returned to application */ #ifdef H5_HAVE_PARALLEL H5_DLL void H5CX_set_mpio_actual_chunk_opt(H5D_mpio_actual_chunk_opt_mode_t chunk_opt); diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index 7784c3b..58b6bbe 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -1012,9 +1012,9 @@ H5C__dump_entry(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr, hbool_t du * If either the parent or the child is not in the metadata * cache, the function sets *fd_exists_ptr to FALSE. * - * If both are in the cache, the childs list of parents is + * If both are in the cache, the child's list of parents is * searched for the proposed parent. If the proposed parent - * is found in the childs parent list, the function sets + * is found in the child's parent list, the function sets * *fd_exists_ptr to TRUE. In all other non-error cases, * the function sets *fd_exists_ptr FALSE. * diff --git a/src/H5Cimage.c b/src/H5Cimage.c index 659382b..bdfba08 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -591,7 +591,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t * * Note that at present, dirty can't be set to true with prefetched * entries. However this may change, so include this functionality - * against that posibility. + * against that possibility. * * Also, note that it is possible for a prefetched entry to be dirty -- * hence the value assigned to ds_entry_ptr->is_dirty below. @@ -1192,7 +1192,7 @@ done: * image superblock extension message must be deleted from * the superblock extension and the image block freed * - * Contrawise, if the file is openened R/O, the metadata + * Contrawise, if the file is opened R/O, the metadata * cache image superblock extension message and image block * must be left as is. Further, any dirty entries in the * cache image block must be marked as clean to avoid @@ -1884,7 +1884,7 @@ done: * Purpose: Decode the metadata cache image entry from the supplied * buffer into the supplied instance of H5C_image_entry_t. * This includes allocating a buffer for the entry image, - * loading it, and seting ie_ptr->image_ptr to point to + * loading it, and setting ie_ptr->image_ptr to point to * the buffer. * * Advances the buffer pointer to the first byte @@ -2401,7 +2401,7 @@ done: * also be a flush dependency child. * * Finally, note that for purposes of the cache image, flush - * dependency height ends when a flush dependecy relation + * dependency height ends when a flush dependency relation * passes off the cache image. * * On exit, the flush dependency height of each entry in the @@ -2563,7 +2563,7 @@ H5C__prep_for_file_close__compute_fd_heights(const H5C_t *cache_ptr) entry_ptr = entry_ptr->il_next; } /* while (entry_ptr != NULL) */ - /* At present, no extenal parent or child flush dependency links + /* At present, no external parent or child flush dependency links * should exist -- hence the following assertions. This will change * if we support ageout of entries in the cache image. */ @@ -3444,7 +3444,7 @@ done: * creating if specified. * * In general, the size and location of the cache image block - * will be unknow at the time that the cache image superblock + * will be unknown at the time that the cache image superblock * message is created. A subsequent call to this routine will * be used to write the correct data. * diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index 35d8dd0..d127961 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -1097,7 +1097,7 @@ done: * * Note that this function is a modified version of * H5C_flush_cache() -- any changes there may need to be - * reflected here and vise versa. + * reflected here and vice versa. * * Return: Non-negative on success/Negative on failure. * diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 5fc255f..dd286a9 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -999,7 +999,7 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ * ***********************************************************************/ -/* H5C__HASH_TABLE_LEN is defined in H5Cpkg.h. It must be a power of two. */ +/* H5C__HASH_TABLE_LEN is defined in H5Cpkg.h. It mut be a power of two. */ #define H5C__HASH_MASK ((size_t)(H5C__HASH_TABLE_LEN - 1) << 3) @@ -1642,7 +1642,7 @@ if ( ( (cache_ptr)->index_size != \ * * H5C_DO_SLIST_SANITY_CHECKS * - * can be selected independantly. This is easy to miss as the + * can be selected independently. This is easy to miss as the * two #defines are easy to confuse. */ @@ -4664,7 +4664,7 @@ typedef struct H5C_tag_info_t { * This field exists because the fixed and variable length * array cache clients allocate numerous entries in a single * block, and sub-allocate metadata cache entries out of this - * block. The effect of this is to break the invarient, + * block. The effect of this is to break the invariant, * normally maintained by the free space managers in paged * allocation mode, that no entry of less than a page in * size crosses page boundaries, and that entries of page @@ -4674,7 +4674,7 @@ typedef struct H5C_tag_info_t { * * The correct solution is to modify the fixed and variable * length array cache client to repair this. However, in - * the interrim, this field exists to detect similar + * the interim, this field exists to detect similar * behaviour elsewhere. * * To complicate matters, speculative reads for metadata @@ -5029,7 +5029,7 @@ typedef struct H5C_tag_info_t { * This field exists because the fixed and variable length * array cache clients allocate numerous entries in a single * block, and sub-allocate metadata cache entries out of this - * block. The effect of this is to break the invarient, + * block. The effect of this is to break the invariant, * normally maintained by the free space managers in paged * allocation mode, that no entry of less than a page in * size crosses page boundaries, and that entries of page @@ -5039,7 +5039,7 @@ typedef struct H5C_tag_info_t { * * The correct solution is to modify the fixed and variable * length array cache client to repair this. However, in - * the interrim, this field exists to detect similar + * the interim, this field exists to detect similar * behaviour elsewhere. * ****************************************************************************/ @@ -5317,7 +5317,6 @@ H5_DLL herr_t H5C__mark_flush_dep_unserialized(H5C_cache_entry_t * entry_ptr); H5_DLL herr_t H5C__make_space_in_cache(H5F_t * f, size_t space_needed, hbool_t write_permitted); H5_DLL herr_t H5C__flush_marked_entries(H5F_t * f); -H5_DLL herr_t H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr); H5_DLL herr_t H5C__serialize_cache(H5F_t *f); H5_DLL herr_t H5C__iter_tagged_entries(H5C_t *cache, haddr_t tag, hbool_t match_global, H5C_tag_iter_cb_t cb, void *cb_ctx); diff --git a/src/H5Cprefetched.c b/src/H5Cprefetched.c index 74573d0..fd5153d 100644 --- a/src/H5Cprefetched.c +++ b/src/H5Cprefetched.c @@ -303,7 +303,7 @@ H5C__prefetched_entry_free_icr(void *_thing) HDassert(entry_ptr->fd_parent_count == 0); if (entry_ptr->image_ptr != NULL) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "prefetched entry image buffer still attatched?") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "prefetched entry image buffer still attached?") entry_ptr = H5FL_FREE(H5C_cache_entry_t, entry_ptr); diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index cbc34b2..524a276 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -213,7 +213,7 @@ #define H5C_DO_TAGGING_SANITY_CHECKS 1 #define H5C_DO_EXTREME_SANITY_CHECKS 0 #else /* NDEBUG */ -/* With rare execptions, the following defines should be set +/* With rare exceptions, the following defines should be set * to 0 if NDEBUG is defined */ #define H5C_DO_SANITY_CHECKS 0 @@ -879,7 +879,7 @@ typedef struct H5C_t H5C_t; * managers. * * The refresh entry callback exists to address this issue. As - * indicated above, it is essential for the superblock, and desireable + * indicated above, it is essential for the superblock, and desirable * whenever it is not possible to simply evict an entry that resides * in a modified page cache page. * @@ -1528,7 +1528,7 @@ typedef int H5C_ring_t; * with no flush dependency children. * * Since the image_fd_height is used to order entries in the - * cache image so that fd parents preceed fd children, for + * cache image so that fd parents precede fd children, for * purposes of this field, and entry is at flush dependency * level 0 if it either has no children, or if all of its * children are not in the cache image. @@ -1643,7 +1643,7 @@ typedef int H5C_ring_t; * number of times each entry is serialized during cache * serialization. While no entry should be serialized more than * once in any serialization call, throw an assertion if any - * flush depencency parent is serialized more than once during + * flush dependency parent is serialized more than once during * a single cache serialization. * * This is a debugging field, and thus is maintained only if @@ -1680,7 +1680,7 @@ typedef int H5C_ring_t; * * refreshed_in_tick: When an entry is refreshed as part of the VFD SWMR * reader end of tick processing, this field is used to - * record the tick in which this occured. The field is + * record the tick in which this occurred. The field is * used primarily for sanity checking. * * pi_next: Next pointer used by the page index hash table that maps @@ -1869,7 +1869,7 @@ typedef struct H5C_cache_entry_t { * with no flush dependency children. * * Since the image_fd_height is used to order entries in the - * cache image so that fd parents preceed fd children, for + * cache image so that fd parents precede fd children, for * purposes of this field, an entry is at flush dependency * level 0 if it either has no children, or if all of its * children are not in the cache image. @@ -2312,7 +2312,7 @@ typedef struct H5C_auto_size_ctl_t { * equivalent of H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE. * * flags: Unsigned integer containing flags controlling which aspects of the - * cache image functinality is actually executed. The primary impetus + * cache image functionality is actually executed. The primary impetus * behind this field is to allow development of tests for partial * implementations that will require little if any modification to run * with the full implementation. In normal operation, all flags should @@ -2434,6 +2434,16 @@ H5_DLL herr_t H5C_cache_image_status(H5F_t *f, hbool_t *load_ci_ptr, hbool_t * H5_DLL hbool_t H5C_cache_image_pending(const H5C_t *cache_ptr); H5_DLL herr_t H5C_get_mdc_image_info(const H5C_t *cache_ptr, haddr_t *image_addr, hsize_t *image_len); +#if H5C_DO_SLIST_SANITY_CHECKS +H5_DLL hbool_t H5C_entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr); +#endif + +#if H5C_DO_EXTREME_SANITY_CHECKS +H5_DLL herr_t H5C_validate_lru_list(H5C_t *cache_ptr); +H5_DLL herr_t H5C_validate_pinned_entry_list(H5C_t *cache_ptr); +H5_DLL herr_t H5C_validate_protected_entry_list(H5C_t *cache_ptr); +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + /* Logging functions */ H5_DLL herr_t H5C_start_logging(H5C_t *cache); H5_DLL herr_t H5C_stop_logging(H5C_t *cache); diff --git a/src/H5Cquery.c b/src/H5Cquery.c index d95dd87..d62c61a 100644 --- a/src/H5Cquery.c +++ b/src/H5Cquery.c @@ -201,7 +201,7 @@ H5C_get_cache_hit_rate(const H5C_t *cache_ptr, double *hit_rate_ptr) if (cache_ptr->cache_accesses > 0) *hit_rate_ptr = ((double)(cache_ptr->cache_hits)) / ((double)(cache_ptr->cache_accesses)); else - *hit_rate_ptr = 0.0f; + *hit_rate_ptr = 0.0; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5D.c b/src/H5D.c index 7153c7d..c10b079 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -62,9 +62,6 @@ static herr_t H5D__set_extent_api_common(hid_t dset_id, const hsize_t size[], vo /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -1805,7 +1802,7 @@ H5Dset_extent(hid_t dset_id, const hsize_t size[]) FUNC_ENTER_API(FAIL) H5TRACE2("e", "i*h", dset_id, size); - /* Change a datset's dimenions synchronously */ + /* Change a datset's dimensions synchronously */ if ((ret_value = H5D__set_extent_api_common(dset_id, size, NULL, NULL)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to synchronously change a dataset's dimensions") @@ -1838,7 +1835,7 @@ H5Dset_extent_async(const char *app_file, const char *app_func, unsigned app_lin if (H5ES_NONE != es_id) token_ptr = &token; /* Point at token for VOL connector to set up */ - /* Change a datset's dimenions asynchronously */ + /* Change a datset's dimensions asynchronously */ if (H5D__set_extent_api_common(dset_id, size, token_ptr, &vol_obj) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to asynchronously change a dataset's dimensions") diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index cd54c3a..9078877 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -239,10 +239,14 @@ typedef struct H5D_chunk_file_iter_ud_t { #ifdef H5_HAVE_PARALLEL /* information to construct a collective I/O operation for filling chunks */ -typedef struct H5D_chunk_coll_info_t { - size_t num_io; /* Number of write operations */ - haddr_t *addr; /* array of the file addresses of the write operation */ -} H5D_chunk_coll_info_t; +typedef struct H5D_chunk_coll_fill_info_t { + size_t num_chunks; /* Number of chunks in the write operation */ + struct chunk_coll_fill_info { + haddr_t addr; /* File address of the chunk */ + size_t chunk_size; /* Size of the chunk in the file */ + hbool_t unfiltered_partial_chunk; + } * chunk_info; +} H5D_chunk_coll_fill_info_t; #endif /* H5_HAVE_PARALLEL */ typedef struct H5D_chunk_iter_ud_t { @@ -258,14 +262,13 @@ typedef struct H5D_chunk_iter_ud_t { static herr_t H5D__chunk_construct(H5F_t *f, H5D_t *dset); static herr_t H5D__chunk_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id); static herr_t H5D__chunk_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, - H5D_chunk_map_t *fm); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); static herr_t H5D__chunk_io_init_selections(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm); static herr_t H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm); + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); static herr_t H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm); + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); static herr_t H5D__chunk_flush(H5D_t *dset); static herr_t H5D__chunk_io_term(const H5D_chunk_map_t *fm); static herr_t H5D__chunk_dest(H5D_t *dset); @@ -290,9 +293,6 @@ static int H5D__chunk_format_convert_cb(const H5D_chunk_rec_t *chunk_rec, void * /* Helper routines */ static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, const hsize_t *max_dims); -static void * H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline); -static void * H5D__chunk_mem_xfree(void *chk, const void *pline); -static void * H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline); static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *udata); static hbool_t H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *udata); @@ -309,8 +309,6 @@ static herr_t H5D__chunk_mem_cb(void *elem, const H5T_t *type, unsigned ndims, static unsigned H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled); static herr_t H5D__chunk_flush_entry(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t reset); static herr_t H5D__chunk_cache_evict(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t flush); -static hbool_t H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, - const hsize_t *chunk_scaled, const hsize_t *dset_dims); static void * H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, hbool_t relax, hbool_t prev_unfilt_chunk); static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata, hbool_t dirty, @@ -318,9 +316,9 @@ static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ static herr_t H5D__chunk_cache_prune(const H5D_t *dset, size_t size); static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk); #ifdef H5_HAVE_PARALLEL -static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, - size_t chunk_size, const void *fill_buf); -static int H5D__chunk_cmp_addr(const void *addr1, const void *addr2); +static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, + const void *fill_buf, const void *partial_chunk_fill_buf); +static int H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2); #endif /* H5_HAVE_PARALLEL */ /* Debugging helper routine callback */ @@ -331,13 +329,24 @@ static int H5D__chunk_dump_index_cb(const H5D_chunk_rec_t *chunk_rec, void *_uda /*********************/ /* Chunked storage layout I/O ops */ -const H5D_layout_ops_t H5D_LOPS_CHUNK[1] = { - {H5D__chunk_construct, H5D__chunk_init, H5D__chunk_is_space_alloc, H5D__chunk_is_data_cached, - H5D__chunk_io_init, H5D__chunk_read, H5D__chunk_write, +const H5D_layout_ops_t H5D_LOPS_CHUNK[1] = {{ + H5D__chunk_construct, /* construct */ + H5D__chunk_init, /* init */ + H5D__chunk_is_space_alloc, /* is_space_alloc */ + H5D__chunk_is_data_cached, /* is_data_cached */ + H5D__chunk_io_init, /* io_init */ + H5D__chunk_read, /* ser_read */ + H5D__chunk_write, /* ser_write */ #ifdef H5_HAVE_PARALLEL - H5D__chunk_collective_read, H5D__chunk_collective_write, -#endif /* H5_HAVE_PARALLEL */ - NULL, NULL, H5D__chunk_flush, H5D__chunk_io_term, H5D__chunk_dest}}; + H5D__chunk_collective_read, /* par_read */ + H5D__chunk_collective_write, /* par_write */ +#endif + NULL, /* readvv */ + NULL, /* writevv */ + H5D__chunk_flush, /* flush */ + H5D__chunk_io_term, /* io_term */ + H5D__chunk_dest /* dest */ +}}; /*******************/ /* Local Variables */ @@ -966,7 +975,7 @@ H5D__chunk_init(H5F_t *f, const H5D_t *const dset, hid_t dapl_id) if (!(scaled_power2up = H5VM_power2up(rdcc->scaled_dims[u]))) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") - /* Inital 'power2up' values for scaled dimensions */ + /* Initial 'power2up' values for scaled dimensions */ rdcc->scaled_power2up[u] = scaled_power2up; /* Number of bits required to encode scaled dimension size */ @@ -1059,7 +1068,7 @@ H5D__chunk_is_data_cached(const H5D_shared_t *shared_dset) */ static herr_t H5D__chunk_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm) + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm) { const H5D_t *dataset = io_info->dset; /* Local pointer to dataset info */ hssize_t old_offset[H5O_LAYOUT_NDIMS]; /* Old selection offset */ @@ -1090,7 +1099,7 @@ H5D__chunk_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_inf * speed up hyperslab calculations by removing the extra checks and/or * additions involving the offset and the hyperslab selection -QAK) */ - if ((file_space_normalized = H5S_hyper_normalize_offset((H5S_t *)file_space, old_offset)) < 0) + if ((file_space_normalized = H5S_hyper_normalize_offset(file_space, old_offset)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to normalize selection") /* Decide the number of chunks in each dimension */ @@ -1354,7 +1363,7 @@ done: * *------------------------------------------------------------------------- */ -static void * +void * H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ @@ -1385,7 +1394,7 @@ H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline) * *------------------------------------------------------------------------- */ -static void * +void * H5D__chunk_mem_xfree(void *chk, const void *_pline) { const H5O_pline_t *pline = (const H5O_pline_t *)_pline; @@ -1409,7 +1418,7 @@ H5D__chunk_mem_xfree(void *chk, const void *_pline) * calls H5D__chunk_mem_xfree and discards the return value. *------------------------------------------------------------------------- */ -static void +void H5D__chunk_mem_free(void *chk, const void *_pline) { (void)H5D__chunk_mem_xfree(chk, _pline); @@ -1429,7 +1438,7 @@ H5D__chunk_mem_free(void *chk, const void *_pline) * *------------------------------------------------------------------------- */ -static void * +void * H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ @@ -2251,7 +2260,6 @@ H5D__chunk_file_cb(void H5_ATTR_UNUSED *elem, const H5T_t H5_ATTR_UNUSED *type, /* Set the chunk's scaled coordinates */ H5MM_memcpy(chunk_info->scaled, scaled, sizeof(hsize_t) * fm->f_ndims); chunk_info->scaled[fm->f_ndims] = 0; - H5MM_memcpy(chunk_info->scaled, scaled, sizeof(hsize_t) * fm->f_ndims); /* Insert the new chunk into the skip list */ if (H5SL_insert(fm->sel_chunks, chunk_info, &chunk_info->index) < 0) { @@ -2466,8 +2474,7 @@ done: */ static herr_t H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t H5_ATTR_UNUSED nelmts, - const H5S_t H5_ATTR_UNUSED *file_space, const H5S_t H5_ATTR_UNUSED *mem_space, - H5D_chunk_map_t *fm) + H5S_t H5_ATTR_UNUSED *file_space, H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) { H5SL_node_t * chunk_node; /* Current node in chunk skip list */ H5D_io_info_t nonexistent_io_info; /* "nonexistent" I/O info object */ @@ -2543,7 +2550,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_ HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) || (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0)); - /* Check for non-existant chunk & skip it if appropriate */ + /* Check for non-existent chunk & skip it if appropriate */ if (H5F_addr_defined(udata.chunk_block.offset) || UINT_MAX != udata.idx_hint || !skip_missing_chunks) { H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */ @@ -2629,8 +2636,7 @@ done: */ static herr_t H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t H5_ATTR_UNUSED nelmts, - const H5S_t H5_ATTR_UNUSED *file_space, const H5S_t H5_ATTR_UNUSED *mem_space, - H5D_chunk_map_t *fm) + H5S_t H5_ATTR_UNUSED *file_space, H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) { H5SL_node_t * chunk_node; /* Current node in chunk skip list */ H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ @@ -3211,7 +3217,9 @@ H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udat unsigned idx = 0; /* Index of chunk in cache, if present */ hbool_t found = FALSE; /* In cache? */ #ifdef H5_HAVE_PARALLEL - hbool_t reenable_coll_md_reads = FALSE; + H5P_coll_md_read_flag_t md_reads_file_flag; + hbool_t md_reads_context_flag; + hbool_t restore_md_reads_state = FALSE; #endif herr_t ret_value = SUCCEED; /* Return value */ @@ -3285,11 +3293,10 @@ H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udat * processes. */ if (H5F_HAS_FEATURE(idx_info.f, H5FD_FEAT_HAS_MPI)) { - hbool_t do_coll_md_reads = H5CX_get_coll_metadata_read(); - if (do_coll_md_reads) { - H5CX_set_coll_metadata_read(FALSE); - reenable_coll_md_reads = TRUE; - } + md_reads_file_flag = H5P_FORCE_FALSE; + md_reads_context_flag = FALSE; + H5F_set_coll_metadata_reads(idx_info.f, &md_reads_file_flag, &md_reads_context_flag); + restore_md_reads_state = TRUE; } #endif /* H5_HAVE_PARALLEL */ @@ -3335,8 +3342,8 @@ H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udat done: #ifdef H5_HAVE_PARALLEL /* Re-enable collective metadata reads if we disabled them */ - if (reenable_coll_md_reads) - H5CX_set_coll_metadata_read(TRUE); + if (restore_md_reads_state) + H5F_set_coll_metadata_reads(dset->oloc.file, &md_reads_file_flag, &md_reads_context_flag); #endif /* H5_HAVE_PARALLEL */ FUNC_LEAVE_NOAPI(ret_value) @@ -3648,7 +3655,7 @@ H5D__chunk_cache_prune(const H5D_t *dset, size_t size) * traversing the list when pointer pN reaches wN percent of the original * list. In other words, preemption method N gets to consider entries in * approximate least recently used order w0 percent before method N+1 - * where 100% means tha method N will run to completion before method N+1 + * where 100% means the method N will run to completion before method N+1 * begins. The pointers participating in the list traversal are each * given a chance at preemption before any of the pointers are advanced. */ @@ -4352,8 +4359,8 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const hbool_t blocks_written = FALSE; /* Flag to indicate that chunk was actually written */ hbool_t using_mpi = FALSE; /* Flag to indicate that the file is being accessed with an MPI-capable file driver */ - H5D_chunk_coll_info_t chunk_info; /* chunk address information for doing I/O */ -#endif /* H5_HAVE_PARALLEL */ + H5D_chunk_coll_fill_info_t chunk_fill_info; /* chunk address information for doing I/O */ +#endif /* H5_HAVE_PARALLEL */ hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ unsigned space_ndims; /* Dataset's space rank */ const hsize_t * space_dim; /* Dataset's dataspace dimensions */ @@ -4400,8 +4407,8 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const using_mpi = TRUE; /* init chunk info stuff for collective I/O */ - chunk_info.num_io = 0; - chunk_info.addr = NULL; + chunk_fill_info.num_chunks = 0; + chunk_fill_info.chunk_info = NULL; } /* end if */ #endif /* H5_HAVE_PARALLEL */ @@ -4673,19 +4680,26 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const if (using_mpi) { /* collect all chunk addresses to be written to write collectively at the end */ - /* allocate/resize address array if no more space left */ - /* Note that if we add support for parallel filters we must - * also store an array of chunk sizes and pass it to the - * apporpriate collective write function */ - if (0 == chunk_info.num_io % 1024) - if (NULL == (chunk_info.addr = (haddr_t *)H5MM_realloc( - chunk_info.addr, (chunk_info.num_io + 1024) * sizeof(haddr_t)))) + + /* allocate/resize chunk info array if no more space left */ + if (0 == chunk_fill_info.num_chunks % 1024) { + void *tmp_realloc; + + if (NULL == (tmp_realloc = H5MM_realloc(chunk_fill_info.chunk_info, + (chunk_fill_info.num_chunks + 1024) * + sizeof(struct chunk_coll_fill_info)))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "memory allocation failed for chunk addresses") + "memory allocation failed for chunk fill info") + + chunk_fill_info.chunk_info = tmp_realloc; + } - /* Store the chunk's address for later */ - chunk_info.addr[chunk_info.num_io] = udata.chunk_block.offset; - chunk_info.num_io++; + /* Store info about the chunk for later */ + chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].addr = udata.chunk_block.offset; + chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].chunk_size = chunk_size; + chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].unfiltered_partial_chunk = + (*fill_buf == unfilt_fill_buf); + chunk_fill_info.num_chunks++; /* Indicate that blocks will be written */ blocks_written = TRUE; @@ -4758,7 +4772,7 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const #ifdef H5_HAVE_PARALLEL /* do final collective I/O */ if (using_mpi && blocks_written) - if (H5D__chunk_collective_fill(dset, &chunk_info, chunk_size, fb_info.fill_buf) < 0) + if (H5D__chunk_collective_fill(dset, &chunk_fill_info, fb_info.fill_buf, unfilt_fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") #endif /* H5_HAVE_PARALLEL */ @@ -4774,8 +4788,8 @@ done: unfilt_fill_buf = H5D__chunk_mem_xfree(unfilt_fill_buf, &def_pline); #ifdef H5_HAVE_PARALLEL - if (using_mpi && chunk_info.addr) - H5MM_free(chunk_info.addr); + if (using_mpi && chunk_fill_info.chunk_info) + H5MM_free(chunk_fill_info.chunk_info); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -4969,27 +4983,35 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, - const void *fill_buf) +H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, + const void *fill_buf, const void *partial_chunk_fill_buf) { - MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ - int mpi_rank = (-1); /* This process's rank */ - int mpi_size = (-1); /* MPI Comm size */ - int mpi_code; /* MPI return code */ - size_t num_blocks; /* Number of blocks between processes. */ - size_t leftover_blocks; /* Number of leftover blocks to handle */ - int blocks, leftover, block_len; /* converted to int for MPI */ + MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ + int mpi_rank = (-1); /* This process's rank */ + int mpi_size = (-1); /* MPI Comm size */ + int mpi_code; /* MPI return code */ + size_t num_blocks; /* Number of blocks between processes. */ + size_t leftover_blocks; /* Number of leftover blocks to handle */ + int blocks, leftover; /* converted to int for MPI */ MPI_Aint * chunk_disp_array = NULL; + MPI_Aint * block_disps = NULL; int * block_lens = NULL; MPI_Datatype mem_type = MPI_BYTE, file_type = MPI_BYTE; H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ hbool_t have_xfer_mode = FALSE; /* Whether the previous xffer mode has been retrieved */ - hbool_t need_addr_sort = FALSE; - int i; /* Local index variable */ + hbool_t need_sort = FALSE; + size_t i; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC + /* + * If a separate fill buffer is provided for partial chunks, ensure + * that the "don't filter partial edge chunks" flag is set. + */ + if (partial_chunk_fill_buf) + HDassert(dset->shared->layout.u.chunk.flags & H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS); + /* Get the MPI communicator */ if (MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(dset->oloc.file))) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI communicator") @@ -5005,39 +5027,89 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, /* Distribute evenly the number of blocks between processes. */ if (mpi_size == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Resulted in division by zero") - num_blocks = (size_t)(chunk_info->num_io / (size_t)mpi_size); /* value should be the same on all procs */ + num_blocks = + (size_t)(chunk_fill_info->num_chunks / (size_t)mpi_size); /* value should be the same on all procs */ /* After evenly distributing the blocks between processes, are there any * leftover blocks for each individual process (round-robin)? */ - leftover_blocks = (size_t)(chunk_info->num_io % (size_t)mpi_size); + leftover_blocks = (size_t)(chunk_fill_info->num_chunks % (size_t)mpi_size); /* Cast values to types needed by MPI */ H5_CHECKED_ASSIGN(blocks, int, num_blocks, size_t); H5_CHECKED_ASSIGN(leftover, int, leftover_blocks, size_t); - H5_CHECKED_ASSIGN(block_len, int, chunk_size, size_t); /* Check if we have any chunks to write on this rank */ if (num_blocks > 0 || (leftover && leftover > mpi_rank)) { + MPI_Aint partial_fill_buf_disp = 0; + hbool_t all_same_block_len = TRUE; + /* Allocate buffers */ - /* (MSC - should not need block_lens if MPI_type_create_hindexed_block is working) */ - if (NULL == (block_lens = (int *)H5MM_malloc((size_t)(blocks + 1) * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk lengths buffer") if (NULL == (chunk_disp_array = (MPI_Aint *)H5MM_malloc((size_t)(blocks + 1) * sizeof(MPI_Aint)))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk file displacement buffer") - for (i = 0; i < blocks; i++) { - /* store the chunk address as an MPI_Aint */ - chunk_disp_array[i] = (MPI_Aint)(chunk_info->addr[i + (mpi_rank * blocks)]); + if (partial_chunk_fill_buf) { + MPI_Aint fill_buf_addr; + MPI_Aint partial_fill_buf_addr; + + /* Calculate the displacement between the fill buffer and partial chunk fill buffer */ + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(fill_buf, &fill_buf_addr))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(partial_chunk_fill_buf, &partial_fill_buf_addr))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) - /* MSC - should not need this if MPI_type_create_hindexed_block is working */ - block_lens[i] = block_len; +#if MPI_VERSION >= 3 && MPI_SUBVERSION >= 1 + partial_fill_buf_disp = MPI_Aint_diff(partial_fill_buf_addr, fill_buf_addr); +#else + partial_fill_buf_disp = partial_fill_buf_addr - fill_buf_addr; +#endif - /* Make sure that the addresses in the datatype are - * monotonically non-decreasing + /* + * Allocate all-zero block displacements array. If a block's displacement + * is left as zero, that block will be written to from the regular fill + * buffer. If a block represents an unfiltered partial edge chunk, its + * displacement will be set so that the block is written to from the + * unfiltered fill buffer. */ - if (i && (chunk_disp_array[i] < chunk_disp_array[i - 1])) - need_addr_sort = TRUE; + if (NULL == (block_disps = (MPI_Aint *)H5MM_calloc((size_t)(blocks + 1) * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate block displacements buffer") + } + + /* + * Perform initial scan of chunk info list to: + * - make sure that chunk addresses are monotonically non-decreasing + * - check if all blocks have the same length + */ + for (i = 1; i < chunk_fill_info->num_chunks; i++) { + if (chunk_fill_info->chunk_info[i].addr < chunk_fill_info->chunk_info[i - 1].addr) + need_sort = TRUE; + + if (chunk_fill_info->chunk_info[i].chunk_size != chunk_fill_info->chunk_info[i - 1].chunk_size) + all_same_block_len = FALSE; + } + + if (need_sort) + HDqsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, + sizeof(struct chunk_coll_fill_info), H5D__chunk_cmp_coll_fill_info); + + /* Allocate buffer for block lengths if necessary */ + if (!all_same_block_len) + if (NULL == (block_lens = (int *)H5MM_malloc((size_t)(blocks + 1) * sizeof(int)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk lengths buffer") + + for (i = 0; i < (size_t)blocks; i++) { + size_t idx = i + (size_t)(mpi_rank * blocks); + + /* store the chunk address as an MPI_Aint */ + chunk_disp_array[i] = (MPI_Aint)(chunk_fill_info->chunk_info[idx].addr); + + if (!all_same_block_len) + H5_CHECKED_ASSIGN(block_lens[i], int, chunk_fill_info->chunk_info[idx].chunk_size, size_t); + + if (chunk_fill_info->chunk_info[idx].unfiltered_partial_chunk) { + HDassert(partial_chunk_fill_buf); + block_disps[i] = partial_fill_buf_disp; + } } /* end for */ /* Calculate if there are any leftover blocks after evenly @@ -5045,32 +5117,71 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, * to processes 0 -> leftover. */ if (leftover && leftover > mpi_rank) { - chunk_disp_array[blocks] = (MPI_Aint)chunk_info->addr[(blocks * mpi_size) + mpi_rank]; - if (blocks && (chunk_disp_array[blocks] < chunk_disp_array[blocks - 1])) - need_addr_sort = TRUE; - block_lens[blocks] = block_len; + chunk_disp_array[blocks] = + (MPI_Aint)chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].addr; + + if (!all_same_block_len) + H5_CHECKED_ASSIGN(block_lens[blocks], int, + chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].chunk_size, + size_t); + + if (chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].unfiltered_partial_chunk) { + HDassert(partial_chunk_fill_buf); + block_disps[blocks] = partial_fill_buf_disp; + } + blocks++; } - /* Ensure that the blocks are sorted in monotonically non-decreasing - * order of offset in the file. - */ - if (need_addr_sort) - HDqsort(chunk_disp_array, (size_t)blocks, sizeof(MPI_Aint), H5D__chunk_cmp_addr); + /* Create file and memory types for the write operation */ + if (all_same_block_len) { + int block_len; + + H5_CHECKED_ASSIGN(block_len, int, chunk_fill_info->chunk_info[0].chunk_size, size_t); + + mpi_code = + MPI_Type_create_hindexed_block(blocks, block_len, chunk_disp_array, MPI_BYTE, &file_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code) + + if (partial_chunk_fill_buf) { + /* + * If filters are disabled for partial edge chunks, those chunks could + * potentially have the same block length as the other chunks, but still + * need to be written to using the unfiltered fill buffer. Use an hindexed + * block type rather than an hvector. + */ + mpi_code = + MPI_Type_create_hindexed_block(blocks, block_len, block_disps, MPI_BYTE, &mem_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code) + } + else { + mpi_code = MPI_Type_create_hvector(blocks, block_len, 0, MPI_BYTE, &mem_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) + } + } + else { + /* + * Currently, different block lengths implies that there are partial + * edge chunks and the "don't filter partial edge chunks" flag is set. + */ + HDassert(partial_chunk_fill_buf); + HDassert(block_lens); + HDassert(block_disps); + + mpi_code = MPI_Type_create_hindexed(blocks, block_lens, chunk_disp_array, MPI_BYTE, &file_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + + mpi_code = MPI_Type_create_hindexed(blocks, block_lens, block_disps, MPI_BYTE, &mem_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + } - /* MSC - should use this if MPI_type_create_hindexed block is working: - * mpi_code = MPI_Type_create_hindexed_block(blocks, block_len, chunk_disp_array, MPI_BYTE, - * &file_type); - */ - mpi_code = MPI_Type_create_hindexed(blocks, block_lens, chunk_disp_array, MPI_BYTE, &file_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - - mpi_code = MPI_Type_create_hvector(blocks, block_len, 0, MPI_BYTE, &mem_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) } /* end if */ @@ -5113,39 +5224,25 @@ done: if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) H5MM_xfree(chunk_disp_array); + H5MM_xfree(block_disps); H5MM_xfree(block_lens); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_collective_fill() */ static int -H5D__chunk_cmp_addr(const void *addr1, const void *addr2) +H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2) { - MPI_Aint _addr1 = (MPI_Aint)0, _addr2 = (MPI_Aint)0; - int ret_value = 0; + const struct chunk_coll_fill_info *entry1; + const struct chunk_coll_fill_info *entry2; FUNC_ENTER_STATIC_NOERR - _addr1 = *((const MPI_Aint *)addr1); - _addr2 = *((const MPI_Aint *)addr2); - -#if MPI_VERSION >= 3 && MPI_SUBVERSION >= 1 - { - MPI_Aint diff = MPI_Aint_diff(_addr1, _addr2); - - if (diff < (MPI_Aint)0) - ret_value = -1; - else if (diff > (MPI_Aint)0) - ret_value = 1; - else - ret_value = 0; - } -#else - ret_value = (_addr1 > _addr2) - (_addr1 < _addr2); -#endif + entry1 = (const struct chunk_coll_fill_info *)_entry1; + entry2 = (const struct chunk_coll_fill_info *)_entry2; - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__chunk_cmp_addr() */ + FUNC_LEAVE_NOAPI(H5F_addr_cmp(entry1->addr, entry2->addr)) +} /* end H5D__chunk_cmp_coll_fill_info() */ #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- @@ -5253,7 +5350,7 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk) /* The number of bytes accessed in the chunk */ /* (i.e. the bytes replaced with fill values) */ - H5_CHECK_OVERFLOW(sel_nelmts, hssize_t, uint32_t); + H5_CHECK_OVERFLOW(sel_nelmts, hsize_t, uint32_t); bytes_accessed = (uint32_t)sel_nelmts * layout->u.chunk.dim[rank]; /* Release lock on chunk */ @@ -5998,7 +6095,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) size_t buf_size = udata->buf_size; /* Size of chunk buffer */ const H5O_pline_t *pline = udata->pline; /* I/O pipeline for applying filters */ - /* needed for commpressed variable length data */ + /* needed for compressed variable length data */ hbool_t must_filter = FALSE; /* Whether chunk must be filtered during copy */ size_t nbytes; /* Size of chunk in file (in bytes) */ H5Z_cb_t filter_cb; /* Filter failure callback struct */ @@ -6729,10 +6826,10 @@ H5D__chunk_stats(const H5D_t *dset, hbool_t headers) miss_rate = 0.0; } if (miss_rate > 100) { - HDsprintf(ascii, "%7d%%", (int)(miss_rate + 0.5)); + HDsnprintf(ascii, sizeof(ascii), "%7d%%", (int)(miss_rate + 0.5)); } else { - HDsprintf(ascii, "%7.2f%%", miss_rate); + HDsnprintf(ascii, sizeof(ascii), "%7.2f%%", miss_rate); } HDfprintf(H5DEBUG(AC), " %-18s %8u %8u %7s %8d+%-9ld\n", "raw data chunks", rdcc->stats.nhits, @@ -6859,7 +6956,7 @@ done: * *------------------------------------------------------------------------- */ -static hbool_t +hbool_t H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, const hsize_t scaled[], const hsize_t *dset_dims) { @@ -7154,6 +7251,89 @@ done: } /* end H5D__chunk_format_convert() */ /*------------------------------------------------------------------------- + * Function: H5D__chunk_index_empty_cb + * + * Purpose: Callback function that simply stops iteration and sets the + * `empty` parameter to FALSE if called. If this callback is + * entered, it means that the chunk index contains at least + * one chunk, so is not empty. + * + * Return: H5_ITER_STOP + * + *------------------------------------------------------------------------- + */ +static int +H5D__chunk_index_empty_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata) +{ + hbool_t *empty = (hbool_t *)_udata; + int ret_value = H5_ITER_STOP; + + FUNC_ENTER_STATIC_NOERR + + *empty = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__chunk_index_empty_cb() */ + +/*------------------------------------------------------------------------- + * Function: H5D__chunk_index_empty + * + * Purpose: Determines whether a chunk index is empty (has no chunks + * inserted into it yet). + * + * Note: This routine is meant to be a little more performant than + * just counting the number of chunks in the index. In the + * future, this is probably a callback that the chunk index + * ops structure should provide. + * + * Return: Non-negative on Success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__chunk_index_empty(const H5D_t *dset, hbool_t *empty) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_rdcc_ent_t * ent; /* Cache entry */ + const H5D_rdcc_t * rdcc = NULL; /* Raw data chunk cache */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + HDassert(empty); + + rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */ + HDassert(rdcc); + + /* Search for cached chunks that haven't been written out */ + for (ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if (H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + *empty = TRUE; + + if (H5F_addr_defined(idx_info.storage->idx_addr)) { + /* Iterate over the allocated chunks */ + if ((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__chunk_index_empty_cb, empty) < + 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "unable to retrieve allocated chunk information from index") + } + +done: + FUNC_LEAVE_NOAPI_TAG(ret_value) +} /* end H5D__chunk_index_empty() */ + +/*------------------------------------------------------------------------- * Function: H5D__get_num_chunks_cb * * Purpose: Callback function that increments the number of written @@ -7558,7 +7738,7 @@ H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata) /*------------------------------------------------------------------------- * Function: H5D__chunk_iter * - * Purpose: Iterate over all the chunks in the dataset with given callbak. + * Purpose: Iterate over all the chunks in the dataset with given callback. * * Return: Success: Non-negative * Failure: Negative diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index fe41298..356a54e 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -47,6 +47,15 @@ /* Local Typedefs */ /******************/ +/* Callback info for I/O operation when file driver + * wishes to do its own memory management + */ +typedef struct H5D_compact_iovv_memmanage_ud_t { + H5F_shared_t *f_sh; /* Shared file for dataset */ + void * dstbuf; /* Pointer to buffer to be read into/written into */ + const void * srcbuf; /* Pointer to buffer to be read from/written from */ +} H5D_compact_iovv_memmanage_ud_t; + /********************/ /* Local Prototypes */ /********************/ @@ -55,8 +64,8 @@ static herr_t H5D__compact_construct(H5F_t *f, H5D_t *dset); static hbool_t H5D__compact_is_space_alloc(const H5O_storage_t *storage); static herr_t H5D__compact_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, - H5D_chunk_map_t *cm); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *cm); +static herr_t H5D__compact_iovv_memmanage_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata); static ssize_t H5D__compact_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[]); @@ -71,13 +80,24 @@ static herr_t H5D__compact_dest(H5D_t *dset); /*********************/ /* Compact storage layout I/O ops */ -const H5D_layout_ops_t H5D_LOPS_COMPACT[1] = { - {H5D__compact_construct, NULL, H5D__compact_is_space_alloc, NULL, H5D__compact_io_init, H5D__contig_read, - H5D__contig_write, +const H5D_layout_ops_t H5D_LOPS_COMPACT[1] = {{ + H5D__compact_construct, /* construct */ + NULL, /* init */ + H5D__compact_is_space_alloc, /* is_space_alloc */ + NULL, /* is_data_cached */ + H5D__compact_io_init, /* io_init */ + H5D__contig_read, /* ser_read */ + H5D__contig_write, /* ser_write */ #ifdef H5_HAVE_PARALLEL - NULL, NULL, -#endif /* H5_HAVE_PARALLEL */ - H5D__compact_readvv, H5D__compact_writevv, H5D__compact_flush, NULL, H5D__compact_dest}}; + NULL, /* par_read */ + NULL, /* par_write */ +#endif + H5D__compact_readvv, /* readvv */ + H5D__compact_writevv, /* writevv */ + H5D__compact_flush, /* flush */ + NULL, /* io_term */ + H5D__compact_dest /* dest */ +}}; /*******************/ /* Local Variables */ @@ -228,8 +248,8 @@ H5D__compact_is_space_alloc(const H5O_storage_t H5_ATTR_UNUSED *storage) */ static herr_t H5D__compact_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *cm) + hsize_t H5_ATTR_UNUSED nelmts, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *cm) { FUNC_ENTER_STATIC_NOERR @@ -240,6 +260,48 @@ H5D__compact_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR } /* end H5D__compact_io_init() */ /*------------------------------------------------------------------------- + * Function: H5D__compact_iovv_memmanage_cb + * + * Purpose: Callback operator for H5D__compact_readvv()/_writevv() to + * send a memory copy request to the underlying file driver. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__compact_iovv_memmanage_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata) +{ + H5D_compact_iovv_memmanage_ud_t *udata = (H5D_compact_iovv_memmanage_ud_t *)_udata; + H5FD_ctl_memcpy_args_t op_args; + uint64_t op_flags; + H5FD_t * file_handle = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Retrieve pointer to file driver structure for ctl call */ + if (H5F_shared_get_file_driver(udata->f_sh, &file_handle) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "can't get file handle") + + /* Setup operation flags and arguments */ + op_flags = H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG | H5FD_CTL__FAIL_IF_UNKNOWN_FLAG; + + op_args.dstbuf = udata->dstbuf; + op_args.dst_off = dst_off; + op_args.srcbuf = udata->srcbuf; + op_args.src_off = src_off; + op_args.len = len; + + /* Make request to file driver */ + if (H5FD_ctl(file_handle, H5FD_CTL__MEM_COPY, op_flags, &op_args, NULL) < 0) + HGOTO_ERROR(H5E_IO, H5E_FCNTL, FAIL, "VFD memcpy request failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__compact_iovv_memmanage_cb() */ + +/*------------------------------------------------------------------------- * Function: H5D__compact_readvv * * Purpose: Reads some data vectors from a dataset into a buffer. @@ -268,11 +330,28 @@ H5D__compact_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t * HDassert(io_info); - /* Use the vectorized memory copy routine to do actual work */ - if ((ret_value = H5VM_memcpyvv(io_info->u.rbuf, mem_max_nseq, mem_curr_seq, mem_size_arr, mem_offset_arr, - io_info->store->compact.buf, dset_max_nseq, dset_curr_seq, dset_size_arr, - dset_offset_arr)) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + /* Check if file driver wishes to do its own memory management */ + if (H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_MEMMANAGE)) { + H5D_compact_iovv_memmanage_ud_t udata; + + /* Set up udata for memory copy operation */ + udata.f_sh = io_info->f_sh; + udata.dstbuf = io_info->u.rbuf; + udata.srcbuf = io_info->store->compact.buf; + + /* Request that file driver does the memory copy */ + if ((ret_value = H5VM_opvv(mem_max_nseq, mem_curr_seq, mem_size_arr, mem_offset_arr, dset_max_nseq, + dset_curr_seq, dset_size_arr, dset_offset_arr, + H5D__compact_iovv_memmanage_cb, &udata)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + } + else { + /* Use the vectorized memory copy routine to do actual work */ + if ((ret_value = H5VM_memcpyvv(io_info->u.rbuf, mem_max_nseq, mem_curr_seq, mem_size_arr, + mem_offset_arr, io_info->store->compact.buf, dset_max_nseq, + dset_curr_seq, dset_size_arr, dset_offset_arr)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -310,11 +389,28 @@ H5D__compact_writevv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t HDassert(io_info); - /* Use the vectorized memory copy routine to do actual work */ - if ((ret_value = H5VM_memcpyvv(io_info->store->compact.buf, dset_max_nseq, dset_curr_seq, dset_size_arr, - dset_offset_arr, io_info->u.wbuf, mem_max_nseq, mem_curr_seq, mem_size_arr, - mem_offset_arr)) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + /* Check if file driver wishes to do its own memory management */ + if (H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_MEMMANAGE)) { + H5D_compact_iovv_memmanage_ud_t udata; + + /* Set up udata for memory copy operation */ + udata.f_sh = io_info->f_sh; + udata.dstbuf = io_info->store->compact.buf; + udata.srcbuf = io_info->u.wbuf; + + /* Request that file driver does the memory copy */ + if ((ret_value = H5VM_opvv(dset_max_nseq, dset_curr_seq, dset_size_arr, dset_offset_arr, mem_max_nseq, + mem_curr_seq, mem_size_arr, mem_offset_arr, H5D__compact_iovv_memmanage_cb, + &udata)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + } + else { + /* Use the vectorized memory copy routine to do actual work */ + if ((ret_value = H5VM_memcpyvv(io_info->store->compact.buf, dset_max_nseq, dset_curr_seq, + dset_size_arr, dset_offset_arr, io_info->u.wbuf, mem_max_nseq, + mem_curr_seq, mem_size_arr, mem_offset_arr)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + } /* Mark the compact dataset's buffer as dirty */ *io_info->store->compact.dirty = TRUE; diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 4dc6f72..3828e8e 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -91,8 +91,7 @@ typedef struct H5D_contig_writevv_ud_t { static herr_t H5D__contig_construct(H5F_t *f, H5D_t *dset); static herr_t H5D__contig_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id); static herr_t H5D__contig_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, - H5D_chunk_map_t *cm); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *cm); static ssize_t H5D__contig_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]); @@ -109,13 +108,24 @@ static herr_t H5D__contig_write_one(H5D_io_info_t *io_info, hsize_t offset, size /*********************/ /* Contiguous storage layout I/O ops */ -const H5D_layout_ops_t H5D_LOPS_CONTIG[1] = { - {H5D__contig_construct, H5D__contig_init, H5D__contig_is_space_alloc, H5D__contig_is_data_cached, - H5D__contig_io_init, H5D__contig_read, H5D__contig_write, +const H5D_layout_ops_t H5D_LOPS_CONTIG[1] = {{ + H5D__contig_construct, /* construct */ + H5D__contig_init, /* init */ + H5D__contig_is_space_alloc, /* is_space_alloc */ + H5D__contig_is_data_cached, /* is_data_cached */ + H5D__contig_io_init, /* io_init */ + H5D__contig_read, /* ser_read */ + H5D__contig_write, /* ser_write */ #ifdef H5_HAVE_PARALLEL - H5D__contig_collective_read, H5D__contig_collective_write, -#endif /* H5_HAVE_PARALLEL */ - H5D__contig_readvv, H5D__contig_writevv, H5D__contig_flush, NULL, NULL}}; + H5D__contig_collective_read, /* par_read */ + H5D__contig_collective_write, /* par_write */ +#endif + H5D__contig_readvv, /* readvv */ + H5D__contig_writevv, /* writevv */ + H5D__contig_flush, /* flush */ + NULL, /* io_term */ + NULL /* dest */ +}}; /*******************/ /* Local Variables */ @@ -268,9 +278,16 @@ H5D__contig_fill(const H5D_io_info_t *io_info) if (using_mpi) { /* Write the chunks out from only one process */ /* !! Use the internal "independent" DXPL!! -QAK */ - if (H5_PAR_META_WRITE == mpi_rank) - if (H5D__contig_write_one(&ioinfo, offset, size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to write fill value to dataset") + if (H5_PAR_META_WRITE == mpi_rank) { + if (H5D__contig_write_one(&ioinfo, offset, size) < 0) { + /* If writing fails, push an error and stop writing, but + * still participate in following MPI_Barrier. + */ + blocks_written = TRUE; + HDONE_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to write fill value to dataset") + break; + } + } /* Indicate that blocks are being written */ blocks_written = TRUE; @@ -550,8 +567,8 @@ H5D__contig_is_data_cached(const H5D_shared_t *shared_dset) */ static herr_t H5D__contig_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *cm) + hsize_t H5_ATTR_UNUSED nelmts, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *cm) { FUNC_ENTER_STATIC_NOERR @@ -574,8 +591,8 @@ H5D__contig_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_ *------------------------------------------------------------------------- */ herr_t -H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) +H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, H5S_t *file_space, + H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { herr_t ret_value = SUCCEED; /*return value */ @@ -609,8 +626,8 @@ done: *------------------------------------------------------------------------- */ herr_t -H5D__contig_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) +H5D__contig_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, H5S_t *file_space, + H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { herr_t ret_value = SUCCEED; /*return value */ diff --git a/src/H5Dearray.c b/src/H5Dearray.c index bef453d..0174a76 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -418,7 +418,7 @@ H5D__earray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const void HDassert(elmt); /* Print element */ - HDsprintf(temp_str, "Element #%" PRIuHSIZE ":", idx); + HDsnprintf(temp_str, sizeof(temp_str), "Element #%" PRIuHSIZE ":", idx); HDfprintf(stream, "%*s%-*s %" PRIuHADDR "\n", indent, "", fwidth, temp_str, *(const haddr_t *)elmt); FUNC_LEAVE_NOAPI(SUCCEED) @@ -574,7 +574,7 @@ H5D__earray_filt_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const HDassert(elmt); /* Print element */ - HDsprintf(temp_str, "Element #%" PRIuHSIZE ":", idx); + HDsnprintf(temp_str, sizeof(temp_str), "Element #%" PRIuHSIZE ":", idx); HDfprintf(stream, "%*s%-*s {%" PRIuHADDR ", %u, %0x}\n", indent, "", fwidth, temp_str, elmt->addr, elmt->nbytes, elmt->filter_mask); diff --git a/src/H5Defl.c b/src/H5Defl.c index 85c9dba..a30955b 100644 --- a/src/H5Defl.c +++ b/src/H5Defl.c @@ -62,7 +62,7 @@ typedef struct H5D_efl_writevv_ud_t { /* Layout operation callbacks */ static herr_t H5D__efl_construct(H5F_t *f, H5D_t *dset); static herr_t H5D__efl_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *cm); + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *cm); static ssize_t H5D__efl_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]); @@ -80,12 +80,24 @@ static herr_t H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t ad /*********************/ /* External File List (EFL) storage layout I/O ops */ -const H5D_layout_ops_t H5D_LOPS_EFL[1] = {{H5D__efl_construct, NULL, H5D__efl_is_space_alloc, NULL, - H5D__efl_io_init, H5D__contig_read, H5D__contig_write, +const H5D_layout_ops_t H5D_LOPS_EFL[1] = {{ + H5D__efl_construct, /* construct */ + NULL, /* init */ + H5D__efl_is_space_alloc, /* is_space_alloc */ + NULL, /* is_data_cached */ + H5D__efl_io_init, /* io_init */ + H5D__contig_read, /* ser_read */ + H5D__contig_write, /* ser_write */ #ifdef H5_HAVE_PARALLEL - NULL, NULL, -#endif /* H5_HAVE_PARALLEL */ - H5D__efl_readvv, H5D__efl_writevv, NULL, NULL, NULL}}; + NULL, /* par_read */ + NULL, /* par_write */ +#endif + H5D__efl_readvv, /* readvv */ + H5D__efl_writevv, /* writevv */ + NULL, /* flush */ + NULL, /* io_term */ + NULL /* dest */ +}}; /*******************/ /* Local Variables */ @@ -198,8 +210,8 @@ H5D__efl_is_space_alloc(const H5O_storage_t H5_ATTR_UNUSED *storage) */ static herr_t H5D__efl_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *cm) + hsize_t H5_ATTR_UNUSED nelmts, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *cm) { FUNC_ENTER_STATIC_NOERR diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c index 8293ec4..8c031e4 100644 --- a/src/H5Dfarray.c +++ b/src/H5Dfarray.c @@ -416,7 +416,7 @@ H5D__farray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const void HDassert(elmt); /* Print element */ - HDsprintf(temp_str, "Element #%" PRIuHSIZE ":", idx); + HDsnprintf(temp_str, sizeof(temp_str), "Element #%" PRIuHSIZE ":", idx); HDfprintf(stream, "%*s%-*s %" PRIuHADDR "\n", indent, "", fwidth, temp_str, *(const haddr_t *)elmt); FUNC_LEAVE_NOAPI(SUCCEED) diff --git a/src/H5Dfill.c b/src/H5Dfill.c index fe98487..cec7e8b 100644 --- a/src/H5Dfill.c +++ b/src/H5Dfill.c @@ -112,7 +112,7 @@ H5FL_EXTERN(H5S_sel_iter_t); on each element so that each of them has a copy of the VL data. --------------------------------------------------------------------------*/ herr_t -H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_type, const H5S_t *space) +H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_type, H5S_t *space) { H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info */ hbool_t mem_iter_init = FALSE; /* Whether the memory selection iterator has been initialized */ diff --git a/src/H5Dint.c b/src/H5Dint.c index a287ae8..15092f3 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -139,9 +139,6 @@ static const H5I_class_t H5I_DATASET_CLS[1] = {{ (H5I_free_t)H5D__close_cb /* Callback routine for closing objects of this class */ }}; -/* Flag indicating "top" of interface has been initialized */ -static hbool_t H5D_top_package_initialize_s = FALSE; - /* Prefixes of VDS and external file from the environment variables * HDF5_EXTFILE_PREFIX and HDF5_VDS_PREFIX */ static const char *H5D_prefix_ext_env = NULL; @@ -160,37 +157,10 @@ static const char *H5D_prefix_vds_env = NULL; herr_t H5D_init(void) { - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_init() */ - -/*-------------------------------------------------------------------------- -NAME - H5D__init_package -- Initialize interface-specific information -USAGE - herr_t H5D__init_package() - -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. -NOTES - Care must be taken when using the H5P functions, since they can cause - a deadlock in the library when the library is attempting to terminate -QAK - ---------------------------------------------------------------------------*/ -herr_t -H5D__init_package(void) -{ H5P_genplist_t *def_dcpl; /* Default Dataset Creation Property list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_NOAPI(FAIL) /* Initialize the ID group for the dataset IDs */ if (H5I_register_type(H5I_DATASET_CLS) < 0) @@ -220,16 +190,13 @@ H5D__init_package(void) if (H5P_get(def_dcpl, H5O_CRT_PIPELINE_NAME, &H5D_def_dset.dcpl_cache.pline) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve pipeline filter") - /* Mark "top" of interface as initialized, too */ - H5D_top_package_initialize_s = TRUE; - /* Retrieve the prefixes of VDS and external file from the environment variable */ H5D_prefix_vds_env = HDgetenv("HDF5_VDS_PREFIX"); H5D_prefix_ext_env = HDgetenv("HDF5_EXTFILE_PREFIX"); done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__init_package() */ +} /* end H5D_init() */ /*------------------------------------------------------------------------- * Function: H5D_top_term_package @@ -248,38 +215,32 @@ H5D_top_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5D_top_package_initialize_s) { - if (H5I_nmembers(H5I_DATASET) > 0) { - /* The dataset API uses the "force" flag set to true because it - * is using the "file objects" (H5FO) API functions to track open - * objects in the file. Using the H5FO code means that dataset - * IDs can have reference counts >1, when an existing dataset is - * opened more than once. However, the H5I code does not attempt - * to close objects with reference counts>1 unless the "force" flag - * is set to true. - * - * At some point (probably after the group and datatypes use the - * the H5FO code), the H5FO code might need to be switched around - * to storing pointers to the objects being tracked (H5D_t, H5G_t, - * etc) and reference count those itself instead of relying on the - * reference counting in the H5I layer. Then, the "force" flag can - * be put back to false. - * - * Setting the "force" flag to true for all the interfaces won't - * work because the "file driver" (H5FD) APIs use the H5I reference - * counting to avoid closing a file driver out from underneath an - * open file... - * - * QAK - 5/13/03 - */ - (void)H5I_clear_type(H5I_DATASET, TRUE, FALSE); - n++; /*H5I*/ - } /* end if */ - - /* Mark closed */ - if (0 == n) - H5D_top_package_initialize_s = FALSE; - } /* end if */ + if (H5I_nmembers(H5I_DATASET) > 0) { + /* The dataset API uses the "force" flag set to true because it + * is using the "file objects" (H5FO) API functions to track open + * objects in the file. Using the H5FO code means that dataset + * IDs can have reference counts >1, when an existing dataset is + * opened more than once. However, the H5I code does not attempt + * to close objects with reference counts>1 unless the "force" flag + * is set to true. + * + * At some point (probably after the group and datatypes use the + * the H5FO code), the H5FO code might need to be switched around + * to storing pointers to the objects being tracked (H5D_t, H5G_t, + * etc) and reference count those itself instead of relying on the + * reference counting in the H5I layer. Then, the "force" flag can + * be put back to false. + * + * Setting the "force" flag to true for all the interfaces won't + * work because the "file driver" (H5FD) APIs use the H5I reference + * counting to avoid closing a file driver out from underneath an + * open file... + * + * QAK - 5/13/03 + */ + (void)H5I_clear_type(H5I_DATASET, TRUE, FALSE); + n++; /*H5I*/ + } FUNC_LEAVE_NOAPI(n) } /* end H5D_top_term_package() */ @@ -304,18 +265,11 @@ H5D_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Sanity checks */ - HDassert(0 == H5I_nmembers(H5I_DATASET)); - HDassert(FALSE == H5D_top_package_initialize_s); - - /* Destroy the dataset object id group */ - n += (H5I_dec_type_ref(H5I_DATASET) > 0); + /* Sanity checks */ + HDassert(0 == H5I_nmembers(H5I_DATASET)); - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Destroy the dataset object id group */ + n += (H5I_dec_type_ref(H5I_DATASET) > 0); FUNC_LEAVE_NOAPI(n) } /* end H5D_term_package() */ @@ -410,7 +364,7 @@ done: * * Return: * Success: Non-negative - * Failture: Negative + * Failure: Negative *------------------------------------------------------------------------- */ herr_t @@ -424,40 +378,18 @@ H5D__get_space_status(const H5D_t *dset, H5D_space_status_t *allocation) /* Check for chunked layout */ if (dset->shared->layout.type == H5D_CHUNKED) { - hsize_t space_allocated; /* The number of bytes allocated for chunks */ - hssize_t snelmts; /* Temporary holder for number of elements in dataspace */ - hsize_t nelmts; /* Number of elements in dataspace */ - size_t dt_size; /* Size of datatype */ - hsize_t full_size; /* The number of bytes in the dataset when fully populated */ - - /* For chunked layout set the space status by the storage size */ - /* Get the dataset's dataspace */ - HDassert(dset->shared->space); - - /* Get the total number of elements in dataset's dataspace */ - if ((snelmts = H5S_GET_EXTENT_NPOINTS(dset->shared->space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve number of elements in dataspace") - nelmts = (hsize_t)snelmts; - - /* Get the size of the dataset's datatype */ - if (0 == (dt_size = H5T_GET_SIZE(dset->shared->type))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve size of datatype") - - /* Compute the maximum size of the dataset in bytes */ - full_size = nelmts * dt_size; - - /* Check for overflow during multiplication */ - if (nelmts != (full_size / dt_size)) - HGOTO_ERROR(H5E_DATASET, H5E_OVERFLOW, FAIL, "size of dataset's storage overflowed") - - /* Difficult to error check, since the error value is 0 and 0 is a valid value... :-/ */ - if (H5D__get_storage_size(dset, &space_allocated) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get size of dataset's storage") - - /* Decide on how much of the space is allocated */ - if (space_allocated == 0) + hsize_t n_chunks_total = dset->shared->layout.u.chunk.nchunks; + hsize_t n_chunks_alloc = 0; + + if (H5D__get_num_chunks(dset, dset->shared->space, &n_chunks_alloc) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "unable to retrieve number of allocated chunks in dataset") + + HDassert(n_chunks_alloc <= n_chunks_total); + + if (n_chunks_alloc == 0) *allocation = H5D_SPACE_STATUS_NOT_ALLOCATED; - else if (space_allocated == full_size) + else if (n_chunks_alloc == n_chunks_total) *allocation = H5D_SPACE_STATUS_ALLOCATED; else *allocation = H5D_SPACE_STATUS_PART_ALLOCATED; @@ -757,7 +689,7 @@ H5D__calculate_minimum_header_size(H5F_t *file, H5D_t *dset, H5O_t *ohdr) H5T_t * type = NULL; H5O_fill_t *fill_prop = NULL; hbool_t use_at_least_v18 = FALSE; - const char continuation[1] = ""; /* requred for work-around */ + const char continuation[1] = ""; /* required for work-around */ size_t get_value = 0; size_t ret_value = 0; @@ -804,7 +736,7 @@ H5D__calculate_minimum_header_size(H5F_t *file, H5D_t *dset, H5O_t *ohdr) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of continuation message") ret_value += get_value; - /* Fill Value (backwards compatability) message size */ + /* Fill Value (backwards compatibility) message size */ if (fill_prop->buf && !use_at_least_v18) { H5O_fill_t old_fill_prop; /* Copy for writing "old" fill value */ @@ -893,7 +825,7 @@ H5D__prepare_minimized_oh(H5F_t *file, H5D_t *dset, H5O_loc_t *oloc) if (ohdr_size == 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "computed header size is invalid") - /* Special allocation of space for compact datsets is handled by the call here. */ + /* Special allocation of space for compact datasets is handled by the call here. */ if (H5O_apply_ohdr(file, oh, dset->shared->dcpl_id, ohdr_size, (size_t)1, oloc) == FAIL) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "can't apply object header to file") @@ -1347,10 +1279,19 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set latest indexing") } /* end if */ - /* Check if this dataset is going into a parallel file and set space allocation time */ + /* Check if the file driver would like to force early space allocation */ if (H5F_HAS_FEATURE(file, H5FD_FEAT_ALLOCATE_EARLY)) new_dset->shared->dcpl_cache.fill.alloc_time = H5D_ALLOC_TIME_EARLY; + /* + * Check if this dataset is going into a parallel file and set space allocation time. + * If the dataset has filters applied to it, writes to the dataset must be collective, + * so we don't need to force early space allocation. Otherwise, we force early space + * allocation to facilitate independent raw data operations. + */ + if (H5F_HAS_FEATURE(file, H5FD_FEAT_HAS_MPI) && (new_dset->shared->dcpl_cache.pline.nused == 0)) + new_dset->shared->dcpl_cache.fill.alloc_time = H5D_ALLOC_TIME_EARLY; + /* Set the dataset's I/O operations */ if (H5D__layout_set_io_ops(new_dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize I/O operations") diff --git a/src/H5Dio.c b/src/H5Dio.c index d1861c4..e226a0a 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -82,8 +82,7 @@ H5FL_DEFINE(H5D_chunk_map_t); *------------------------------------------------------------------------- */ herr_t -H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, - void *buf /*out*/) +H5D__read(H5D_t *dataset, hid_t mem_type_id, H5S_t *mem_space, H5S_t *file_space, void *buf /*out*/) { H5D_chunk_map_t *fm = NULL; /* Chunk file<->memory mapping */ H5D_io_info_t io_info; /* Dataset I/O info */ @@ -166,7 +165,7 @@ H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t * difficulties with the notion. * * To solve this, we check to see if H5S_select_shape_same() returns true, - * and if the ranks of the mem and file spaces are different. If the are, + * and if the ranks of the mem and file spaces are different. If they are, * construct a new mem space that is equivalent to the old mem space, and * use that instead. * @@ -295,13 +294,13 @@ done: *------------------------------------------------------------------------- */ herr_t -H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, - const void *buf) +H5D__write(H5D_t *dataset, hid_t mem_type_id, H5S_t *mem_space, H5S_t *file_space, const void *buf) { H5D_chunk_map_t *fm = NULL; /* Chunk file<->memory mapping */ H5D_io_info_t io_info; /* Dataset I/O info */ H5D_type_info_t type_info; /* Datatype info for operation */ hbool_t type_info_init = FALSE; /* Whether the datatype info has been initialized */ + hbool_t should_alloc_space = FALSE; /* Whether or not to initialize dataset's storage */ H5S_t * projected_mem_space = NULL; /* If not NULL, ptr to dataspace containing a */ /* projection of the supplied mem_space to a new */ /* dataspace with rank equal to that of */ @@ -434,8 +433,20 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up I/O operation") /* Allocate dataspace and initialize it if it hasn't been. */ - if (nelmts > 0 && dataset->shared->dcpl_cache.efl.nused == 0 && - !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage)) { + should_alloc_space = dataset->shared->dcpl_cache.efl.nused == 0 && + !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage); + + /* + * If not using an MPI-based VFD, we only need to allocate + * and initialize storage if there's a selection in the + * dataset's dataspace. Otherwise, we always need to participate + * in the storage allocation since this may use collective + * operations and we will hang if we don't participate. + */ + if (!H5F_HAS_FEATURE(dataset->oloc.file, H5FD_FEAT_HAS_MPI)) + should_alloc_space = should_alloc_space && (nelmts > 0); + + if (should_alloc_space) { hssize_t file_nelmts; /* Number of elements in file dataset's dataspace */ hbool_t full_overwrite; /* Whether we are over-writing all the elements */ @@ -810,86 +821,35 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, const H5S_t *file_ io_info->io_ops.single_write = H5D__mpio_select_write; } /* end if */ else { - int comm_size = 0; - - /* Retrieve size of MPI communicator used for file */ - if ((comm_size = H5F_shared_mpi_get_size(io_info->f_sh)) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator size") - /* Check if there are any filters in the pipeline. If there are, * we cannot break to independent I/O if this is a write operation * with multiple ranks involved; otherwise, there will be metadata * inconsistencies in the file. */ - if (comm_size > 1 && io_info->op_type == H5D_IO_OP_WRITE && - io_info->dset->shared->dcpl_cache.pline.nused > 0) { - H5D_mpio_no_collective_cause_t cause; - uint32_t local_no_collective_cause; - uint32_t global_no_collective_cause; - hbool_t local_error_message_previously_written = FALSE; - hbool_t global_error_message_previously_written = FALSE; - size_t idx; - size_t cause_strings_len; - char local_no_collective_cause_string[512] = ""; - char global_no_collective_cause_string[512] = ""; - const char * cause_strings[] = { - "independent I/O was requested", - "datatype conversions were required", - "data transforms needed to be applied", - "optimized MPI types flag wasn't set", - "one of the dataspaces was neither simple nor scalar", - "dataset was not contiguous or chunked", - "parallel writes to filtered datasets are disabled", - "an error occurred while checking if collective I/O was possible"}; - - cause_strings_len = sizeof(cause_strings) / sizeof(cause_strings[0]); - - if (H5CX_get_mpio_local_no_coll_cause(&local_no_collective_cause) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, - "unable to get local no collective cause value") - if (H5CX_get_mpio_global_no_coll_cause(&global_no_collective_cause) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, - "unable to get global no collective cause value") - - /* Append each of the "reason for breaking collective I/O" error messages to the - * local and global no collective cause strings */ - for (cause = 1, idx = 0; - (cause < H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE) && (idx < cause_strings_len); - cause <<= 1, idx++) { - size_t cause_strlen = HDstrlen(cause_strings[idx]); - - if (cause & local_no_collective_cause) { - /* Check if there were any previous error messages included. If so, prepend a - * semicolon to separate the messages. - */ - if (local_error_message_previously_written) - HDstrncat(local_no_collective_cause_string, "; ", 2); - - HDstrncat(local_no_collective_cause_string, cause_strings[idx], cause_strlen); - - local_error_message_previously_written = TRUE; - } /* end if */ - - if (cause & global_no_collective_cause) { - /* Check if there were any previous error messages included. If so, prepend a - * semicolon to separate the messages. - */ - if (global_error_message_previously_written) - HDstrncat(global_no_collective_cause_string, "; ", 2); - - HDstrncat(global_no_collective_cause_string, cause_strings[idx], cause_strlen); - - global_error_message_previously_written = TRUE; - } /* end if */ - } /* end for */ - - HGOTO_ERROR(H5E_IO, H5E_NO_INDEPENDENT, FAIL, - "Can't perform independent write with filters in pipeline.\n" - " The following caused a break from collective I/O:\n" - " Local causes: %s\n" - " Global causes: %s", - local_no_collective_cause_string, global_no_collective_cause_string); - } /* end if */ + if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) { + int comm_size = 0; + + /* Retrieve size of MPI communicator used for file */ + if ((comm_size = H5F_shared_mpi_get_size(io_info->f_sh)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator size") + + if (comm_size > 1) { + char local_no_coll_cause_string[512]; + char global_no_coll_cause_string[512]; + + if (H5D__mpio_get_no_coll_cause_strings(local_no_coll_cause_string, 512, + global_no_coll_cause_string, 512) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't get reasons for breaking collective I/O") + + HGOTO_ERROR(H5E_IO, H5E_NO_INDEPENDENT, FAIL, + "Can't perform independent write with filters in pipeline.\n" + " The following caused a break from collective I/O:\n" + " Local causes: %s\n" + " Global causes: %s", + local_no_coll_cause_string, global_no_coll_cause_string); + } + } /* If we won't be doing collective I/O, but the user asked for * collective I/O, change the request to use independent I/O diff --git a/src/H5Dlayout.c b/src/H5Dlayout.c index 6c4fc12..6fdec05 100644 --- a/src/H5Dlayout.c +++ b/src/H5Dlayout.c @@ -213,7 +213,7 @@ H5D__layout_meta_size(const H5F_t *f, const H5O_layout_t *layout, hbool_t includ ret_value++; /* Dimension sizes */ - ret_value += layout->u.chunk.ndims * layout->u.chunk.enc_bytes_per_dim; + ret_value += layout->u.chunk.ndims * (size_t)layout->u.chunk.enc_bytes_per_dim; /* Type of chunk index */ ret_value++; diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 448e92d..527fc7b 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -36,6 +36,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" /* Object headers */ @@ -43,6 +44,15 @@ #include "H5Sprivate.h" /* Dataspaces */ #include "H5VMprivate.h" /* Vector */ +/* uthash is an external, header-only hash table implementation. + * + * We include the file directly in src/ and #define a few functions + * to use our internal memory calls. + */ +#define uthash_malloc(sz) H5MM_malloc(sz) +#define uthash_free(ptr, sz) H5MM_free(ptr) /* Ignoring sz is intentional */ +#include "uthash.h" + #ifdef H5_HAVE_PARALLEL /****************/ @@ -81,9 +91,54 @@ /* Macros to represent the regularity of the selection for multiple chunk IO case. */ #define H5D_CHUNK_SELECT_REG 1 +/* + * Threshold value for redistributing shared filtered chunks + * on all MPI ranks, or just MPI rank 0 + */ +#define H5D_CHUNK_REDISTRIBUTE_THRES ((size_t)((25 * H5_MB) / sizeof(H5D_chunk_redistribute_info_t))) + +/* + * Initial allocation size for the arrays that hold + * buffers for chunk modification data that is sent + * to other ranks and the MPI_Request objects for + * those send operations + */ +#define H5D_CHUNK_NUM_SEND_MSGS_INIT 64 + +/* + * Define a tag value for the MPI messages sent/received for + * chunk modification data + */ +#define H5D_CHUNK_MOD_DATA_TAG 64 + +/* + * Macro to initialize a H5D_chk_idx_info_t + * structure, given a pointer to a H5D_io_info_t + * structure + */ +#define H5D_MPIO_INIT_CHUNK_IDX_INFO(index_info, io_info_ptr) \ + do { \ + index_info.f = (io_info_ptr)->dset->oloc.file; \ + index_info.pline = &((io_info_ptr)->dset->shared->dcpl_cache.pline); \ + index_info.layout = &((io_info_ptr)->dset->shared->layout.u.chunk); \ + index_info.storage = &((io_info_ptr)->dset->shared->layout.storage.u.chunk); \ + } while (0) + +/* + * Macro to initialize a H5D_chunk_ud_t structure + * given a pointer to a H5D_chk_idx_info_t structure + */ +#define H5D_MPIO_INIT_CHUNK_UD_INFO(chunk_ud, index_info_ptr) \ + do { \ + HDmemset(&chunk_ud, 0, sizeof(H5D_chunk_ud_t)); \ + chunk_ud.common.layout = (index_info_ptr)->layout; \ + chunk_ud.common.storage = (index_info_ptr)->storage; \ + } while (0) + /******************/ /* Local Typedefs */ /******************/ + /* Combine chunk address and chunk info into a struct for better performance. */ typedef struct H5D_chunk_addr_info_t { haddr_t chunk_addr; @@ -100,115 +155,137 @@ typedef enum H5D_mpio_no_rank0_bcast_cause_t { } H5D_mpio_no_rank0_bcast_cause_t; /* + * Information necessary for re-allocating file space for a chunk + * during a parallel write of a chunked dataset with filters + * applied. + */ +typedef struct H5D_chunk_alloc_info_t { + H5F_block_t chunk_current; + H5F_block_t chunk_new; + hsize_t chunk_idx; +} H5D_chunk_alloc_info_t; + +/* + * Information for a chunk pertaining to the dataset's chunk + * index entry for the chunk + */ +typedef struct H5D_chunk_index_info_t { + hsize_t chunk_idx; + unsigned filter_mask; + hbool_t need_insert; +} H5D_chunk_index_info_t; + +/* * Information about a single chunk when performing collective filtered I/O. All * of the fields of one of these structs are initialized at the start of collective - * filtered I/O in the function H5D__construct_filtered_io_info_list(). - * - * This struct's fields are as follows: - * - * index - The "Index" of the chunk in the dataset. The index of a chunk is used during - * the collective re-insertion of chunks into the chunk index after the collective - * I/O has been performed. - * - * scaled - The scaled coordinates of the chunk in the dataset's file dataspace. The - * coordinates are used in both the collective re-allocation of space in the file - * and the collective re-insertion of chunks into the chunk index after the collective - * I/O has been performed. + * filtered I/O in the function H5D__mpio_collective_filtered_chunk_io_setup(). This + * struct's fields are as follows: * - * full_overwrite - A flag which determines whether or not a chunk needs to be read from the - * file when being updated. If a chunk is being fully overwritten (the entire - * extent is selected in its file dataspace), then it is not necessary to - * read the chunk from the file. However, if the chunk is not being fully - * overwritten, it has to be read from the file in order to update the chunk - * without trashing the parts of the chunk that are not selected. + * index_info - A structure containing the information needed when collectively + * re-inserting the chunk into the dataset's chunk index. The structure + * is distributed to all ranks during the re-insertion operation. Its fields + * are as follows: * - * num_writers - The total number of processors writing to this chunk. This field is used - * when the new owner of a chunk is receiving messages, which contain selections in - * the chunk and data to update the chunk with, from other processors which have this - * chunk selected in the I/O operation. The new owner must know how many processors it - * should expect messages from so that it can post an equal number of receive calls. + * chunk_idx - The index of the chunk in the dataset's chunk index. * - * io_size - The total size of I/O to this chunk. This field is an accumulation of the size of - * I/O to the chunk from each processor which has the chunk selected and is used to - * determine the value for the previous full_overwrite flag. + * filter_mask - A bit-mask that indicates which filters are to be applied to the + * chunk. Each filter in a chunk's filter pipeline has a bit position + * that can be masked to disable that particular filter for the chunk. + * This filter mask is saved alongside the chunk in the file. * - * buf - A pointer which serves the dual purpose of holding either the chunk data which is to be - * written to the file or the chunk data which has been read from the file. + * need_insert - A flag which determines whether or not a chunk needs to be re-inserted into + * the chunk index after the write operation. * - * chunk_states - In the case of dataset writes only, this struct is used to track a chunk's size and - * address in the file before and after the filtering operation has occurred. + * chunk_info - A pointer to the chunk's H5D_chunk_info_t structure, which contains useful + * information like the dataspaces containing the selection in the chunk. * - * Its fields are as follows: + * chunk_current - The address in the file and size of this chunk before the filtering + * operation. When reading a chunk from the file, this field is used to + * read the correct amount of bytes. It is also used when redistributing + * shared chunks among MPI ranks and as a parameter to the chunk file + * space reallocation function. * - * chunk_current - The address in the file and size of this chunk before the filtering - * operation. When reading a chunk from the file, this field is used to - * read the correct amount of bytes. It is also used when redistributing - * shared chunks among processors and as a parameter to the chunk file - * space reallocation function. + * chunk_new - The address in the file and size of this chunk after the filtering + * operation. This field is relevant when collectively re-allocating space + * in the file for all of the chunks written to in the I/O operation, as + * their sizes may have changed after their data has been filtered. * - * new_chunk - The address in the file and size of this chunk after the filtering - * operation. This field is relevant when collectively re-allocating space - * in the file for all of the chunks written to in the I/O operation, as - * their sizes may have changed after their data has been filtered. + * need_read - A flag which determines whether or not a chunk needs to be read from the + * file. During writes, if a chunk is being fully overwritten (the entire extent + * is selected in its file dataspace), then it is not necessary to read the chunk + * from the file. However, if the chunk is not being fully overwritten, it has to + * be read from the file in order to update the chunk without trashing the parts + * of the chunk that are not selected. During reads, this field should generally + * be true, but may be false if the chunk isn't allocated, for example. * - * owners - In the case of dataset writes only, this struct is used to manage which single processor - * will ultimately write data out to the chunk. It allows the other processors to act according - * to the decision and send their selection in the chunk, as well as the data they wish - * to update the chunk with, to the processor which is writing to the chunk. + * skip_filter_pline - A flag which determines whether to skip calls to the filter pipeline + * for this chunk. This flag is mostly useful for correct handling of + * partial edge chunks when the "don't filter partial edge chunks" flag + * is set on the dataset's DCPL. * - * Its fields are as follows: + * io_size - The total size of I/O to this chunk. This field is an accumulation of the size of + * I/O to the chunk from each MPI rank which has the chunk selected and is used to + * determine the value for the previous `full_overwrite` flag. * - * original_owner - The processor which originally had this chunk selected at the beginning of - * the collective filtered I/O operation. This field is currently used when - * redistributing shared chunks among processors. + * chunk_buf_size - The size in bytes of the data buffer allocated for the chunk * - * new_owner - The processor which has been selected to perform the write to this chunk. + * orig_owner - The MPI rank which originally had this chunk selected at the beginning of + * the collective filtered I/O operation. This field is currently used when + * redistributing shared chunks among MPI ranks. * - * async_info - In the case of dataset writes only, this struct is used by the owning processor of the - * chunk in order to manage the MPI send and receive calls made between it and all of - * the other processors which have this chunk selected in the I/O operation. + * new_owner - The MPI rank which has been selected to perform the modifications to this chunk. * - * Its fields are as follows: + * num_writers - The total number of MPI ranks writing to this chunk. This field is used when + * the new owner of a chunk is receiving messages from other MPI ranks that + * contain their selections in the chunk and the data to update the chunk with. + * The new owner must know how many MPI ranks it should expect messages from so + * that it can post an equal number of receive calls. * - * receive_requests_array - An array containing one MPI_Request for each of the - * asynchronous MPI receive calls the owning processor of this - * chunk makes to another processor in order to receive that - * processor's chunk modification data and selection in the chunk. + * buf - A pointer which serves the dual purpose of holding either the chunk data which is to be + * written to the file or the chunk data which has been read from the file. * - * receive_buffer_array - An array of buffers into which the owning processor of this chunk - * will store chunk modification data and the selection in the chunk - * received from another processor. + * hh - A handle for hash tables provided by the uthash.h header * - * num_receive_requests - The number of entries in the receive_request_array and - * receive_buffer_array fields. */ typedef struct H5D_filtered_collective_io_info_t { - hsize_t index; - hsize_t scaled[H5O_LAYOUT_NDIMS]; - hbool_t full_overwrite; - size_t num_writers; - size_t io_size; - void * buf; - - struct { - H5F_block_t chunk_current; - H5F_block_t new_chunk; - } chunk_states; - - struct { - int original_owner; - int new_owner; - } owners; - - struct { - MPI_Request * receive_requests_array; - unsigned char **receive_buffer_array; - int num_receive_requests; - } async_info; + H5D_chunk_index_info_t index_info; + + H5D_chunk_info_t *chunk_info; + H5F_block_t chunk_current; + H5F_block_t chunk_new; + hbool_t need_read; + hbool_t skip_filter_pline; + size_t io_size; + size_t chunk_buf_size; + int orig_owner; + int new_owner; + int num_writers; + void * buf; + + UT_hash_handle hh; } H5D_filtered_collective_io_info_t; -/* Function pointer typedef for sort function */ -typedef int (*H5D_mpio_sort_func_cb_t)(const void *, const void *); +/* + * Information necessary for redistributing shared chunks during + * a parallel write of a chunked dataset with filters applied. + */ +typedef struct H5D_chunk_redistribute_info_t { + H5F_block_t chunk_block; + hsize_t chunk_idx; + int orig_owner; + int new_owner; + int num_writers; +} H5D_chunk_redistribute_info_t; + +/* + * Information used when re-inserting a chunk into a dataset's + * chunk index during a parallel write of a chunked dataset with + * filters applied. + */ +typedef struct H5D_chunk_insert_info_t { + H5F_block_t chunk_block; + H5D_chunk_index_info_t index_info; +} H5D_chunk_insert_info_t; /********************/ /* Local Prototypes */ @@ -216,53 +293,98 @@ typedef int (*H5D_mpio_sort_func_cb_t)(const void *, const void *); static herr_t H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm); static herr_t H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm); + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size); static herr_t H5D__multi_chunk_filtered_collective_io(H5D_io_info_t * io_info, - const H5D_type_info_t *type_info, H5D_chunk_map_t *fm); + const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size); static herr_t H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm, int sum_chunk); + H5D_chunk_map_t *fm, int sum_chunk, int mpi_rank, int mpi_size); static herr_t H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm); + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size); static herr_t H5D__inter_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5S_t *file_space, const H5S_t *mem_space); static herr_t H5D__final_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, MPI_Datatype mpi_file_type, MPI_Datatype mpi_buf_type); static herr_t H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, - H5D_chunk_addr_info_t chunk_addr_info_array[], int many_chunk_opt); + H5D_chunk_addr_info_t chunk_addr_info_array[], int many_chunk_opt, int mpi_rank, + int mpi_size); static herr_t H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assign_io_mode[], - haddr_t chunk_addr[]); + haddr_t chunk_addr[], int mpi_rank, int mpi_size); static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, int *sum_chunkf); -static herr_t H5D__construct_filtered_io_info_list(const H5D_io_info_t * io_info, - const H5D_type_info_t * type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t **chunk_list, - size_t * num_entries); -#if MPI_VERSION >= 3 -static herr_t H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t * io_info, - const H5D_type_info_t * type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t *local_chunk_array, - size_t *local_chunk_array_num_entries); -#endif -static herr_t H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries, - size_t array_entry_size, void **gathered_array, - size_t *gathered_array_num_entries, hbool_t allgather, int root, - MPI_Comm comm, int (*sort_func)(const void *, const void *)); -static herr_t H5D__mpio_filtered_collective_write_type(H5D_filtered_collective_io_info_t *chunk_list, - size_t num_entries, MPI_Datatype *new_mem_type, - hbool_t *mem_type_derived, MPI_Datatype *new_file_type, - hbool_t *file_type_derived); -static herr_t H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry, - const H5D_io_info_t * io_info, - const H5D_type_info_t * type_info, - const H5D_chunk_map_t * fm); +static herr_t H5D__mpio_collective_filtered_chunk_io_setup(const H5D_io_info_t * io_info, + const H5D_type_info_t * type_info, + const H5D_chunk_map_t * fm, + H5D_filtered_collective_io_info_t **chunk_list, + size_t *num_entries, int mpi_rank); +static herr_t H5D__mpio_redistribute_shared_chunks(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size, + size_t **rank_chunks_assigned_map); +static herr_t H5D__mpio_redistribute_shared_chunks_int(H5D_filtered_collective_io_info_t *chunk_list, + size_t * num_chunks_assigned_map, + hbool_t all_ranks_involved, + const H5D_io_info_t * io_info, + const H5D_chunk_map_t *fm, int mpi_rank, int mpi_size); +static herr_t H5D__mpio_share_chunk_modification_data(H5D_filtered_collective_io_info_t *chunk_list, + size_t *chunk_list_num_entries, H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, + int mpi_size, + H5D_filtered_collective_io_info_t **chunk_hash_table, + unsigned char *** chunk_msg_bufs, + int * chunk_msg_bufs_len); +static herr_t H5D__mpio_collective_filtered_chunk_common_io(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + const H5D_io_info_t * io_info, + const H5D_type_info_t *type_info, int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_read(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + const H5D_io_info_t * io_info, + const H5D_type_info_t *type_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_update(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + H5D_filtered_collective_io_info_t *chunk_hash_table, + unsigned char ** chunk_msg_bufs, + int chunk_msg_bufs_len, const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_reallocate(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + size_t * num_chunks_assigned_map, + H5D_io_info_t * io_info, + H5D_chk_idx_info_t *idx_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_reinsert(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + size_t * num_chunks_assigned_map, + H5D_io_info_t * io_info, + H5D_chk_idx_info_t *idx_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_get_chunk_redistribute_info_types(MPI_Datatype *contig_type, + hbool_t * contig_type_derived, + MPI_Datatype *resized_type, + hbool_t * resized_type_derived); +static herr_t H5D__mpio_get_chunk_alloc_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived); +static herr_t H5D__mpio_get_chunk_insert_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, + hbool_t * resized_type_derived); +static herr_t H5D__mpio_collective_filtered_io_type(H5D_filtered_collective_io_info_t *chunk_list, + size_t num_entries, H5D_io_op_type_t op_type, + MPI_Datatype *new_mem_type, hbool_t *mem_type_derived, + MPI_Datatype *new_file_type, hbool_t *file_type_derived); static int H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2); static int H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2); -#if MPI_VERSION >= 3 -static int H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, - const void *filtered_collective_io_info_entry2); +static int H5D__cmp_chunk_redistribute_info(const void *entry1, const void *entry2); +static int H5D__cmp_chunk_redistribute_info_orig_owner(const void *entry1, const void *entry2); + +#ifdef H5Dmpio_DEBUG +static herr_t H5D__mpio_debug_init(void); +static herr_t H5D__mpio_dump_collective_filtered_chunk_list(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, int mpi_rank); #endif /*********************/ @@ -273,6 +395,188 @@ static int H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered /* Local Variables */ /*******************/ +/* Declare extern free list to manage the H5S_sel_iter_t struct */ +H5FL_EXTERN(H5S_sel_iter_t); + +#ifdef H5Dmpio_DEBUG + +/* Flags to control debug actions in this file. + * (Meant to be indexed by characters) + * + * These flags can be set with either (or both) the environment variable + * "H5D_mpio_Debug" set to a string containing one or more characters + * (flags) or by setting them as a string value for the + * "H5D_mpio_debug_key" MPI Info key. + * + * Supported characters in 'H5D_mpio_Debug' string: + * 't' trace function entry and exit + * 'f' log to file rather than debugging stream + * 'm' show (rough) memory usage statistics + * 'c' show critical timing information + * + * To only show output from a particular MPI rank, specify its rank + * number as a character, e.g.: + * + * '0' only show output from rank 0 + * + * To only show output from a particular range (up to 8 ranks supported + * between 0-9) of MPI ranks, specify the start and end ranks separated + * by a hyphen, e.g.: + * + * '0-7' only show output from ranks 0 through 7 + * + */ +static int H5D_mpio_debug_flags_s[256]; +static int H5D_mpio_debug_rank_s[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; +static hbool_t H5D_mpio_debug_inited = FALSE; +static const char *const trace_in_pre = "-> "; +static const char *const trace_out_pre = "<- "; +static int debug_indent = 0; +static FILE * debug_stream = NULL; + +/* Determine if this rank should output debugging info */ +#define H5D_MPIO_DEBUG_THIS_RANK(rank) \ + (H5D_mpio_debug_rank_s[0] < 0 || rank == H5D_mpio_debug_rank_s[0] || rank == H5D_mpio_debug_rank_s[1] || \ + rank == H5D_mpio_debug_rank_s[2] || rank == H5D_mpio_debug_rank_s[3] || \ + rank == H5D_mpio_debug_rank_s[4] || rank == H5D_mpio_debug_rank_s[5] || \ + rank == H5D_mpio_debug_rank_s[6] || rank == H5D_mpio_debug_rank_s[7]) + +/* Print some debugging string */ +#define H5D_MPIO_DEBUG(rank, string) \ + do { \ + if (debug_stream && H5D_MPIO_DEBUG_THIS_RANK(rank)) { \ + HDfprintf(debug_stream, "%*s(Rank %d) " string "\n", debug_indent, "", rank); \ + fflush(debug_stream); \ + } \ + } while (0) + +/* Print some debugging string with printf-style arguments */ +#define H5D_MPIO_DEBUG_VA(rank, string, ...) \ + do { \ + if (debug_stream && H5D_MPIO_DEBUG_THIS_RANK(rank)) { \ + HDfprintf(debug_stream, "%*s(Rank %d) " string "\n", debug_indent, "", rank, __VA_ARGS__); \ + fflush(debug_stream); \ + } \ + } while (0) + +#define H5D_MPIO_TRACE_ENTER(rank) \ + do { \ + hbool_t trace_flag = H5D_mpio_debug_flags_s[(int)'t']; \ + \ + if (trace_flag) { \ + H5D_MPIO_DEBUG_VA(rank, "%s%s", trace_in_pre, __func__); \ + debug_indent += (int)strlen(trace_in_pre); \ + } \ + } while (0) + +#define H5D_MPIO_TRACE_EXIT(rank) \ + do { \ + hbool_t trace_flag = H5D_mpio_debug_flags_s[(int)'t']; \ + \ + if (trace_flag) { \ + debug_indent -= (int)strlen(trace_out_pre); \ + H5D_MPIO_DEBUG_VA(rank, "%s%s", trace_out_pre, __func__); \ + } \ + } while (0) + +#define H5D_MPIO_TIME_START(rank, op_name) \ + { \ + hbool_t time_flag = H5D_mpio_debug_flags_s[(int)'c']; \ + double start_time = 0.0, end_time = 0.0; \ + const char *const op = op_name; \ + \ + if (time_flag) { \ + start_time = MPI_Wtime(); \ + } + +#define H5D_MPIO_TIME_STOP(rank) \ + if (time_flag) { \ + end_time = MPI_Wtime(); \ + H5D_MPIO_DEBUG_VA(rank, "'%s' took %f seconds", op, (end_time - start_time)); \ + } \ + } + +/*--------------------------------------------------------------------------- + * Function: H5D__mpio_parse_debug_str + * + * Purpose: Parse a string for H5Dmpio-related debugging flags + * + * Returns: N/A + * + *--------------------------------------------------------------------------- + */ +static void +H5D__mpio_parse_debug_str(const char *s) +{ + FUNC_ENTER_STATIC_NOERR + + HDassert(s); + + while (*s) { + int c = (int)(*s); + + if (c >= (int)'0' && c <= (int)'9') { + hbool_t range = FALSE; + + if (*(s + 1) && *(s + 2)) + range = (int)*(s + 1) == '-' && (int)*(s + 2) >= (int)'0' && (int)*(s + 2) <= (int)'9'; + + if (range) { + int start_rank = c - (int)'0'; + int end_rank = (int)*(s + 2) - '0'; + int num_ranks = end_rank - start_rank + 1; + int i; + + if (num_ranks > 8) { + end_rank = start_rank + 7; + num_ranks = 8; + } + + for (i = 0; i < num_ranks; i++) + H5D_mpio_debug_rank_s[i] = start_rank++; + + s += 3; + } + else + H5D_mpio_debug_rank_s[0] = c - (int)'0'; + } + else + H5D_mpio_debug_flags_s[c]++; + + s++; + } + + FUNC_LEAVE_NOAPI_VOID +} + +static herr_t +H5D__mpio_debug_init(void) +{ + const char *debug_str; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC_NOERR + + HDassert(!H5D_mpio_debug_inited); + + /* Clear the debug flag buffer */ + HDmemset(H5D_mpio_debug_flags_s, 0, sizeof(H5D_mpio_debug_flags_s)); + + /* Retrieve and parse the H5Dmpio debug string */ + debug_str = HDgetenv("H5D_mpio_Debug"); + if (debug_str) + H5D__mpio_parse_debug_str(debug_str); + + if (H5DEBUG(D)) + debug_stream = H5DEBUG(D); + + H5D_mpio_debug_inited = TRUE; + + FUNC_LEAVE_NOAPI(ret_value) +} + +#endif + /*------------------------------------------------------------------------- * Function: H5D__mpio_opt_possible * @@ -347,14 +651,9 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, co * use collective IO will defer until each chunk IO is reached. */ -#if MPI_VERSION < 3 - /* - * Don't allow parallel writes to filtered datasets if the MPI version - * is less than 3. The functions needed (MPI_Mprobe and MPI_Imrecv) will - * not be available. - */ - if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->layout.type == H5D_CHUNKED && - io_info->dset->shared->dcpl_cache.pline.nused > 0) +#ifndef H5_HAVE_PARALLEL_FILTERED_WRITES + /* Don't allow writes to filtered datasets if the functionality is disabled */ + if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) local_cause[0] |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED; #endif @@ -365,7 +664,7 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, co /* Check to see if the process is reading the entire dataset */ if (H5S_GET_SELECT_TYPE(file_space) != H5S_SEL_ALL) local_cause[1] |= H5D_MPIO_RANK0_NOT_H5S_ALL; - /* Only perform this optimization for contigous datasets, currently */ + /* Only perform this optimization for contiguous datasets, currently */ else if (H5D_CONTIGUOUS != io_info->dset->shared->layout.type) /* Flag to do a MPI_Bcast of the data from one proc instead of * having all the processes involved in the collective I/O. @@ -437,6 +736,150 @@ done: } /* H5D__mpio_opt_possible() */ /*------------------------------------------------------------------------- + * Function: H5D__mpio_get_no_coll_cause_strings + * + * Purpose: When collective I/O is broken internally, it can be useful + * for users to see a representative string for the reason(s) + * why it was broken. This routine inspects the current + * "cause" flags from the API context and prints strings into + * the caller's buffers for the local and global reasons that + * collective I/O was broken. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, char *global_cause, + size_t global_cause_len) +{ + uint32_t local_no_coll_cause; + uint32_t global_no_coll_cause; + size_t local_cause_bytes_written = 0; + size_t global_cause_bytes_written = 0; + int nbits; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert((local_cause && local_cause_len > 0) || (global_cause && global_cause_len > 0)); + + /* + * Use compile-time assertion so this routine is updated + * when any new "no collective cause" values are added + */ + HDcompile_assert(H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE == (H5D_mpio_no_collective_cause_t)256); + + /* Initialize output buffers */ + if (local_cause) + *local_cause = '\0'; + if (global_cause) + *global_cause = '\0'; + + /* Retrieve the local and global cause flags from the API context */ + if (H5CX_get_mpio_local_no_coll_cause(&local_no_coll_cause) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "unable to get local no collective cause value") + if (H5CX_get_mpio_global_no_coll_cause(&global_no_coll_cause) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "unable to get global no collective cause value") + + /* + * Append each of the "reason for breaking collective I/O" + * error messages to the local and global cause string buffers + */ + nbits = 8 * sizeof(local_no_coll_cause); + for (int bit_pos = 0; bit_pos < nbits; bit_pos++) { + H5D_mpio_no_collective_cause_t cur_cause; + const char * cause_str; + size_t buf_space_left; + + cur_cause = (H5D_mpio_no_collective_cause_t)(1 << bit_pos); + if (cur_cause == H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE) + break; + + switch (cur_cause) { + case H5D_MPIO_SET_INDEPENDENT: + cause_str = "independent I/O was requested"; + break; + case H5D_MPIO_DATATYPE_CONVERSION: + cause_str = "datatype conversions were required"; + break; + case H5D_MPIO_DATA_TRANSFORMS: + cause_str = "data transforms needed to be applied"; + break; + case H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED: + cause_str = "optimized MPI types flag wasn't set"; + break; + case H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES: + cause_str = "one of the dataspaces was neither simple nor scalar"; + break; + case H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET: + cause_str = "dataset was not contiguous or chunked"; + break; + case H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED: + cause_str = "parallel writes to filtered datasets are disabled"; + break; + case H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE: + cause_str = "an error occurred while checking if collective I/O was possible"; + break; + case H5D_MPIO_COLLECTIVE: + case H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE: + default: + HDassert(0 && "invalid no collective cause reason"); + break; + } + + /* + * Determine if the local reasons for breaking collective I/O + * included the current cause + */ + if (local_cause && (cur_cause & local_no_coll_cause)) { + buf_space_left = local_cause_len - local_cause_bytes_written; + + /* + * Check if there were any previous error messages included. If + * so, prepend a semicolon to separate the messages. + */ + if (buf_space_left && local_cause_bytes_written) { + HDstrncat(local_cause, "; ", buf_space_left); + local_cause_bytes_written += MIN(buf_space_left, 2); + buf_space_left -= MIN(buf_space_left, 2); + } + + if (buf_space_left) { + HDstrncat(local_cause, cause_str, buf_space_left); + local_cause_bytes_written += MIN(buf_space_left, HDstrlen(cause_str)); + } + } + + /* + * Determine if the global reasons for breaking collective I/O + * included the current cause + */ + if (global_cause && (cur_cause & global_no_coll_cause)) { + buf_space_left = global_cause_len - global_cause_bytes_written; + + /* + * Check if there were any previous error messages included. If + * so, prepend a semicolon to separate the messages. + */ + if (buf_space_left && global_cause_bytes_written) { + HDstrncat(global_cause, "; ", buf_space_left); + global_cause_bytes_written += MIN(buf_space_left, 2); + buf_space_left -= MIN(buf_space_left, 2); + } + + if (buf_space_left) { + HDstrncat(global_cause, cause_str, buf_space_left); + global_cause_bytes_written += MIN(buf_space_left, HDstrlen(cause_str)); + } + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_get_no_coll_cause_strings() */ + +/*------------------------------------------------------------------------- * Function: H5D__mpio_select_read * * Purpose: MPI-IO function to read directly from app buffer to file. @@ -449,8 +892,8 @@ done: */ herr_t H5D__mpio_select_read(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED *type_info, - hsize_t mpi_buf_count, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space) + hsize_t mpi_buf_count, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space) { const H5D_contig_storage_t *store_contig = &(io_info->store->contig); /* Contiguous storage info for this I/O operation */ @@ -480,8 +923,8 @@ done: */ herr_t H5D__mpio_select_write(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED *type_info, - hsize_t mpi_buf_count, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space) + hsize_t mpi_buf_count, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space) { const H5D_contig_storage_t *store_contig = &(io_info->store->contig); /* Contiguous storage info for this I/O operation */ @@ -500,145 +943,6 @@ done: } /* end H5D__mpio_select_write() */ /*------------------------------------------------------------------------- - * Function: H5D__mpio_array_gatherv - * - * Purpose: Given an array, specified in local_array, by each processor - * calling this function, collects each array into a single - * array which is then either gathered to the processor - * specified by root, when allgather is false, or is - * distributed back to all processors when allgather is true. - * - * The number of entries in the array contributed by an - * individual processor and the size of each entry should be - * specified in local_array_num_entries and array_entry_size, - * respectively. - * - * The MPI communicator to use should be specified for comm. - * - * If the sort_func argument is supplied, the array is sorted - * before the function returns. - * - * Note: if allgather is specified as true, root is ignored. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Jordan Henderson - * Sunday, April 9th, 2017 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries, size_t array_entry_size, - void **_gathered_array, size_t *_gathered_array_num_entries, hbool_t allgather, - int root, MPI_Comm comm, H5D_mpio_sort_func_cb_t sort_func) -{ - size_t gathered_array_num_entries = 0; /* The size of the newly-constructed array */ - void * gathered_array = NULL; /* The newly-constructed array returned to the caller */ - int *receive_counts_array = NULL; /* Array containing number of entries each processor is contributing */ - int *displacements_array = - NULL; /* Array of displacements where each processor places its data in the final array */ - int mpi_code, mpi_rank, mpi_size; - int sendcount; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - HDassert(_gathered_array); - HDassert(_gathered_array_num_entries); - - MPI_Comm_size(comm, &mpi_size); - MPI_Comm_rank(comm, &mpi_rank); - - /* Determine the size of the end result array by collecting the number - * of entries contributed by each processor into a single total. - */ - if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_array_num_entries, &gathered_array_num_entries, 1, - MPI_INT, MPI_SUM, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) - - /* If 0 entries resulted from the collective operation, no processor is contributing anything and there is - * nothing to do */ - if (gathered_array_num_entries > 0) { - /* - * If gathering to all processors, all processors need to allocate space for the resulting array, as - * well as the receive counts and displacements arrays for the collective MPI_Allgatherv call. - * Otherwise, only the root processor needs to allocate the space for an MPI_Gatherv call. - */ - if (allgather || (mpi_rank == root)) { - if (NULL == (gathered_array = H5MM_malloc(gathered_array_num_entries * array_entry_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate gathered array") - - if (NULL == (receive_counts_array = (int *)H5MM_malloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive counts array") - - if (NULL == (displacements_array = (int *)H5MM_malloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive displacements array") - } /* end if */ - - /* - * If gathering to all processors, inform each processor of how many entries each other processor is - * contributing to the resulting array by collecting the counts into each processor's "receive counts" - * array. Otherwise, inform only the root processor of how many entries each other processor is - * contributing. - */ - if (allgather) { - if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&local_array_num_entries, 1, MPI_INT, - receive_counts_array, 1, MPI_INT, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) - } /* end if */ - else { - if (MPI_SUCCESS != (mpi_code = MPI_Gather(&local_array_num_entries, 1, MPI_INT, - receive_counts_array, 1, MPI_INT, root, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Gather failed", mpi_code) - } /* end else */ - - if (allgather || (mpi_rank == root)) { - size_t i; - - /* Multiply each receive count by the size of the array entry, since the data is sent as bytes. */ - for (i = 0; i < (size_t)mpi_size; i++) - H5_CHECKED_ASSIGN(receive_counts_array[i], int, - (size_t)receive_counts_array[i] * array_entry_size, size_t); - - /* Set receive buffer offsets for the collective MPI_Allgatherv/MPI_Gatherv call. */ - displacements_array[0] = 0; - for (i = 1; i < (size_t)mpi_size; i++) - displacements_array[i] = displacements_array[i - 1] + receive_counts_array[i - 1]; - } /* end if */ - - /* As the data is sent as bytes, calculate the true sendcount for the data. */ - H5_CHECKED_ASSIGN(sendcount, int, local_array_num_entries *array_entry_size, size_t); - - if (allgather) { - if (MPI_SUCCESS != - (mpi_code = MPI_Allgatherv(local_array, sendcount, MPI_BYTE, gathered_array, - receive_counts_array, displacements_array, MPI_BYTE, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code) - } /* end if */ - else { - if (MPI_SUCCESS != - (mpi_code = MPI_Gatherv(local_array, sendcount, MPI_BYTE, gathered_array, - receive_counts_array, displacements_array, MPI_BYTE, root, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Gatherv failed", mpi_code) - } /* end else */ - - if (sort_func && (allgather || (mpi_rank == root))) - HDqsort(gathered_array, gathered_array_num_entries, array_entry_size, sort_func); - } /* end if */ - - *_gathered_array = gathered_array; - *_gathered_array_num_entries = gathered_array_num_entries; - -done: - if (receive_counts_array) - H5MM_free(receive_counts_array); - if (displacements_array) - H5MM_free(displacements_array); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__mpio_array_gatherv() */ - -/*------------------------------------------------------------------------- * Function: H5D__mpio_get_sum_chunk * * Purpose: Routine for obtaining total number of chunks to cover @@ -690,7 +994,7 @@ done: */ herr_t H5D__contig_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t H5_ATTR_UNUSED nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CONTIGUOUS_COLLECTIVE; @@ -729,7 +1033,7 @@ done: */ herr_t H5D__contig_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t H5_ATTR_UNUSED nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CONTIGUOUS_COLLECTIVE; @@ -793,11 +1097,17 @@ static herr_t H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm) { H5FD_mpio_chunk_opt_t chunk_opt_mode; - int io_option = H5D_MULTI_CHUNK_IO_MORE_OPT; - int sum_chunk = -1; +#ifdef H5Dmpio_DEBUG + hbool_t log_file_flag = FALSE; + FILE * debug_log_file = NULL; +#endif #ifdef H5_HAVE_INSTRUMENTED_LIBRARY htri_t temp_not_link_io = FALSE; #endif + int io_option = H5D_MULTI_CHUNK_IO_MORE_OPT; + int sum_chunk = -1; + int mpi_rank; + int mpi_size; herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -808,9 +1118,35 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf HDassert(type_info); HDassert(fm); - /* Disable collective metadata reads for chunked dataset I/O operations - * in order to prevent potential hangs */ - H5CX_set_coll_metadata_read(FALSE); + /* Obtain the current rank of the process and the number of ranks */ + if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI rank") + if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI size") + +#ifdef H5Dmpio_DEBUG + /* Initialize file-level debugging if not initialized */ + if (!H5D_mpio_debug_inited && H5D__mpio_debug_init() < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize H5Dmpio debugging") + + /* Open file for debugging if necessary */ + log_file_flag = H5D_mpio_debug_flags_s[(int)'f']; + if (log_file_flag) { + char debug_log_filename[1024]; + time_t time_now; + + HDsnprintf(debug_log_filename, 1024, "H5Dmpio_debug.rank%d", mpi_rank); + + if (NULL == (debug_log_file = HDfopen(debug_log_filename, "a"))) + HGOTO_ERROR(H5E_IO, H5E_OPENERROR, FAIL, "couldn't open debugging log file") + + /* Print a short header for this I/O operation */ + time_now = time(NULL); + HDfprintf(debug_log_file, "##### %s", asctime(localtime(&time_now))); + + debug_stream = debug_log_file; + } +#endif /* Check the optional property list for the collective chunk IO optimization option */ if (H5CX_get_mpio_chunk_opt_mode(&chunk_opt_mode) < 0) @@ -824,13 +1160,10 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf /* via default path. branch by num threshold */ else { unsigned one_link_chunk_io_threshold; /* Threshold to use single collective I/O for all chunks */ - int mpi_size; /* Number of processes in MPI job */ if (H5D__mpio_get_sum_chunk(io_info, fm, &sum_chunk) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSWAP, FAIL, "unable to obtain the total chunk number of all processes"); - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") /* Get the chunk optimization option threshold */ if (H5CX_get_mpio_chunk_opt_num(&one_link_chunk_io_threshold) < 0) @@ -876,22 +1209,12 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf case H5D_ONE_LINK_CHUNK_IO_MORE_OPT: /* Check if there are any filters in the pipeline */ if (io_info->dset->shared->dcpl_cache.pline.nused > 0) { - /* For now, Multi-chunk IO must be forced for parallel filtered read, - * so that data can be unfiltered as it is received. There is significant - * complexity in unfiltering the data when it is read all at once into a - * single buffer. - */ - if (io_info->op_type == H5D_IO_OP_READ) { - if (H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, - "couldn't finish optimized multiple filtered chunk MPI-IO") - } /* end if */ - else if (H5D__link_chunk_filtered_collective_io(io_info, type_info, fm) < 0) + if (H5D__link_chunk_filtered_collective_io(io_info, type_info, fm, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish filtered linked chunk MPI-IO") } /* end if */ else /* Perform unfiltered link chunk collective IO */ - if (H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk) < 0) + if (H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO") break; @@ -899,18 +1222,28 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf default: /* multiple chunk IO via threshold */ /* Check if there are any filters in the pipeline */ if (io_info->dset->shared->dcpl_cache.pline.nused > 0) { - if (H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm) < 0) + if (H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO") } /* end if */ else /* Perform unfiltered multi chunk collective IO */ - if (H5D__multi_chunk_collective_io(io_info, type_info, fm) < 0) + if (H5D__multi_chunk_collective_io(io_info, type_info, fm, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO") break; } /* end switch */ done: +#ifdef H5Dmpio_DEBUG + /* Close debugging log file */ + if (debug_log_file) { + HDfprintf(debug_log_file, "##############\n\n"); + if (EOF == HDfclose(debug_log_file)) + HDONE_ERROR(H5E_IO, H5E_CLOSEERROR, FAIL, "couldn't close debugging log file") + debug_stream = H5DEBUG(D); + } +#endif + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_collective_io */ @@ -929,8 +1262,8 @@ done: */ herr_t H5D__chunk_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) + hsize_t H5_ATTR_UNUSED nelmts, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) { herr_t ret_value = SUCCEED; /* Return value */ @@ -959,8 +1292,8 @@ done: */ herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) + hsize_t H5_ATTR_UNUSED nelmts, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) { herr_t ret_value = SUCCEED; /* Return value */ @@ -993,12 +1326,12 @@ done: */ static herr_t H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, - int sum_chunk) + int sum_chunk, int mpi_rank, int mpi_size) { H5D_chunk_addr_info_t *chunk_addr_info_array = NULL; - MPI_Datatype chunk_final_mtype; /* Final memory MPI datatype for all chunks with seletion */ + MPI_Datatype chunk_final_mtype; /* Final memory MPI datatype for all chunks with selection */ hbool_t chunk_final_mtype_is_derived = FALSE; - MPI_Datatype chunk_final_ftype; /* Final file MPI datatype for all chunks with seletion */ + MPI_Datatype chunk_final_ftype; /* Final file MPI datatype for all chunks with selection */ hbool_t chunk_final_ftype_is_derived = FALSE; H5D_storage_t ctg_store; /* Storage info for "fake" contiguous dataset */ size_t total_chunks; @@ -1074,9 +1407,8 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ /* Set up the base storage address for this chunk */ io_info->store = &ctg_store; -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before inter_collective_io for total chunk = 1 \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before inter_collective_io for total chunk = 1"); #endif /* Perform I/O */ @@ -1092,9 +1424,8 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ num_chunk = H5SL_count(fm->sel_chunks); H5_CHECK_OVERFLOW(num_chunk, size_t, int); -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "total_chunks = %zu, num_chunk = %zu\n", total_chunks, num_chunk); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "total_chunks = %zu, num_chunk = %zu", total_chunks, num_chunk); #endif /* Set up MPI datatype for chunks selected */ @@ -1125,18 +1456,17 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk file is derived datatype flags buffer") -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before sorting the chunk address \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before sorting chunk addresses"); #endif + /* Sort the chunk address */ - if (H5D__sort_chunk(io_info, fm, chunk_addr_info_array, sum_chunk) < 0) + if (H5D__sort_chunk(io_info, fm, chunk_addr_info_array, sum_chunk, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSWAP, FAIL, "unable to sort chunk address") ctg_store.contig.dset_addr = chunk_addr_info_array[0].chunk_addr; -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "after sorting the chunk address \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "after sorting chunk addresses"); #endif /* Obtain MPI derived datatype from all individual chunks */ @@ -1241,9 +1571,9 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ /* No chunks selected for this process */ mpi_buf_count = (hsize_t)0; } /* end else */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before coming to final collective IO\n"); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before coming to final collective I/O"); #endif /* Set up the base storage address for this chunk */ @@ -1256,11 +1586,11 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ } /* end else */ done: -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before freeing memory inside H5D_link_collective_io ret_value = %d\n", - ret_value); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "before freeing memory inside H5D_link_collective_io ret_value = %d", + ret_value); #endif + /* Release resources */ if (chunk_addr_info_array) H5MM_xfree(chunk_addr_info_array); @@ -1293,68 +1623,89 @@ done: /*------------------------------------------------------------------------- * Function: H5D__link_chunk_filtered_collective_io * - * Purpose: Routine for one collective IO with one MPI derived datatype - * to link with all filtered chunks - * - * 1. Construct a list of selected chunks in the collective IO - * operation - * A. If any chunk is being written to by more than 1 - * process, the process writing to the chunk which - * currently has the least amount of chunks assigned - * to it becomes the new owner (in the case of ties, - * the lowest MPI rank becomes the new owner) - * 2. If the operation is a write operation - * A. Loop through each chunk in the operation - * I. If this is not a full overwrite of the chunk - * a) Read the chunk from file and pass the chunk - * through the filter pipeline in reverse order - * (Unfilter the chunk) + * Purpose: Performs collective I/O on filtered chunks by creating a + * single MPI derived datatype to link with all filtered + * chunks. The general algorithm is as follows: + * + * 1. Construct a list of selected chunks in the collective + * I/O operation + * 2. If the operation is a read operation + * A. Ensure that the list of chunks is sorted in + * monotonically non-decreasing order of chunk offset + * in the file + * B. Participate in a collective read of chunks from + * the file + * C. Loop through each selected chunk, unfiltering it and + * scattering the data to the application's read buffer + * 3. If the operation is a write operation + * A. Redistribute any chunks being written by more than 1 + * MPI rank, such that the chunk is only owned by 1 MPI + * rank. The rank writing to the chunk which currently + * has the least amount of chunks assigned to it becomes + * the new owner (in the case of ties, the lowest MPI + * rank becomes the new owner) + * B. Participate in a collective read of chunks from the + * file + * C. Loop through each chunk selected in the operation + * and for each chunk: + * I. If we actually read the chunk from the file (if + * a chunk is being fully overwritten, we skip + * reading it), pass the chunk through the filter + * pipeline in reverse order (unfilter the chunk) * II. Update the chunk data with the modifications from - * the owning process + * the owning MPI rank * III. Receive any modification data from other - * processes and update the chunk data with these + * ranks and update the chunk data with those * modifications * IV. Filter the chunk - * B. Contribute the modified chunks to an array gathered - * by all processes which contains the new sizes of - * every chunk modified in the collective IO operation - * C. All processes collectively re-allocate each chunk - * from the gathered array with their new sizes after - * the filter operation - * D. If this process has any chunks selected in the IO - * operation, create an MPI derived type for memory and - * file to write out the process' selected chunks to the - * file - * E. Perform the collective write - * F. All processes collectively re-insert each modified + * D. Contribute the modified chunks to an array gathered + * by all ranks which contains information for + * re-allocating space in the file for every chunk + * modified. Then, each rank collectively re-allocates + * each chunk from the gathered array with their new + * sizes after the filter operation + * E. Proceed with the collective write operation for all + * the modified chunks + * F. Contribute the modified chunks to an array gathered + * by all ranks which contains information for + * re-inserting every chunk modified into the chunk + * index. Then, each rank collectively re-inserts each * chunk from the gathered array into the chunk index * + * TODO: Note that steps D. and F. here are both collective + * operations that partially share data from the + * H5D_filtered_collective_io_info_t structure. To + * try to conserve on memory a bit, the distributed + * arrays these operations create are discarded after + * each operation is performed. If memory consumption + * here proves to not be an issue, the necessary data + * for both operations could be combined into a single + * structure so that only one collective MPI operation + * is needed to carry out both operations, rather than + * two. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Friday, Nov. 4th, 2016 - * *------------------------------------------------------------------------- */ static herr_t H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm) + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size) { - H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ - H5D_filtered_collective_io_info_t *collective_chunk_list = - NULL; /* The list of chunks used during collective operations */ - H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ - MPI_Datatype mem_type = MPI_BYTE; - MPI_Datatype file_type = MPI_BYTE; - hbool_t mem_type_is_derived = FALSE; - hbool_t file_type_is_derived = FALSE; - size_t chunk_list_num_entries; - size_t collective_chunk_list_num_entries; - size_t * num_chunks_selected_array = NULL; /* Array of number of chunks selected on each process */ - size_t i; /* Local index variable */ - int mpi_rank, mpi_size, mpi_code; - herr_t ret_value = SUCCEED; + H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ + H5D_filtered_collective_io_info_t *chunk_hash_table = NULL; + unsigned char ** chunk_msg_bufs = NULL; + H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ + MPI_Datatype mem_type = MPI_BYTE; + MPI_Datatype file_type = MPI_BYTE; + hbool_t mem_type_is_derived = FALSE; + hbool_t file_type_is_derived = FALSE; + size_t * rank_chunks_assigned_map = NULL; + size_t chunk_list_num_entries; + size_t i; + int chunk_msg_bufs_len = 0; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -1362,11 +1713,12 @@ H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_in HDassert(type_info); HDassert(fm); - /* Obtain the current rank of the process and the number of processes */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "Performing Linked-chunk I/O (%s) with MPI Comm size of %d", + io_info->op_type == H5D_IO_OP_WRITE ? "write" : "read", mpi_size); + H5D_MPIO_TIME_START(mpi_rank, "Linked-chunk I/O"); +#endif /* Set the actual-chunk-opt-mode property. */ H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); @@ -1377,123 +1729,127 @@ H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_in H5CX_set_mpio_actual_io_mode(H5D_MPIO_CHUNK_COLLECTIVE); /* Build a list of selected chunks in the collective io operation */ - if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < - 0) + if (H5D__mpio_collective_filtered_chunk_io_setup(io_info, type_info, fm, &chunk_list, + &chunk_list_num_entries, mpi_rank) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list") - if (io_info->op_type == H5D_IO_OP_WRITE) { /* Filtered collective write */ + if (io_info->op_type == H5D_IO_OP_READ) { /* Filtered collective read */ + if (H5D__mpio_collective_filtered_chunk_read(chunk_list, chunk_list_num_entries, io_info, type_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't read filtered chunks") + } + else { /* Filtered collective write */ H5D_chk_idx_info_t index_info; - H5D_chunk_ud_t udata; hsize_t mpi_buf_count; - /* Construct chunked index info */ - index_info.f = io_info->dset->oloc.file; - index_info.pline = &(io_info->dset->shared->dcpl_cache.pline); - index_info.layout = &(io_info->dset->shared->layout.u.chunk); - index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk); - - /* Set up chunk information for insertion to chunk index */ - udata.common.layout = index_info.layout; - udata.common.storage = index_info.storage; - udata.filter_mask = 0; - - /* Iterate through all the chunks in the collective write operation, - * updating each chunk with the data modifications from other processes, - * then re-filtering the chunk. + H5D_MPIO_INIT_CHUNK_IDX_INFO(index_info, io_info); + + if (mpi_size > 1) { + /* Redistribute shared chunks being written to */ + if (H5D__mpio_redistribute_shared_chunks(chunk_list, chunk_list_num_entries, io_info, fm, + mpi_rank, mpi_size, &rank_chunks_assigned_map) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks") + + /* Send any chunk modification messages for chunks this rank no longer owns */ + if (H5D__mpio_share_chunk_modification_data(chunk_list, &chunk_list_num_entries, io_info, + type_info, mpi_rank, mpi_size, &chunk_hash_table, + &chunk_msg_bufs, &chunk_msg_bufs_len) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "unable to send chunk modification data between MPI ranks") + + /* Make sure the local chunk list was updated correctly */ + HDassert(chunk_list_num_entries == rank_chunks_assigned_map[mpi_rank]); + } + + /* Proceed to update all the chunks this rank owns with its own + * modification data and data from other ranks, before re-filtering + * the chunks. As chunk reads are done collectively here, all ranks + * must participate. */ - for (i = 0; i < chunk_list_num_entries; i++) - if (mpi_rank == chunk_list[i].owners.new_owner) - if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry") - - /* Gather the new chunk sizes to all processes for a collective reallocation - * of the chunks in the file. + if (H5D__mpio_collective_filtered_chunk_update(chunk_list, chunk_list_num_entries, chunk_hash_table, + chunk_msg_bufs, chunk_msg_bufs_len, io_info, type_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't update modified chunks") + + /* Free up resources used by chunk hash table now that we're done updating chunks */ + HASH_CLEAR(hh, chunk_hash_table); + + /* All ranks now collectively re-allocate file space for all chunks */ + if (H5D__mpio_collective_filtered_chunk_reallocate(chunk_list, chunk_list_num_entries, + rank_chunks_assigned_map, io_info, &index_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-allocate file space for chunks") + + /* If this rank has any chunks selected, create a MPI type for collectively + * writing out the chunks to file. Otherwise, the rank contributes to the + * collective write with a none type. */ - if (H5D__mpio_array_gatherv(chunk_list, chunk_list_num_entries, - sizeof(H5D_filtered_collective_io_info_t), - (void **)&collective_chunk_list, &collective_chunk_list_num_entries, true, - 0, io_info->comm, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes") - - /* Collectively re-allocate the modified chunks (from each process) in the file */ - for (i = 0; i < collective_chunk_list_num_entries; i++) { - hbool_t insert; - - if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[i].chunk_states.chunk_current, - &collective_chunk_list[i].chunk_states.new_chunk, &insert, - collective_chunk_list[i].scaled) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") - } /* end for */ - - if (NULL == (num_chunks_selected_array = (size_t *)H5MM_malloc((size_t)mpi_size * sizeof(size_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array") + if (H5D__mpio_collective_filtered_io_type(chunk_list, chunk_list_num_entries, io_info->op_type, + &mem_type, &mem_type_is_derived, &file_type, + &file_type_is_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "couldn't create MPI type for writing filtered chunks") - if (MPI_SUCCESS != - (mpi_code = MPI_Allgather(&chunk_list_num_entries, 1, MPI_UNSIGNED_LONG_LONG, - num_chunks_selected_array, 1, MPI_UNSIGNED_LONG_LONG, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + mpi_buf_count = (file_type_is_derived || mem_type_is_derived) ? 1 : 0; - /* If this process has any chunks selected, create a MPI type for collectively - * writing out the chunks to file. Otherwise, the process contributes to the - * collective write with a none type. - */ + /* Setup contig storage info for I/O operation */ if (chunk_list_num_entries) { - size_t offset; - - /* During the collective re-allocation of chunks in the file, the record for each - * chunk is only updated in the collective array, not in the local copy of chunks on each - * process. However, each process needs the updated chunk records so that they can create - * a MPI type for the collective write that will write to the chunk's possible new locations - * in the file instead of the old ones. This ugly hack seems to be the best solution to - * copy the information back to the local array and avoid having to modify the collective - * write type function in an ugly way so that it will accept the collective array instead - * of the local array. This works correctly because the array gather function guarantees - * that the chunk data in the collective array is ordered in blocks by rank. + /* + * Override the write buffer to point to the first + * chunk's data buffer */ - for (i = 0, offset = 0; i < (size_t)mpi_rank; i++) - offset += num_chunks_selected_array[i]; - - H5MM_memcpy(chunk_list, &collective_chunk_list[offset], - num_chunks_selected_array[mpi_rank] * sizeof(H5D_filtered_collective_io_info_t)); - - /* Create single MPI type encompassing each selection in the dataspace */ - if (H5D__mpio_filtered_collective_write_type(chunk_list, chunk_list_num_entries, &mem_type, - &mem_type_is_derived, &file_type, - &file_type_is_derived) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "couldn't create MPI link chunk I/O type") + io_info->u.wbuf = chunk_list[0].buf; - /* Override the write buffer to point to the address of the first - * chunk data buffer + /* + * Setup the base storage address for this operation + * to be the first chunk's file address */ - io_info->u.wbuf = chunk_list[0].buf; - } /* end if */ - - /* We have a single, complicated MPI datatype for both memory & file */ - mpi_buf_count = (mem_type_is_derived && file_type_is_derived) ? (hsize_t)1 : (hsize_t)0; - - /* Set up the base storage address for this operation */ - ctg_store.contig.dset_addr = 0; /* Write address must be set to address 0 */ - io_info->store = &ctg_store; + ctg_store.contig.dset_addr = chunk_list[0].chunk_new.offset; + } + else + ctg_store.contig.dset_addr = 0; /* Perform I/O */ + io_info->store = &ctg_store; if (H5D__final_collective_io(io_info, type_info, mpi_buf_count, file_type, mem_type) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO") + /* Free up resources in anticipation of following collective operation */ + for (i = 0; i < chunk_list_num_entries; i++) { + if (chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + /* Participate in the collective re-insertion of all chunks modified - * in this iteration into the chunk index + * into the chunk index */ - for (i = 0; i < collective_chunk_list_num_entries; i++) { - udata.chunk_block = collective_chunk_list[i].chunk_states.new_chunk; - udata.common.scaled = collective_chunk_list[i].scaled; - udata.chunk_idx = collective_chunk_list[i].index; - - if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index") - } /* end for */ - } /* end if */ + if (H5D__mpio_collective_filtered_chunk_reinsert(chunk_list, chunk_list_num_entries, + rank_chunks_assigned_map, io_info, &index_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-insert modified chunks into chunk index") + } done: - /* Free resources used by a process which had some selection */ + /* Free the MPI buf and file types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + if (chunk_msg_bufs) { + for (i = 0; i < (size_t)chunk_msg_bufs_len; i++) + H5MM_free(chunk_msg_bufs[i]); + + H5MM_free(chunk_msg_bufs); + } + + HASH_CLEAR(hh, chunk_hash_table); + + /* Free resources used by a rank which had some selection */ if (chunk_list) { for (i = 0; i < chunk_list_num_entries; i++) if (chunk_list[i].buf) @@ -1502,16 +1858,13 @@ done: H5MM_free(chunk_list); } /* end if */ - if (num_chunks_selected_array) - H5MM_free(num_chunks_selected_array); - if (collective_chunk_list) - H5MM_free(collective_chunk_list); + if (rank_chunks_assigned_map) + H5MM_free(rank_chunks_assigned_map); - /* Free the MPI buf and file types, if they were derived */ - if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__link_chunk_filtered_collective_io() */ @@ -1534,7 +1887,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm) +H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size) { H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ @@ -1547,11 +1901,8 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty H5FD_mpio_collective_opt_t last_coll_opt_mode = H5FD_MPIO_COLLECTIVE_IO; /* Last parallel transfer with independent IO or collective IO with this mode */ - size_t total_chunk; /* Total # of chunks in dataset */ -#ifdef H5Dmpio_DEBUG - int mpi_rank; -#endif - size_t u; /* Local index variable */ + size_t total_chunk; /* Total # of chunks in dataset */ + size_t u; /* Local index variable */ H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_NO_COLLECTIVE; /* Local variable for tracking the I/O mode used. */ herr_t ret_value = SUCCEED; @@ -1561,10 +1912,6 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty /* Set the actual chunk opt mode property */ H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_MULTI_CHUNK); -#ifdef H5Dmpio_DEBUG - mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file); -#endif - /* Retrieve total # of chunks in dataset */ H5_CHECKED_ASSIGN(total_chunk, size_t, fm->layout->u.chunk.nchunks, hsize_t); HDassert(total_chunk != 0); @@ -1572,13 +1919,13 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty /* Allocate memories */ chunk_io_option = (uint8_t *)H5MM_calloc(total_chunk); chunk_addr = (haddr_t *)H5MM_calloc(total_chunk * sizeof(haddr_t)); -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "total_chunk %zu\n", total_chunk); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "total_chunk %zu", total_chunk); #endif /* Obtain IO option for each chunk */ - if (H5D__obtain_mpio_mode(io_info, fm, chunk_io_option, chunk_addr) < 0) + if (H5D__obtain_mpio_mode(io_info, fm, chunk_io_option, chunk_addr, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTRECV, FAIL, "unable to obtain MPIO mode") /* Set up contiguous I/O info object */ @@ -1606,9 +1953,8 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty H5S_t * fspace; /* Dataspace describing chunk & selection in it */ H5S_t * mspace; /* Dataspace describing selection in memory corresponding to this chunk */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "mpi_rank = %d, chunk index = %zu\n", mpi_rank, u); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "mpi_rank = %d, chunk index = %zu", mpi_rank, u); #endif /* Get the chunk info for this chunk, if there are elements selected */ chunk_info = fm->select_chunk[u]; @@ -1626,10 +1972,9 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty * needs to contribute MPI NONE TYPE. */ if (chunk_io_option[u] == H5D_CHUNK_IO_MODE_COL) { -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "inside collective chunk IO mpi_rank = %d, chunk index = %zu\n", - mpi_rank, u); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "inside collective chunk IO mpi_rank = %d, chunk index = %zu", + mpi_rank, u); #endif /* Set the file & memory dataspaces */ @@ -1665,10 +2010,9 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish shared collective MPI-IO") } /* end if */ else { /* possible independent IO for this chunk */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "inside independent IO mpi_rank = %d, chunk index = %zu\n", mpi_rank, - u); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "inside independent IO mpi_rank = %d, chunk index = %zu", mpi_rank, + u); #endif HDassert(chunk_io_option[u] == 0); @@ -1698,9 +2042,8 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty /* Perform the I/O */ if (H5D__inter_collective_io(&ctg_io_info, type_info, fspace, mspace) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish shared collective MPI-IO") -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "after inter collective IO\n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "after inter collective IO"); #endif } /* end else */ } /* end for */ @@ -1720,80 +2063,101 @@ done: /*------------------------------------------------------------------------- * Function: H5D__multi_chunk_filtered_collective_io * - * Purpose: To do filtered collective IO iteratively to save on memory. - * While link_chunk_filtered_collective_io will construct and - * work on a list of all of the chunks selected in the IO - * operation at once, this function works iteratively on a set - * of chunks at a time; at most one chunk per rank per - * iteration. - * - * 1. Construct a list of selected chunks in the collective IO - * operation - * A. If any chunk is being written to by more than 1 - * process, the process writing to the chunk which - * currently has the least amount of chunks assigned - * to it becomes the new owner (in the case of ties, - * the lowest MPI rank becomes the new owner) - * 2. If the operation is a read operation - * A. Loop through each chunk in the operation - * I. Read the chunk from the file - * II. Unfilter the chunk - * III. Scatter the read chunk data to the user's buffer - * 3. If the operation is a write operation - * A. Loop through each chunk in the operation - * I. If this is not a full overwrite of the chunk - * a) Read the chunk from file and pass the chunk - * through the filter pipeline in reverse order - * (Unfilter the chunk) - * II. Update the chunk data with the modifications from - * the owning process - * III. Receive any modification data from other - * processes and update the chunk data with these - * modifications - * IV. Filter the chunk - * V. Contribute the chunk to an array gathered by - * all processes which contains every chunk - * modified in this iteration (up to one chunk - * per process, some processes may not have a - * selection/may have less chunks to work on than - * other processes) - * VI. All processes collectively re-allocate each - * chunk from the gathered array with their new - * sizes after the filter operation - * VII. Proceed with the collective write operation - * for the chunks modified on this iteration - * VIII. All processes collectively re-insert each - * chunk from the gathered array into the chunk - * index + * Purpose: Performs collective I/O on filtered chunks iteratively to + * save on memory and potentially get better performance + * depending on the average number of chunks per rank. While + * linked-chunk I/O will construct and work on a list of all + * of the chunks selected in the I/O operation at once, this + * function works iteratively on a set of chunks at a time; at + * most one chunk per rank per iteration. The general + * algorithm is as follows: + * + * 1. Construct a list of selected chunks in the collective + * I/O operation + * 2. If the operation is a read operation, loop an amount of + * times equal to the maximum number of chunks selected on + * any particular rank and on each iteration: + * A. Participate in a collective read of chunks from + * the file (ranks that run out of chunks still need + * to participate) + * B. Unfilter the chunk that was read (if any) + * C. Scatter the read chunk's data to the application's + * read buffer + * 3. If the operation is a write operation, redistribute any + * chunks being written to by more than 1 MPI rank, such + * that the chunk is only owned by 1 MPI rank. The rank + * writing to the chunk which currently has the least + * amount of chunks assigned to it becomes the new owner + * (in the case of ties, the lowest MPI rank becomes the + * new owner). Then, loop an amount of times equal to the + * maximum number of chunks selected on any particular + * rank and on each iteration: + * A. Participate in a collective read of chunks from + * the file (ranks that run out of chunks still need + * to participate) + * I. If we actually read a chunk from the file (if + * a chunk is being fully overwritten, we skip + * reading it), pass the chunk through the filter + * pipeline in reverse order (unfilter the chunk) + * B. Update the chunk data with the modifications from + * the owning rank + * C. Receive any modification data from other ranks and + * update the chunk data with those modifications + * D. Filter the chunk + * E. Contribute the chunk to an array gathered by + * all ranks which contains information for + * re-allocating space in the file for every chunk + * modified in this iteration (up to one chunk per + * rank; some ranks may not have a selection/may have + * less chunks to work on than other ranks). Then, + * each rank collectively re-allocates each chunk + * from the gathered array with their new sizes + * after the filter operation + * F. Proceed with the collective write operation + * for the chunks modified on this iteration + * G. Contribute the chunk to an array gathered by + * all ranks which contains information for + * re-inserting every chunk modified on this + * iteration into the chunk index. Then, each rank + * collectively re-inserts each chunk from the + * gathered array into the chunk index + * + * TODO: Note that steps E. and G. here are both collective + * operations that partially share data from the + * H5D_filtered_collective_io_info_t structure. To + * try to conserve on memory a bit, the distributed + * arrays these operations create are discarded after + * each operation is performed. If memory consumption + * here proves to not be an issue, the necessary data + * for both operations could be combined into a single + * structure so that only one collective MPI operation + * is needed to carry out both operations, rather than + * two. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Friday, Dec. 2nd, 2016 - * *------------------------------------------------------------------------- */ static herr_t H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm) + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size) { - H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ - H5D_filtered_collective_io_info_t *collective_chunk_list = - NULL; /* The list of chunks used during collective operations */ - H5D_storage_t store; /* union of EFL and chunk pointer in file space */ - H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ - H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ - MPI_Datatype *file_type_array = NULL; - MPI_Datatype *mem_type_array = NULL; - hbool_t * file_type_is_derived_array = NULL; - hbool_t * mem_type_is_derived_array = NULL; - hbool_t * has_chunk_selected_array = - NULL; /* Array of whether or not each process is contributing a chunk to each iteration */ - size_t chunk_list_num_entries; - size_t collective_chunk_list_num_entries; - size_t i, j; /* Local index variable */ - int mpi_rank, mpi_size, mpi_code; - herr_t ret_value = SUCCEED; + H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ + H5D_filtered_collective_io_info_t *chunk_hash_table = NULL; + unsigned char ** chunk_msg_bufs = NULL; + H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ + H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ + MPI_Datatype mem_type = MPI_BYTE; + MPI_Datatype file_type = MPI_BYTE; + hbool_t mem_type_is_derived = FALSE; + hbool_t file_type_is_derived = FALSE; + hbool_t have_chunk_to_process; + size_t chunk_list_num_entries; + size_t i; + size_t max_num_chunks; + int chunk_msg_bufs_len = 0; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -1801,11 +2165,12 @@ H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_i HDassert(type_info); HDassert(fm); - /* Obtain the current rank of the process and the number of processes */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "Performing Multi-chunk I/O (%s) with MPI Comm size of %d", + io_info->op_type == H5D_IO_OP_WRITE ? "write" : "read", mpi_size); + H5D_MPIO_TIME_START(mpi_rank, "Multi-chunk I/O"); +#endif /* Set the actual chunk opt mode property */ H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_MULTI_CHUNK); @@ -1816,10 +2181,19 @@ H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_i H5CX_set_mpio_actual_io_mode(H5D_MPIO_CHUNK_COLLECTIVE); /* Build a list of selected chunks in the collective IO operation */ - if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < - 0) + if (H5D__mpio_collective_filtered_chunk_io_setup(io_info, type_info, fm, &chunk_list, + &chunk_list_num_entries, mpi_rank) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list") + /* Retrieve the maximum number of chunks selected for any rank */ + if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&chunk_list_num_entries, &max_num_chunks, 1, + MPI_UNSIGNED_LONG_LONG, MPI_MAX, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + + /* If no one has anything selected at all, end the operation */ + if (0 == max_num_chunks) + HGOTO_DONE(SUCCEED); + /* Set up contiguous I/O info object */ H5MM_memcpy(&ctg_io_info, io_info, sizeof(ctg_io_info)); ctg_io_info.store = &ctg_store; @@ -1827,190 +2201,147 @@ H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_i /* Initialize temporary contiguous storage info */ ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size; - ctg_store.contig.dset_addr = 0; - - /* Set dataset storage for I/O info */ - io_info->store = &store; if (io_info->op_type == H5D_IO_OP_READ) { /* Filtered collective read */ - for (i = 0; i < chunk_list_num_entries; i++) - if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't process chunk entry") - } /* end if */ + for (i = 0; i < max_num_chunks; i++) { + /* Check if this rank has a chunk to work on for this iteration */ + have_chunk_to_process = (i < chunk_list_num_entries); + + if (H5D__mpio_collective_filtered_chunk_read(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, io_info, type_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't read filtered chunks") + + if (have_chunk_to_process && chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + } else { /* Filtered collective write */ H5D_chk_idx_info_t index_info; - H5D_chunk_ud_t udata; - size_t max_num_chunks; hsize_t mpi_buf_count; /* Construct chunked index info */ - index_info.f = io_info->dset->oloc.file; - index_info.pline = &(io_info->dset->shared->dcpl_cache.pline); - index_info.layout = &(io_info->dset->shared->layout.u.chunk); - index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk); - - /* Set up chunk information for insertion to chunk index */ - udata.common.layout = index_info.layout; - udata.common.storage = index_info.storage; - udata.filter_mask = 0; - - /* Retrieve the maximum number of chunks being written among all processes */ - if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&chunk_list_num_entries, &max_num_chunks, 1, - MPI_UNSIGNED_LONG_LONG, MPI_MAX, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) - - /* If no one is writing anything at all, end the operation */ - if (!(max_num_chunks > 0)) - HGOTO_DONE(SUCCEED); - - /* Allocate arrays for storing MPI file and mem types and whether or not the - * types were derived. - */ - if (NULL == (file_type_array = (MPI_Datatype *)H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type array") - - if (NULL == (file_type_is_derived_array = (hbool_t *)H5MM_calloc(max_num_chunks * sizeof(hbool_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type is derived array") - - if (NULL == (mem_type_array = (MPI_Datatype *)H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type array") - - if (NULL == (mem_type_is_derived_array = (hbool_t *)H5MM_calloc(max_num_chunks * sizeof(hbool_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type is derived array") - - /* Iterate over the max number of chunks among all processes, as this process could - * have no chunks left to work on, but it still needs to participate in the collective - * re-allocation and re-insertion of chunks modified by other processes. + H5D_MPIO_INIT_CHUNK_IDX_INFO(index_info, io_info); + + if (mpi_size > 1) { + /* Redistribute shared chunks being written to */ + if (H5D__mpio_redistribute_shared_chunks(chunk_list, chunk_list_num_entries, io_info, fm, + mpi_rank, mpi_size, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks") + + /* Send any chunk modification messages for chunks this rank no longer owns */ + if (H5D__mpio_share_chunk_modification_data(chunk_list, &chunk_list_num_entries, io_info, + type_info, mpi_rank, mpi_size, &chunk_hash_table, + &chunk_msg_bufs, &chunk_msg_bufs_len) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "unable to send chunk modification data between MPI ranks") + } + + /* Iterate over the max number of chunks among all ranks, as this rank could + * have no chunks left to work on, but it still needs to participate in the + * collective re-allocation and re-insertion of chunks modified by other ranks. */ for (i = 0; i < max_num_chunks; i++) { - /* Check if this process has a chunk to work on for this iteration */ - hbool_t have_chunk_to_process = - (i < chunk_list_num_entries) && (mpi_rank == chunk_list[i].owners.new_owner); + /* Check if this rank has a chunk to work on for this iteration */ + have_chunk_to_process = (i < chunk_list_num_entries) && (mpi_rank == chunk_list[i].new_owner); - if (have_chunk_to_process) - if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry") - - /* Gather the new chunk sizes to all processes for a collective re-allocation - * of the chunks in the file + /* Proceed to update the chunk this rank owns (if any left) with its + * own modification data and data from other ranks, before re-filtering + * the chunks. As chunk reads are done collectively here, all ranks + * must participate. */ - if (H5D__mpio_array_gatherv(&chunk_list[i], have_chunk_to_process ? 1 : 0, - sizeof(H5D_filtered_collective_io_info_t), - (void **)&collective_chunk_list, &collective_chunk_list_num_entries, - true, 0, io_info->comm, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes") - - /* Participate in the collective re-allocation of all chunks modified - * in this iteration. + if (H5D__mpio_collective_filtered_chunk_update(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, chunk_hash_table, + chunk_msg_bufs, chunk_msg_bufs_len, io_info, + type_info, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't update modified chunks") + + /* All ranks now collectively re-allocate file space for all chunks */ + if (H5D__mpio_collective_filtered_chunk_reallocate(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, NULL, io_info, + &index_info, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-allocate file space for chunks") + + /* + * If this rank has a chunk to work on, create a MPI type + * for writing out the chunk. Otherwise, the rank will + * use MPI_BYTE for the file and memory type and specify + * a count of 0. */ - for (j = 0; j < collective_chunk_list_num_entries; j++) { - hbool_t insert = FALSE; - - if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[j].chunk_states.chunk_current, - &collective_chunk_list[j].chunk_states.new_chunk, &insert, - chunk_list[j].scaled) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") - } /* end for */ - - if (NULL == - (has_chunk_selected_array = (hbool_t *)H5MM_malloc((size_t)mpi_size * sizeof(hbool_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array") + if (H5D__mpio_collective_filtered_io_type( + have_chunk_to_process ? &chunk_list[i] : NULL, have_chunk_to_process ? 1 : 0, + io_info->op_type, &mem_type, &mem_type_is_derived, &file_type, &file_type_is_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "couldn't create MPI type for writing filtered chunks") - if (MPI_SUCCESS != - (mpi_code = MPI_Allgather(&have_chunk_to_process, 1, MPI_C_BOOL, has_chunk_selected_array, 1, - MPI_C_BOOL, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + mpi_buf_count = (file_type_is_derived || mem_type_is_derived) ? 1 : 0; - /* If this process has a chunk to work on, create a MPI type for the - * memory and file for writing out the chunk - */ + /* Override the write buffer to point to the chunk data buffer */ if (have_chunk_to_process) { - size_t offset; - int mpi_type_count; - - for (j = 0, offset = 0; j < (size_t)mpi_rank; j++) - offset += has_chunk_selected_array[j]; - - /* Collect the new chunk info back to the local copy, since only the record in the - * collective array gets updated by the chunk re-allocation */ - H5MM_memcpy(&chunk_list[i].chunk_states.new_chunk, - &collective_chunk_list[offset].chunk_states.new_chunk, - sizeof(chunk_list[i].chunk_states.new_chunk)); - - H5_CHECKED_ASSIGN(mpi_type_count, int, chunk_list[i].chunk_states.new_chunk.length, hsize_t); - - /* Create MPI memory type for writing to chunk */ - if (MPI_SUCCESS != - (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &mem_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - mem_type_is_derived_array[i] = TRUE; - - /* Create MPI file type for writing to chunk */ - if (MPI_SUCCESS != - (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &file_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - file_type_is_derived_array[i] = TRUE; - - mpi_buf_count = 1; - - /* Set up the base storage address for this operation */ - ctg_store.contig.dset_addr = chunk_list[i].chunk_states.new_chunk.offset; - - /* Override the write buffer to point to the address of the - * chunk data buffer + /* + * Override the write buffer to point to the + * chunk's data buffer */ ctg_io_info.u.wbuf = chunk_list[i].buf; - } /* end if */ - else { - mem_type_array[i] = file_type_array[i] = MPI_BYTE; - mpi_buf_count = 0; - } /* end else */ + + /* + * Setup the base storage address for this + * operation to be the chunk's file address + */ + ctg_store.contig.dset_addr = chunk_list[i].chunk_new.offset; + } + else + ctg_store.contig.dset_addr = 0; /* Perform the I/O */ - if (H5D__final_collective_io(&ctg_io_info, type_info, mpi_buf_count, file_type_array[i], - mem_type_array[i]) < 0) + if (H5D__final_collective_io(&ctg_io_info, type_info, mpi_buf_count, file_type, mem_type) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO") + /* Free up resources in anticipation of following collective operation */ + if (have_chunk_to_process && chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + /* Participate in the collective re-insertion of all chunks modified * in this iteration into the chunk index */ - for (j = 0; j < collective_chunk_list_num_entries; j++) { - udata.chunk_block = collective_chunk_list[j].chunk_states.new_chunk; - udata.common.scaled = collective_chunk_list[j].scaled; - udata.chunk_idx = collective_chunk_list[j].index; - - if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, - "unable to insert chunk address into index") - } /* end for */ + if (H5D__mpio_collective_filtered_chunk_reinsert(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, NULL, io_info, + &index_info, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-insert modified chunks into chunk index") + + /* Free the MPI types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + mem_type_is_derived = FALSE; + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + file_type_is_derived = FALSE; + } /* end for */ + } - if (collective_chunk_list) { - H5MM_free(collective_chunk_list); - collective_chunk_list = NULL; - } /* end if */ - if (has_chunk_selected_array) { - H5MM_free(has_chunk_selected_array); - has_chunk_selected_array = NULL; - } /* end if */ - } /* end for */ +done: + /* Free the MPI buf and file types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - /* Free the MPI file and memory types, if they were derived */ - for (i = 0; i < max_num_chunks; i++) { - if (file_type_is_derived_array[i]) - if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type_array[i]))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (chunk_msg_bufs) { + for (i = 0; i < (size_t)chunk_msg_bufs_len; i++) + H5MM_free(chunk_msg_bufs[i]); - if (mem_type_is_derived_array[i]) - if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type_array[i]))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - } /* end for */ - } /* end else */ + H5MM_free(chunk_msg_bufs); + } -done: + HASH_CLEAR(hh, chunk_hash_table); + + /* Free resources used by a rank which had some selection */ if (chunk_list) { for (i = 0; i < chunk_list_num_entries; i++) if (chunk_list[i].buf) @@ -2019,16 +2350,10 @@ done: H5MM_free(chunk_list); } /* end if */ - if (collective_chunk_list) - H5MM_free(collective_chunk_list); - if (file_type_array) - H5MM_free(file_type_array); - if (mem_type_array) - H5MM_free(mem_type_array); - if (file_type_is_derived_array) - H5MM_free(file_type_is_derived_array); - if (mem_type_is_derived_array) - H5MM_free(mem_type_is_derived_array); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__multi_chunk_filtered_collective_io() */ @@ -2054,11 +2379,22 @@ H5D__inter_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf hbool_t mbt_is_derived = FALSE; hbool_t mft_is_derived = FALSE; MPI_Datatype mpi_file_type, mpi_buf_type; - int mpi_code; /* MPI return code */ - herr_t ret_value = SUCCEED; /* return value */ + int mpi_code; /* MPI return code */ +#ifdef H5Dmpio_DEBUG + int mpi_rank; +#endif + herr_t ret_value = SUCCEED; /* return value */ FUNC_ENTER_STATIC +#ifdef H5Dmpio_DEBUG + mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file); + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Inter collective I/O"); + if (mpi_rank < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI rank") +#endif + if ((file_space != NULL) && (mem_space != NULL)) { int mpi_file_count; /* Number of file "objects" to transfer */ hsize_t *permute_map = NULL; /* array that holds the mapping from the old, @@ -2117,9 +2453,8 @@ H5D__inter_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf mft_is_derived = FALSE; } /* end else */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before final collective IO \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before final collective I/O"); #endif /* Perform final collective I/O operation */ @@ -2133,9 +2468,10 @@ done: if (mft_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mpi_file_type))) HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before leaving inter_collective_io ret_value = %d\n", ret_value); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "before leaving inter_collective_io ret_value = %d", ret_value); + H5D_MPIO_TRACE_EXIT(mpi_rank); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -2157,10 +2493,21 @@ static herr_t H5D__final_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t mpi_buf_count, MPI_Datatype mpi_file_type, MPI_Datatype mpi_buf_type) { +#ifdef H5Dmpio_DEBUG + int mpi_rank; +#endif herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC +#ifdef H5Dmpio_DEBUG + mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file); + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Final collective I/O"); + if (mpi_rank < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI rank") +#endif + /* Pass buf type, file type to the file driver. */ if (H5CX_set_mpi_coll_datatypes(mpi_buf_type, mpi_file_type) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O collective I/O datatypes") @@ -2175,10 +2522,12 @@ H5D__final_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf } /* end else */ done: -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "ret_value before leaving final_collective_io=%d\n", ret_value); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "ret_value before leaving final_collective_io=%d", ret_value); + H5D_MPIO_TRACE_EXIT(mpi_rank); #endif + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__final_collective_io */ @@ -2220,62 +2569,149 @@ H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2) * * Return: -1, 0, 1 * - * Programmer: Jordan Henderson - * Wednesday, Nov. 30th, 2016 - * *------------------------------------------------------------------------- */ static int H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2) { - haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF; + const H5D_filtered_collective_io_info_t *entry1; + const H5D_filtered_collective_io_info_t *entry2; + haddr_t addr1 = HADDR_UNDEF; + haddr_t addr2 = HADDR_UNDEF; + int ret_value; FUNC_ENTER_STATIC_NOERR - addr1 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry1) - ->chunk_states.new_chunk.offset; - addr2 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry2) - ->chunk_states.new_chunk.offset; + entry1 = (const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry1; + entry2 = (const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry2; - FUNC_LEAVE_NOAPI(H5F_addr_cmp(addr1, addr2)) -} /* end H5D__cmp_filtered_collective_io_info_entry() */ + addr1 = entry1->chunk_new.offset; + addr2 = entry2->chunk_new.offset; -#if MPI_VERSION >= 3 + /* + * If both chunk addresses are defined, H5F_addr_cmp is safe to use. + * Otherwise, if both addresses aren't defined, compared chunk + * entries based on their chunk index. Finally, if only one chunk + * address is defined, return the appropriate value based on which + * is defined. + */ + if (H5F_addr_defined(addr1) && H5F_addr_defined(addr2)) { + ret_value = H5F_addr_cmp(addr1, addr2); + } + else if (!H5F_addr_defined(addr1) && !H5F_addr_defined(addr2)) { + hsize_t chunk_idx1 = entry1->index_info.chunk_idx; + hsize_t chunk_idx2 = entry2->index_info.chunk_idx; + + ret_value = (chunk_idx1 > chunk_idx2) - (chunk_idx1 < chunk_idx2); + } + else + ret_value = H5F_addr_defined(addr1) ? 1 : -1; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__cmp_filtered_collective_io_info_entry() */ /*------------------------------------------------------------------------- - * Function: H5D__cmp_filtered_collective_io_info_entry_owner + * Function: H5D__cmp_chunk_redistribute_info * - * Purpose: Routine to compare filtered collective chunk io info - * entries's original owner fields + * Purpose: Routine to compare two H5D_chunk_redistribute_info_t + * structures * - * Description: Callback for qsort() to compare filtered collective chunk - * io info entries's original owner fields + * Description: Callback for qsort() to compare two + * H5D_chunk_redistribute_info_t structures + * + * Return: -1, 0, 1 + * + *------------------------------------------------------------------------- + */ +static int +H5D__cmp_chunk_redistribute_info(const void *_entry1, const void *_entry2) +{ + const H5D_chunk_redistribute_info_t *entry1; + const H5D_chunk_redistribute_info_t *entry2; + hsize_t chunk_index1; + hsize_t chunk_index2; + int ret_value; + + FUNC_ENTER_STATIC_NOERR + + entry1 = (const H5D_chunk_redistribute_info_t *)_entry1; + entry2 = (const H5D_chunk_redistribute_info_t *)_entry2; + + chunk_index1 = entry1->chunk_idx; + chunk_index2 = entry2->chunk_idx; + + if (chunk_index1 == chunk_index2) { + int orig_owner1 = entry1->orig_owner; + int orig_owner2 = entry2->orig_owner; + + ret_value = (orig_owner1 > orig_owner2) - (orig_owner1 < orig_owner2); + } + else + ret_value = (chunk_index1 > chunk_index2) - (chunk_index1 < chunk_index2); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__cmp_chunk_redistribute_info() */ + +/*------------------------------------------------------------------------- + * Function: H5D__cmp_chunk_redistribute_info_orig_owner * - * Return: The difference between the two - * H5D_filtered_collective_io_info_t's original owner fields + * Purpose: Routine to compare the original owning MPI rank for two + * H5D_chunk_redistribute_info_t structures * - * Programmer: Jordan Henderson - * Monday, Apr. 10th, 2017 + * Description: Callback for qsort() to compare the original owning MPI + * rank for two H5D_chunk_redistribute_info_t + * structures + * + * Return: -1, 0, 1 * *------------------------------------------------------------------------- */ static int -H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, - const void *filtered_collective_io_info_entry2) +H5D__cmp_chunk_redistribute_info_orig_owner(const void *_entry1, const void *_entry2) { - int owner1 = -1, owner2 = -1; + const H5D_chunk_redistribute_info_t *entry1; + const H5D_chunk_redistribute_info_t *entry2; + int owner1 = -1; + int owner2 = -1; + int ret_value; FUNC_ENTER_STATIC_NOERR - owner1 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry1) - ->owners.original_owner; - owner2 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry2) - ->owners.original_owner; + entry1 = (const H5D_chunk_redistribute_info_t *)_entry1; + entry2 = (const H5D_chunk_redistribute_info_t *)_entry2; - FUNC_LEAVE_NOAPI(owner1 - owner2) -} /* end H5D__cmp_filtered_collective_io_info_entry_owner() */ -#endif + owner1 = entry1->orig_owner; + owner2 = entry2->orig_owner; + + if (owner1 == owner2) { + haddr_t addr1 = entry1->chunk_block.offset; + haddr_t addr2 = entry2->chunk_block.offset; + + /* + * If both chunk addresses are defined, H5F_addr_cmp is safe to use. + * Otherwise, if both addresses aren't defined, compared chunk + * entries based on their chunk index. Finally, if only one chunk + * address is defined, return the appropriate value based on which + * is defined. + */ + if (H5F_addr_defined(addr1) && H5F_addr_defined(addr2)) { + ret_value = H5F_addr_cmp(addr1, addr2); + } + else if (!H5F_addr_defined(addr1) && !H5F_addr_defined(addr2)) { + hsize_t chunk_idx1 = entry1->chunk_idx; + hsize_t chunk_idx2 = entry2->chunk_idx; + + ret_value = (chunk_idx1 > chunk_idx2) - (chunk_idx1 < chunk_idx2); + } + else + ret_value = H5F_addr_defined(addr1) ? 1 : -1; + } + else + ret_value = (owner1 > owner2) - (owner1 < owner2); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__cmp_chunk_redistribute_info_orig_owner() */ /*------------------------------------------------------------------------- * Function: H5D__sort_chunk @@ -2304,26 +2740,24 @@ H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective */ static herr_t H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, - H5D_chunk_addr_info_t chunk_addr_info_array[], int sum_chunk) + H5D_chunk_addr_info_t chunk_addr_info_array[], int sum_chunk, int mpi_rank, int mpi_size) { - H5SL_node_t * chunk_node; /* Current node in chunk skip list */ - H5D_chunk_info_t *chunk_info; /* Current chunking info. of this node. */ - haddr_t chunk_addr; /* Current chunking address of this node */ - haddr_t *total_chunk_addr_array = NULL; /* The array of chunk address for the total number of chunk */ - hbool_t do_sort = FALSE; /* Whether the addresses need to be sorted */ - int bsearch_coll_chunk_threshold; - int many_chunk_opt = H5D_OBTAIN_ONE_CHUNK_ADDR_IND; - int mpi_size; /* Number of MPI processes */ - int mpi_code; /* MPI return code */ - int i; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + H5SL_node_t * chunk_node; /* Current node in chunk skip list */ + H5D_chunk_info_t *chunk_info; /* Current chunking info. of this node. */ + haddr_t chunk_addr; /* Current chunking address of this node */ + haddr_t *total_chunk_addr_array = NULL; /* The array of chunk address for the total number of chunk */ + H5P_coll_md_read_flag_t md_reads_file_flag; + hbool_t md_reads_context_flag; + hbool_t restore_md_reads_state = FALSE; + hbool_t do_sort = FALSE; /* Whether the addresses need to be sorted */ + int bsearch_coll_chunk_threshold; + int many_chunk_opt = H5D_OBTAIN_ONE_CHUNK_ADDR_IND; + int mpi_code; /* MPI return code */ + int i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC - /* Retrieve # of MPI processes */ - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") - /* Calculate the actual threshold to obtain all chunk addresses collectively * The bigger this number is, the more possible the use of obtaining chunk * address collectively. @@ -2337,31 +2771,56 @@ H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, ((sum_chunk / mpi_size) >= H5D_ALL_CHUNK_ADDR_THRES_COL_NUM)) many_chunk_opt = H5D_OBTAIN_ALL_CHUNK_ADDR_COL; -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "many_chunk_opt= %d\n", many_chunk_opt); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "many_chunk_opt = %d", many_chunk_opt); #endif /* If we need to optimize the way to obtain the chunk address */ if (many_chunk_opt != H5D_OBTAIN_ONE_CHUNK_ADDR_IND) { - int mpi_rank; - -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "Coming inside H5D_OBTAIN_ALL_CHUNK_ADDR_COL\n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "Coming inside H5D_OBTAIN_ALL_CHUNK_ADDR_COL"); #endif /* Allocate array for chunk addresses */ if (NULL == (total_chunk_addr_array = (haddr_t *)H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate memory chunk address array") - /* Retrieve all the chunk addresses with process 0 */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if (mpi_rank == 0) { - if (H5D__chunk_addrmap(io_info, total_chunk_addr_array) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + herr_t result; + + /* + * If enabled, disable collective metadata reads here. + * Since the chunk address mapping is done on rank 0 + * only here, it will cause problems if collective + * metadata reads are enabled. + */ + if (H5F_get_coll_metadata_reads(io_info->dset->oloc.file)) { + md_reads_file_flag = H5P_FORCE_FALSE; + md_reads_context_flag = FALSE; + H5F_set_coll_metadata_reads(io_info->dset->oloc.file, &md_reads_file_flag, + &md_reads_context_flag); + restore_md_reads_state = TRUE; + } + + result = H5D__chunk_addrmap(io_info, total_chunk_addr_array); + + /* Ensure that we restore the old collective metadata reads state */ + if (restore_md_reads_state) { + H5F_set_coll_metadata_reads(io_info->dset->oloc.file, &md_reads_file_flag, + &md_reads_context_flag); + restore_md_reads_state = FALSE; + } + + if (result < 0) { + size_t u; + + /* Clear total chunk address array */ + for (u = 0; u < (size_t)fm->layout->u.chunk.nchunks; u++) + total_chunk_addr_array[u] = HADDR_UNDEF; + + /* Push error, but still participate in following MPI_Bcast */ + HDONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + } } /* end if */ /* Broadcasting the MPI_IO option info. and chunk address info. */ @@ -2405,10 +2864,10 @@ H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, chunk_node = H5SL_next(chunk_node); } /* end while */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before Qsort\n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before Qsort"); #endif + if (do_sort) { size_t num_chunks = H5SL_count(fm->sel_chunks); @@ -2416,6 +2875,10 @@ H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, } /* end if */ done: + /* Re-enable collective metadata reads if we disabled them */ + if (restore_md_reads_state) + H5F_set_coll_metadata_reads(io_info->dset->oloc.file, &md_reads_file_flag, &md_reads_context_flag); + if (total_chunk_addr_array) H5MM_xfree(total_chunk_addr_array); @@ -2432,7 +2895,7 @@ done: * * 1) Each process provides two piece of information for all chunks having selection * a) chunk index - * b) wheather this chunk is regular(for MPI derived datatype not working case) + * b) whether this chunk is regular(for MPI derived datatype not working case) * * 2) Gather all the information to the root process * @@ -2461,22 +2924,24 @@ done: */ static herr_t H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assign_io_mode[], - haddr_t chunk_addr[]) + haddr_t chunk_addr[], int mpi_rank, int mpi_size) { - size_t total_chunks; - unsigned percent_nproc_per_chunk, threshold_nproc_per_chunk; - uint8_t * io_mode_info = NULL; - uint8_t * recv_io_mode_info = NULL; - uint8_t * mergebuf = NULL; - uint8_t * tempbuf; - H5SL_node_t * chunk_node; - H5D_chunk_info_t *chunk_info; - int mpi_size, mpi_rank; - MPI_Comm comm; - int root; - size_t ic; - int mpi_code; - herr_t ret_value = SUCCEED; + size_t total_chunks; + unsigned percent_nproc_per_chunk, threshold_nproc_per_chunk; + uint8_t * io_mode_info = NULL; + uint8_t * recv_io_mode_info = NULL; + uint8_t * mergebuf = NULL; + uint8_t * tempbuf; + H5SL_node_t * chunk_node; + H5D_chunk_info_t * chunk_info; + H5P_coll_md_read_flag_t md_reads_file_flag; + hbool_t md_reads_context_flag; + hbool_t restore_md_reads_state = FALSE; + MPI_Comm comm; + int root; + size_t ic; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -2484,12 +2949,6 @@ H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assig root = 0; comm = io_info->comm; - /* Obtain the number of process and the current rank of the process */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") - /* Setup parameters */ H5_CHECKED_ASSIGN(total_chunks, size_t, fm->layout->u.chunk.nchunks, hsize_t); if (H5CX_get_mpio_chunk_opt_ratio(&percent_nproc_per_chunk) < 0) @@ -2536,6 +2995,20 @@ H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assig size_t nproc; unsigned *nproc_per_chunk; + /* + * If enabled, disable collective metadata reads here. + * Since the chunk address mapping is done on rank 0 + * only here, it will cause problems if collective + * metadata reads are enabled. + */ + if (H5F_get_coll_metadata_reads(io_info->dset->oloc.file)) { + md_reads_file_flag = H5P_FORCE_FALSE; + md_reads_context_flag = FALSE; + H5F_set_coll_metadata_reads(io_info->dset->oloc.file, &md_reads_file_flag, + &md_reads_context_flag); + restore_md_reads_state = TRUE; + } + /* pre-computing: calculate number of processes and regularity of the selection occupied in each chunk */ if (NULL == (nproc_per_chunk = (unsigned *)H5MM_calloc(total_chunks * sizeof(unsigned)))) @@ -2602,6 +3075,10 @@ H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assig #endif done: + /* Re-enable collective metadata reads if we disabled them */ + if (restore_md_reads_state) + H5F_set_coll_metadata_reads(io_info->dset->oloc.file, &md_reads_file_flag, &md_reads_context_flag); + if (io_mode_info) H5MM_free(io_mode_info); if (mergebuf) @@ -2615,34 +3092,32 @@ done: } /* end H5D__obtain_mpio_mode() */ /*------------------------------------------------------------------------- - * Function: H5D__construct_filtered_io_info_list + * Function: H5D__mpio_collective_filtered_chunk_io_setup * * Purpose: Constructs a list of entries which contain the necessary * information for inter-process communication when performing * collective io on filtered chunks. This list is used by - * each process when performing I/O on locally selected chunks - * and also in operations that must be collectively done - * on every chunk, such as chunk re-allocation, insertion of - * chunks into the chunk index, etc. + * each MPI rank when performing I/O on locally selected + * chunks and also in operations that must be collectively + * done on every chunk, such as chunk re-allocation, insertion + * of chunks into the chunk index, etc. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Tuesday, January 10th, 2017 - * *------------------------------------------------------------------------- */ static herr_t -H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries) +H5D__mpio_collective_filtered_chunk_io_setup(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5D_chunk_map_t * fm, + H5D_filtered_collective_io_info_t **chunk_list, + size_t *num_entries, int mpi_rank) { - H5D_filtered_collective_io_info_t *local_info_array = - NULL; /* The list of initially selected chunks for this process */ - size_t num_chunks_selected; - size_t i; - int mpi_rank; - herr_t ret_value = SUCCEED; + H5D_filtered_collective_io_info_t *local_info_array = NULL; + H5D_chunk_ud_t udata; + hbool_t filter_partial_edge_chunks; + size_t num_chunks_selected; + size_t i; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -2652,19 +3127,23 @@ H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_typ HDassert(chunk_list); HDassert(num_entries); - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Filtered Collective I/O Setup"); +#endif - /* Each process builds a local list of the chunks they have selected */ + /* Each rank builds a local list of the chunks they have selected */ if ((num_chunks_selected = H5SL_count(fm->sel_chunks))) { H5D_chunk_info_t *chunk_info; - H5D_chunk_ud_t udata; H5SL_node_t * chunk_node; hsize_t select_npoints; - hssize_t chunk_npoints; + hbool_t need_sort = FALSE; - if (NULL == (local_info_array = (H5D_filtered_collective_io_info_t *)H5MM_malloc( - num_chunks_selected * sizeof(H5D_filtered_collective_io_info_t)))) + /* Determine whether partial edge chunks should be filtered */ + filter_partial_edge_chunks = !(io_info->dset->shared->layout.u.chunk.flags & + H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS); + + if (NULL == (local_info_array = H5MM_malloc(num_chunks_selected * sizeof(*local_info_array)))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate local io info array buffer") chunk_node = H5SL_first(fm->sel_chunks); @@ -2675,743 +3154,2846 @@ H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_typ if (H5D__chunk_lookup(io_info->dset, chunk_info->scaled, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") - local_info_array[i].index = chunk_info->index; - local_info_array[i].chunk_states.chunk_current = local_info_array[i].chunk_states.new_chunk = - udata.chunk_block; - local_info_array[i].num_writers = 0; - local_info_array[i].owners.original_owner = local_info_array[i].owners.new_owner = mpi_rank; - local_info_array[i].buf = NULL; - - local_info_array[i].async_info.num_receive_requests = 0; - local_info_array[i].async_info.receive_buffer_array = NULL; - local_info_array[i].async_info.receive_requests_array = NULL; - - H5MM_memcpy(local_info_array[i].scaled, chunk_info->scaled, sizeof(chunk_info->scaled)); - - select_npoints = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); - local_info_array[i].io_size = (size_t)select_npoints * type_info->src_type_size; - - /* Currently the full overwrite status of a chunk is only obtained on a per-process - * basis. This means that if the total selection in the chunk, as determined by the combination - * of selections of all of the processes interested in the chunk, covers the entire chunk, - * the performance optimization of not reading the chunk from the file is still valid, but - * is not applied in the current implementation. Something like an appropriately placed - * MPI_Allreduce or a running total of the number of chunk points selected during chunk - * redistribution should suffice for implementing this case - JTH. + /* Initialize rank-local chunk info */ + local_info_array[i].chunk_info = chunk_info; + local_info_array[i].chunk_buf_size = 0; + local_info_array[i].num_writers = 0; + local_info_array[i].orig_owner = mpi_rank; + local_info_array[i].new_owner = mpi_rank; + local_info_array[i].buf = NULL; + + select_npoints = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + local_info_array[i].io_size = (size_t)select_npoints * type_info->dst_type_size; + + /* + * Determine whether this chunk will need to be read from the file. If this is + * a read operation, the chunk will be read. If this is a write operation, we + * generally need to read a filtered chunk from the file before modifying it, + * unless the chunk is being fully overwritten. + * + * TODO: Currently the full overwrite status of a chunk is only obtained on a + * per-rank basis. This means that if the total selection in the chunk, as + * determined by the combination of selections of all of the ranks interested in + * the chunk, covers the entire chunk, the performance optimization of not reading + * the chunk from the file is still valid, but is not applied in the current + * implementation. + * + * To implement this case, a few approaches were considered: + * + * - Keep a running total (distributed to each rank) of the number of chunk + * elements selected during chunk redistribution and compare that to the total + * number of elements in the chunk once redistribution is finished + * + * - Process all incoming chunk messages before doing I/O (these are currently + * processed AFTER doing I/O), combine the owning rank's selection in a chunk + * with the selections received from other ranks and check to see whether that + * combined selection covers the entire chunk + * + * The first approach will be dangerous if the application performs an overlapping + * write to a chunk, as the number of selected elements can equal or exceed the + * number of elements in the chunk without the whole chunk selection being covered. + * While it might be considered erroneous for an application to do an overlapping + * write, we don't explicitly disallow it. + * + * The second approach contains a bit of complexity in that part of the chunk + * messages will be needed before doing I/O and part will be needed after doing I/O. + * Since modification data from chunk messages can't be applied until after any I/O + * is performed (otherwise, we'll overwrite any applied modification data), chunk + * messages are currently entirely processed after I/O. However, in order to determine + * if a chunk is being fully overwritten, we need the dataspace portion of the chunk + * messages before doing I/O. The naive way to do this is to process chunk messages + * twice, using just the relevant information from the message before and after I/O. + * The better way would be to avoid processing chunk messages twice by extracting (and + * keeping around) the dataspace portion of the message before I/O and processing the + * rest of the chunk message after I/O. Note that the dataspace portion of each chunk + * message is used to correctly apply chunk modification data from the message, so + * must be kept around both before and after I/O in this case. + */ + if (io_info->op_type == H5D_IO_OP_READ) + local_info_array[i].need_read = TRUE; + else { + local_info_array[i].need_read = + local_info_array[i].io_size < (size_t)io_info->dset->shared->layout.u.chunk.size; + } + + local_info_array[i].skip_filter_pline = FALSE; + if (!filter_partial_edge_chunks) { + /* + * If this is a partial edge chunk and the "don't filter partial edge + * chunks" flag is set, make sure not to apply filters to the chunk. + */ + if (H5D__chunk_is_partial_edge_chunk(io_info->dset->shared->ndims, + io_info->dset->shared->layout.u.chunk.dim, + chunk_info->scaled, io_info->dset->shared->curr_dims)) + local_info_array[i].skip_filter_pline = TRUE; + } + + /* Initialize the chunk's shared info */ + local_info_array[i].chunk_current = udata.chunk_block; + local_info_array[i].chunk_new = udata.chunk_block; + + /* + * Check if the list is not in ascending order of offset in the file + * or has unallocated chunks. In either case, the list should get + * sorted. */ - if ((chunk_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") - local_info_array[i].full_overwrite = - (local_info_array[i].io_size >= (hsize_t)chunk_npoints * type_info->dst_type_size) ? TRUE - : FALSE; + if (i) { + haddr_t curr_chunk_offset = local_info_array[i].chunk_current.offset; + haddr_t prev_chunk_offset = local_info_array[i - 1].chunk_current.offset; + + if (!H5F_addr_defined(prev_chunk_offset) || !H5F_addr_defined(curr_chunk_offset) || + (curr_chunk_offset < prev_chunk_offset)) + need_sort = TRUE; + } + + /* + * Extensible arrays may calculate a chunk's index a little differently + * than normal when the dataset's unlimited dimension is not the + * slowest-changing dimension, so set the index here based on what the + * extensible array code calculated instead of what was calculated + * in the chunk file mapping. + */ + if (io_info->dset->shared->layout.u.chunk.idx_type == H5D_CHUNK_IDX_EARRAY) + local_info_array[i].index_info.chunk_idx = udata.chunk_idx; + else + local_info_array[i].index_info.chunk_idx = chunk_info->index; + + local_info_array[i].index_info.filter_mask = udata.filter_mask; + local_info_array[i].index_info.need_insert = FALSE; chunk_node = H5SL_next(chunk_node); - } /* end for */ - } /* end if */ + } - /* Redistribute shared chunks to new owners as necessary */ - if (io_info->op_type == H5D_IO_OP_WRITE) -#if MPI_VERSION >= 3 - if (H5D__chunk_redistribute_shared_chunks(io_info, type_info, fm, local_info_array, - &num_chunks_selected) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks") -#else - HGOTO_ERROR( - H5E_DATASET, H5E_WRITEERROR, FAIL, - "unable to redistribute shared chunks - MPI version < 3 (MPI_Mprobe and MPI_Imrecv missing)") + /* Ensure the chunk list is sorted in ascending order of offset in the file */ + if (need_sort) + HDqsort(local_info_array, num_chunks_selected, sizeof(H5D_filtered_collective_io_info_t), + H5D__cmp_filtered_collective_io_info_entry); + +#ifdef H5Dmpio_DEBUG + H5D__mpio_dump_collective_filtered_chunk_list(local_info_array, num_chunks_selected, mpi_rank); #endif + } + else if (H5F_get_coll_metadata_reads(io_info->dset->oloc.file)) { + hsize_t scaled[H5O_LAYOUT_NDIMS] = {0}; + + /* + * If this rank has no selection in the dataset and collective + * metadata reads are enabled, do a fake lookup of a chunk to + * ensure that this rank has the chunk index opened. Otherwise, + * only the ranks that had a selection will have opened the + * chunk index and they will have done so independently. Therefore, + * when ranks with no selection participate in later collective + * metadata reads, they will try to open the chunk index collectively + * and issues will occur since other ranks won't participate. + * + * In the future, we should consider having a chunk index "open" + * callback that can be used to ensure collectivity between ranks + * in a more natural way, but this hack should suffice for now. + */ + if (H5D__chunk_lookup(io_info->dset, scaled, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") + } *chunk_list = local_info_array; *num_entries = num_chunks_selected; done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__construct_filtered_io_info_list() */ +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif -#if MPI_VERSION >= 3 + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_io_setup() */ /*------------------------------------------------------------------------- - * Function: H5D__chunk_redistribute_shared_chunks - * - * Purpose: When performing a collective write on a Dataset with - * filters applied, this function is used to redistribute any - * chunks which are selected by more than one process, so as - * to preserve file integrity after the write by ensuring - * that any shared chunks are only modified by one process. - * - * The current implementation follows this 3-phase process: - * - * - Collect everyone's list of chunks into one large list, - * sort the list in increasing order of chunk offset in the - * file and hand the list off to rank 0 - * - * - Rank 0 scans the list looking for matching runs of chunk - * offset in the file (corresponding to a shared chunk which - * has been selected by more than one rank in the I/O - * operation) and for each shared chunk, it redistributes - * the chunk to the process writing to the chunk which - * currently has the least amount of chunks assigned to it - * by modifying the "new_owner" field in each of the list - * entries corresponding to that chunk - * - * - After the chunks have been redistributed, rank 0 re-sorts - * the list in order of previous owner so that each rank - * will get back exactly the array that they contributed to - * the redistribution operation, with the "new_owner" field - * of each chunk they are modifying having possibly been - * modified. Rank 0 then scatters each segment of the list - * back to its corresponding rank + * Function: H5D__mpio_redistribute_shared_chunks * - * Return: Non-negative on success/Negative on failure + * Purpose: When performing a parallel write on a chunked Dataset with + * filters applied, we must ensure that any particular chunk + * is only written to by a single MPI rank in order to avoid + * potential data races on the chunk. This function is used to + * redistribute (by assigning ownership to a single rank) any + * chunks which are selected by more than one MPI rank. + * + * An initial Allgather is performed to determine how many + * chunks each rank has selected in the write operation and + * then that number is compared against a threshold value to + * determine whether chunk redistribution should be done on + * MPI rank 0 only, or on all MPI ranks. * - * Programmer: Jordan Henderson - * Monday, May 1, 2017 + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ static herr_t -H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t *local_chunk_array, - size_t * local_chunk_array_num_entries) +H5D__mpio_redistribute_shared_chunks(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, const H5D_io_info_t *io_info, + const H5D_chunk_map_t *fm, int mpi_rank, int mpi_size, + size_t **rank_chunks_assigned_map) { - H5D_filtered_collective_io_info_t *shared_chunks_info_array = - NULL; /* The list of all chunks selected in the operation by all processes */ - H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__gather_mem */ - unsigned char **mod_data = - NULL; /* Array of chunk modification data buffers sent by a process to new chunk owners */ - MPI_Request *send_requests = NULL; /* Array of MPI_Isend chunk modification data send requests */ - MPI_Status * send_statuses = NULL; /* Array of MPI_Isend chunk modification send statuses */ - hbool_t mem_iter_init = FALSE; - size_t shared_chunks_info_array_num_entries = 0; - size_t num_send_requests = 0; - size_t * num_assigned_chunks_array = NULL; - size_t i, last_assigned_idx; - int * send_counts = NULL; - int * send_displacements = NULL; - int scatter_recvcount_int; - int mpi_rank, mpi_size, mpi_code; - herr_t ret_value = SUCCEED; + hbool_t redistribute_on_all_ranks; + size_t *num_chunks_map = NULL; + size_t coll_chunk_list_size = 0; + size_t i; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + HDassert(chunk_list || 0 == chunk_list_num_entries); HDassert(io_info); - HDassert(type_info); HDassert(fm); - HDassert(local_chunk_array_num_entries); + HDassert(mpi_size > 1); /* No chunk sharing is possible for MPI Comm size of 1 */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Redistribute shared chunks"); +#endif - /* Set to latest format for encoding dataspace */ - H5CX_set_libver_bounds(NULL); + /* + * Allocate an array for each rank to keep track of the number of + * chunks assigned to any other rank in order to cut down on future + * MPI communication. + */ + if (NULL == (num_chunks_map = H5MM_malloc((size_t)mpi_size * sizeof(*num_chunks_map)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate assigned chunks array") - if (*local_chunk_array_num_entries) - if (NULL == (send_requests = - (MPI_Request *)H5MM_malloc(*local_chunk_array_num_entries * sizeof(MPI_Request)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send requests buffer") + /* Perform initial Allgather to determine the collective chunk list size */ + if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&chunk_list_num_entries, 1, H5_SIZE_T_AS_MPI_TYPE, + num_chunks_map, 1, H5_SIZE_T_AS_MPI_TYPE, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) - if (NULL == (mem_iter = (H5S_sel_iter_t *)H5MM_malloc(sizeof(H5S_sel_iter_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + for (i = 0; i < (size_t)mpi_size; i++) + coll_chunk_list_size += num_chunks_map[i]; - /* Gather every rank's list of chunks to rank 0 to allow it to perform the redistribution operation. After - * this call, the gathered list will initially be sorted in increasing order of chunk offset in the file. + /* + * Determine whether we should perform chunk redistribution on all + * ranks or just rank 0. For a relatively small number of chunks, + * we redistribute on all ranks to cut down on MPI communication + * overhead. For a larger number of chunks, we redistribute on + * rank 0 only to cut down on memory usage. */ - if (H5D__mpio_array_gatherv(local_chunk_array, *local_chunk_array_num_entries, - sizeof(H5D_filtered_collective_io_info_t), (void **)&shared_chunks_info_array, - &shared_chunks_info_array_num_entries, false, 0, io_info->comm, - H5D__cmp_filtered_collective_io_info_entry) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather array") + redistribute_on_all_ranks = coll_chunk_list_size < H5D_CHUNK_REDISTRIBUTE_THRES; - /* Rank 0 redistributes any shared chunks to new owners as necessary */ - if (mpi_rank == 0) { - if (NULL == (send_counts = (int *)H5MM_calloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send counts buffer") + if (H5D__mpio_redistribute_shared_chunks_int(chunk_list, num_chunks_map, redistribute_on_all_ranks, + io_info, fm, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTREDISTRIBUTE, FAIL, "can't redistribute shared chunks") - if (NULL == (send_displacements = (int *)H5MM_malloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send displacements buffer") + /* + * If the caller provided a pointer for the mapping from + * rank value -> number of chunks assigned, return that + * mapping here. + */ + if (rank_chunks_assigned_map) { + /* + * If we performed chunk redistribution on rank 0 only, distribute + * the rank value -> number of chunks assigned mapping back to all + * ranks. + */ + if (!redistribute_on_all_ranks) { + if (MPI_SUCCESS != + (mpi_code = MPI_Bcast(num_chunks_map, mpi_size, H5_SIZE_T_AS_MPI_TYPE, 0, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "couldn't broadcast chunk mapping to other ranks", mpi_code) + } - if (NULL == (num_assigned_chunks_array = (size_t *)H5MM_calloc((size_t)mpi_size * sizeof(size_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "unable to allocate number of assigned chunks array") + *rank_chunks_assigned_map = num_chunks_map; + } - for (i = 0; i < shared_chunks_info_array_num_entries;) { - H5D_filtered_collective_io_info_t *chunk_entry; - haddr_t last_seen_addr = shared_chunks_info_array[i].chunk_states.chunk_current.offset; - size_t set_begin_index = i; - size_t num_writers = 0; - int new_chunk_owner = shared_chunks_info_array[i].owners.original_owner; +done: + if (!rank_chunks_assigned_map || (ret_value < 0)) { + num_chunks_map = H5MM_xfree(num_chunks_map); + } - /* Process each set of duplicate entries caused by another process writing to the same chunk */ - do { - chunk_entry = &shared_chunks_info_array[i]; +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_redistribute_shared_chunks() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_redistribute_shared_chunks_int + * + * Purpose: Routine to perform redistribution of shared chunks during + * parallel writes to datasets with filters applied. + * + * If `all_ranks_involved` is TRUE, chunk redistribution + * occurs on all MPI ranks. This is usually done when there + * is a relatively small number of chunks involved in order to + * cut down on MPI communication overhead while increasing + * total memory usage a bit. + * + * If `all_ranks_involved` is FALSE, only rank 0 will perform + * chunk redistribution. This is usually done when there is + * a relatively large number of chunks involved in order to + * cut down on total memory usage at the cost of increased + * overhead from MPI communication. + * + * This implementation is as follows: + * + * - All MPI ranks send their list of selected chunks to the + * ranks involved in chunk redistribution. Then, the + * involved ranks sort this new list in order of chunk + * index. + * + * - The involved ranks scan the list looking for matching + * runs of chunk index values (corresponding to a shared + * chunk which has been selected by more than one rank in + * the I/O operation) and for each shared chunk, + * redistribute the chunk to the MPI rank writing to the + * chunk which currently has the least amount of chunks + * assigned to it. This is done by modifying the "new_owner" + * field in each of the list entries corresponding to that + * chunk. The involved ranks then re-sort the list in order + * of original chunk owner so that each rank's section of + * contributed chunks is contiguous in the collective chunk + * list. + * + * - If chunk redistribution occurred on all ranks, each rank + * scans through the collective chunk list to find their + * contributed section of chunks and uses that to update + * their local chunk list with the newly-updated "new_owner" + * and "num_writers" fields. If chunk redistribution + * occurred only on rank 0, an MPI_Scatterv operation will + * be used to scatter the segments of the collective chunk + * list from rank 0 back to the corresponding ranks. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_redistribute_shared_chunks_int(H5D_filtered_collective_io_info_t *chunk_list, + size_t *num_chunks_assigned_map, hbool_t all_ranks_involved, + const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size) +{ + MPI_Datatype struct_type; + MPI_Datatype packed_type; + hbool_t struct_type_derived = FALSE; + hbool_t packed_type_derived = FALSE; + size_t i; + size_t coll_chunk_list_num_entries = 0; + void * coll_chunk_list = NULL; + int * counts_disps_array = NULL; + int * counts_ptr = NULL; + int * displacements_ptr = NULL; + int num_chunks_int; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(num_chunks_assigned_map); + HDassert(chunk_list || 0 == num_chunks_assigned_map[mpi_rank]); + HDassert(io_info); + HDassert(fm); + HDassert(mpi_size > 1); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Redistribute shared chunks (internal)"); +#endif + + /* + * Make sure it's safe to cast this rank's number + * of chunks to be sent into an int for MPI + */ + H5_CHECKED_ASSIGN(num_chunks_int, int, num_chunks_assigned_map[mpi_rank], size_t); + + /* + * Phase 1 - Participate in collective gathering of every rank's + * list of chunks to the ranks which are performing the redistribution + * operation. + */ + + if (all_ranks_involved || (mpi_rank == 0)) { + /* + * Allocate array to store the receive counts of each rank, as well as + * the displacements into the final array where each rank will place + * their data. The first half of the array contains the receive counts + * (in rank order), while the latter half contains the displacements + * (also in rank order). + */ + if (NULL == (counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*counts_disps_array)))) { + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + else { + /* Set the receive counts from the assigned chunks map */ + counts_ptr = counts_disps_array; + + for (i = 0; i < (size_t)mpi_size; i++) + H5_CHECKED_ASSIGN(counts_ptr[i], int, num_chunks_assigned_map[i], size_t); + + /* Set the displacements into the receive buffer for the gather operation */ + displacements_ptr = &counts_disps_array[mpi_size]; + + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + counts_ptr[i - 1]; + } + } + + /* + * Construct MPI derived types for extracting information + * necessary for MPI communication + */ + if (H5D__mpio_get_chunk_redistribute_info_types(&packed_type, &packed_type_derived, &struct_type, + &struct_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't create derived datatypes for chunk redistribution info") + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(chunk_list, num_chunks_int, struct_type, counts_ptr, displacements_ptr, + packed_type, all_ranks_involved, 0, io_info->comm, mpi_rank, mpi_size, + &coll_chunk_list, &coll_chunk_list_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, + "can't gather chunk redistribution info to involved ranks") + + /* + * If all ranks are redistributing shared chunks, we no + * longer need the receive counts and displacements array + */ + if (all_ranks_involved) { + counts_disps_array = H5MM_xfree(counts_disps_array); + } - send_counts[chunk_entry->owners.original_owner] += (int)sizeof(*chunk_entry); + /* + * Phase 2 - Involved ranks now redistribute any shared chunks to new + * owners as necessary. + */ + + if (all_ranks_involved || (mpi_rank == 0)) { + H5D_chunk_redistribute_info_t *chunk_entry; + hsize_t curr_chunk_idx; + size_t set_begin_index; + int num_writers; + int new_chunk_owner; + + /* Clear the mapping from rank value -> number of assigned chunks */ + HDmemset(num_chunks_assigned_map, 0, (size_t)mpi_size * sizeof(*num_chunks_assigned_map)); + + /* Sort collective chunk list according to chunk index */ + HDqsort(coll_chunk_list, coll_chunk_list_num_entries, sizeof(H5D_chunk_redistribute_info_t), + H5D__cmp_chunk_redistribute_info); + + /* + * Process all chunks in the collective chunk list. + * Note that the loop counter is incremented by both + * the outer loop (while processing each entry in + * the collective chunk list) and the inner loop + * (while processing duplicate entries for shared + * chunks). + */ + chunk_entry = &((H5D_chunk_redistribute_info_t *)coll_chunk_list)[0]; + for (i = 0; i < coll_chunk_list_num_entries;) { + /* Set chunk's initial new owner to its original owner */ + new_chunk_owner = chunk_entry->orig_owner; + + /* + * Set the current chunk index so we know when we've processed + * all duplicate entries for a particular shared chunk + */ + curr_chunk_idx = chunk_entry->chunk_idx; + + /* Reset the initial number of writers to this chunk */ + num_writers = 0; - /* The new owner of the chunk is determined by the process + /* Set index for the beginning of this section of duplicate chunk entries */ + set_begin_index = i; + + /* + * Process each chunk entry in the set for the current + * (possibly shared) chunk and increment the loop counter + * while doing so. + */ + do { + /* + * The new owner of the chunk is determined by the rank * writing to the chunk which currently has the least amount * of chunks assigned to it */ - if (num_assigned_chunks_array[chunk_entry->owners.original_owner] < - num_assigned_chunks_array[new_chunk_owner]) - new_chunk_owner = chunk_entry->owners.original_owner; + if (num_chunks_assigned_map[chunk_entry->orig_owner] < + num_chunks_assigned_map[new_chunk_owner]) + new_chunk_owner = chunk_entry->orig_owner; + + /* Update the number of writers to this particular chunk */ + num_writers++; + + chunk_entry++; + } while (++i < coll_chunk_list_num_entries && chunk_entry->chunk_idx == curr_chunk_idx); + + /* We should never have more writers to a chunk than the number of MPI ranks */ + HDassert(num_writers <= mpi_size); + + /* Set all processed chunk entries' "new_owner" and "num_writers" fields */ + for (; set_begin_index < i; set_begin_index++) { + H5D_chunk_redistribute_info_t *entry; + + entry = &((H5D_chunk_redistribute_info_t *)coll_chunk_list)[set_begin_index]; + + entry->new_owner = new_chunk_owner; + entry->num_writers = num_writers; + } + + /* Update the number of chunks assigned to the MPI rank that now owns this chunk */ + num_chunks_assigned_map[new_chunk_owner]++; + } + + /* + * Re-sort the collective chunk list in order of original chunk owner + * so that each rank's section of contributed chunks is contiguous in + * the collective chunk list. + * + * NOTE: this re-sort is frail in that it needs to sort the collective + * chunk list so that each rank's section of contributed chunks + * is in the exact order it was contributed in, or things will + * be scrambled when each rank's local chunk list is updated. + * Therefore, the sorting algorithm here is tied to the one + * used during the I/O setup operation. Specifically, chunks + * are first sorted by ascending order of offset in the file and + * then by chunk index. In the future, a better redistribution + * algorithm may be devised that doesn't rely on frail sorting, + * but the current implementation is a quick and naive approach. + */ + HDqsort(coll_chunk_list, coll_chunk_list_num_entries, sizeof(H5D_chunk_redistribute_info_t), + H5D__cmp_chunk_redistribute_info_orig_owner); + } + + if (all_ranks_involved) { + /* + * If redistribution occurred on all ranks, search for the section + * in the collective chunk list corresponding to this rank's locally + * selected chunks and update the local list after redistribution. + */ + for (i = 0; i < coll_chunk_list_num_entries; i++) + if (mpi_rank == ((H5D_chunk_redistribute_info_t *)coll_chunk_list)[i].orig_owner) + break; + + for (size_t j = 0; j < (size_t)num_chunks_int; j++) { + H5D_chunk_redistribute_info_t *coll_entry; + + coll_entry = &((H5D_chunk_redistribute_info_t *)coll_chunk_list)[i++]; + + chunk_list[j].new_owner = coll_entry->new_owner; + chunk_list[j].num_writers = coll_entry->num_writers; + } + } + else { + /* + * If redistribution occurred only on rank 0, scatter the segments + * of the collective chunk list back to each rank so that their + * local chunk lists get updated + */ + if (MPI_SUCCESS != + (mpi_code = MPI_Scatterv(coll_chunk_list, counts_ptr, displacements_ptr, packed_type, chunk_list, + num_chunks_int, struct_type, 0, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "unable to scatter shared chunks info buffer", mpi_code) + } + +#ifdef H5Dmpio_DEBUG + H5D__mpio_dump_collective_filtered_chunk_list(chunk_list, num_chunks_assigned_map[mpi_rank], mpi_rank); +#endif + +done: + H5MM_free(coll_chunk_list); + + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (packed_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&packed_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + + H5MM_free(counts_disps_array); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_redistribute_shared_chunks_int() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_share_chunk_modification_data + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, we must first ensure that any particular + * chunk is only written to by a single MPI rank in order to + * avoid potential data races on the chunk. Once dataset + * chunks have been redistributed in a suitable manner, each + * MPI rank must send its chunk data to other ranks for each + * chunk it no longer owns. + * + * The current implementation here follows the Nonblocking + * Consensus algorithm described in: + * http://unixer.de/publications/img/hoefler-dsde-protocols.pdf + * + * First, each MPI rank scans through its list of selected + * chunks and does the following for each chunk: + * + * * If a chunk in the MPI rank's chunk list is still owned + * by that rank, the rank checks how many messages are + * incoming for that chunk and adds that to its running + * total. Then, the rank updates its local chunk list so + * that any previous chunk entries for chunks that are no + * longer owned by the rank get overwritten by chunk + * entries for chunks the rank still owns. Since the data + * for the chunks no longer owned will have already been + * sent, those chunks can effectively be discarded. + * * If a chunk in the MPI rank's chunk list is no longer + * owned by that rank, the rank sends the data it wishes to + * update the chunk with to the MPI rank that now has + * ownership of that chunk. To do this, it encodes the + * chunk's index, its selection in the chunk and its + * modification data into a buffer and then posts a + * non-blocking MPI_Issend to the owning rank. + * + * Once this step is complete, all MPI ranks allocate arrays + * to hold chunk message receive buffers and MPI request + * objects for each non-blocking receive they will post for + * incoming chunk modification messages. Then, all MPI ranks + * enter a loop that alternates between non-blocking + * MPI_Iprobe calls to probe for incoming messages and + * MPI_Testall calls to see if all send requests have + * completed. As chunk modification messages arrive, + * non-blocking MPI_Irecv calls will be posted for each + * message. + * + * Once all send requests have completed, an MPI_Ibarrier is + * posted and the loop then alternates between MPI_Iprobe + * calls and MPI_Test calls to check if all ranks have reached + * the non-blocking barrier. Once all ranks have reached the + * barrier, processing can move on to updating the selected + * chunks that are owned in the operation. + * + * Any chunk messages that were received from other ranks + * will be returned through the `chunk_msg_bufs` array and + * `chunk_msg_bufs_len` will be set appropriately. + * + * NOTE: The use of non-blocking sends and receives of chunk + * data here may contribute to large amounts of memory + * usage/MPI request overhead if the number of shared + * chunks is high. If this becomes a problem, it may be + * useful to split the message receiving loop away so + * that chunk modification messages can be received and + * processed immediately (MPI_Recv) using a single chunk + * message buffer. However, it's possible this may + * degrade performance since the chunk message sends + * are synchronous (MPI_Issend) in the Nonblocking + * Consensus algorithm. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_share_chunk_modification_data(H5D_filtered_collective_io_info_t *chunk_list, + size_t *chunk_list_num_entries, H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, int mpi_size, + H5D_filtered_collective_io_info_t **chunk_hash_table, + unsigned char ***chunk_msg_bufs, int *chunk_msg_bufs_len) +{ +#if MPI_VERSION >= 3 + H5D_filtered_collective_io_info_t *chunk_table = NULL; + H5S_sel_iter_t * mem_iter = NULL; + unsigned char ** msg_send_bufs = NULL; + unsigned char ** msg_recv_bufs = NULL; + MPI_Request * send_requests = NULL; + MPI_Request * recv_requests = NULL; + MPI_Request ibarrier = MPI_REQUEST_NULL; + hbool_t mem_iter_init = FALSE; + hbool_t ibarrier_posted = FALSE; + size_t send_bufs_nalloc = 0; + size_t num_send_requests = 0; + size_t num_recv_requests = 0; + size_t num_msgs_incoming = 0; + size_t last_assigned_idx; + size_t i; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list_num_entries); + HDassert(chunk_list || 0 == *chunk_list_num_entries); + HDassert(io_info); + HDassert(type_info); + HDassert(mpi_size > 1); + HDassert(chunk_msg_bufs); + HDassert(chunk_msg_bufs_len); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Share chunk modification data"); +#endif + + /* Set to latest format for encoding dataspace */ + H5CX_set_libver_bounds(NULL); + + if (*chunk_list_num_entries) { + /* Allocate a selection iterator for iterating over chunk dataspaces */ + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate dataspace selection iterator") + + /* + * Allocate send buffer and MPI_Request arrays for non-blocking + * sends of outgoing chunk messages + */ + send_bufs_nalloc = H5D_CHUNK_NUM_SEND_MSGS_INIT; + if (NULL == (msg_send_bufs = H5MM_malloc(send_bufs_nalloc * sizeof(*msg_send_bufs)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message buffer array") + + if (NULL == (send_requests = H5MM_malloc(send_bufs_nalloc * sizeof(*send_requests)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send requests array") + } + + /* + * For each chunk this rank owns, add to the total number of + * incoming MPI messages, then update the local chunk list to + * overwrite any previous chunks no longer owned by this rank. + * Since the data for those chunks will have already been sent, + * this rank should no longer be interested in them and they + * can effectively be discarded. This bookkeeping also makes + * the code for the collective file space re-allocation and + * chunk re-insertion operations a bit simpler. + * + * For each chunk this rank doesn't own, use non-blocking + * synchronous sends to send the data this rank is writing to + * the rank that does own the chunk. + */ + for (i = 0, last_assigned_idx = 0; i < *chunk_list_num_entries; i++) { + H5D_filtered_collective_io_info_t *chunk_entry = &chunk_list[i]; + + if (mpi_rank == chunk_entry->new_owner) { + num_msgs_incoming += (size_t)(chunk_entry->num_writers - 1); + + /* + * Overwrite chunk entries this rank doesn't own with entries that it + * does own, since it has sent the necessary data and is no longer + * interested in the chunks it doesn't own. + */ + chunk_list[last_assigned_idx] = chunk_list[i]; + + /* + * Since, at large scale, a chunk's index value may be larger than + * the maximum value that can be stored in an int, we cannot rely + * on using a chunk's index value as the tag for the MPI messages + * sent/received for a chunk. Therefore, add this chunk to a hash + * table with the chunk's index as a key so that we can quickly find + * the chunk when processing chunk messages that were received. The + * message itself will contain the chunk's index so we can update + * the correct chunk with the received data. + */ + HASH_ADD(hh, chunk_table, index_info.chunk_idx, sizeof(hsize_t), &chunk_list[last_assigned_idx]); + + last_assigned_idx++; + } + else { + H5D_chunk_info_t *chunk_info = chunk_entry->chunk_info; + unsigned char * mod_data_p = NULL; + hsize_t iter_nelmts; + size_t mod_data_size = 0; + size_t space_size = 0; + + /* Add the size of the chunk index to the encoded size */ + mod_data_size += sizeof(hsize_t); + + /* Determine size of serialized chunk file dataspace */ + if (H5S_encode(chunk_info->fspace, &mod_data_p, &space_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get encoded dataspace size") + mod_data_size += space_size; + + /* Determine size of data being written */ + iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + H5_CHECK_OVERFLOW(iter_nelmts, hsize_t, size_t); + + mod_data_size += (size_t)iter_nelmts * type_info->src_type_size; + + if (NULL == (msg_send_bufs[num_send_requests] = H5MM_malloc(mod_data_size))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message buffer") + + mod_data_p = msg_send_bufs[num_send_requests]; + + /* Store the chunk's index into the buffer */ + HDmemcpy(mod_data_p, &chunk_entry->index_info.chunk_idx, sizeof(hsize_t)); + mod_data_p += sizeof(hsize_t); + + /* Serialize the chunk's file dataspace into the buffer */ + if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to encode dataspace") + + /* Initialize iterator for memory selection */ + if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size, + H5S_SEL_ITER_SHARE_WITH_DATASPACE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; + + /* Collect the modification data into the buffer */ + if (0 == H5D__gather_mem(io_info->u.wbuf, mem_iter, (size_t)iter_nelmts, mod_data_p)) + HGOTO_ERROR(H5E_IO, H5E_CANTGATHER, FAIL, "couldn't gather from write buffer") + + /* + * Ensure that the size of the chunk data being sent can be + * safely cast to an int for MPI. Note that this should + * generally be OK for now (unless a rank is sending a + * whole 32-bit-sized chunk of data + its encoded selection), + * but if we allow larger than 32-bit-sized chunks in the + * future, this may become a problem and derived datatypes + * will need to be used. + */ + H5_CHECK_OVERFLOW(mod_data_size, size_t, int) + + /* Send modification data to new owner */ + if (MPI_SUCCESS != + (mpi_code = MPI_Issend(msg_send_bufs[num_send_requests], (int)mod_data_size, MPI_BYTE, + chunk_entry->new_owner, H5D_CHUNK_MOD_DATA_TAG, io_info->comm, + &send_requests[num_send_requests]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Issend failed", mpi_code) + + num_send_requests++; + + /* Resize send buffer and send request arrays if necessary */ + if (num_send_requests == send_bufs_nalloc) { + void *tmp_alloc; + + send_bufs_nalloc = (size_t)((double)send_bufs_nalloc * 1.5); + + if (NULL == + (tmp_alloc = H5MM_realloc(msg_send_bufs, send_bufs_nalloc * sizeof(*msg_send_bufs)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't resize chunk modification message buffer array") + msg_send_bufs = tmp_alloc; + + if (NULL == + (tmp_alloc = H5MM_realloc(send_requests, send_bufs_nalloc * sizeof(*send_requests)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't resize send requests array") + send_requests = tmp_alloc; + } + + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release memory selection iterator") + mem_iter_init = FALSE; + } + } + + /* Check if the number of send or receive requests will overflow an int (MPI requirement) */ + if (num_send_requests > INT_MAX || num_msgs_incoming > INT_MAX) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "too many shared chunks in parallel filtered write operation") + + H5_CHECK_OVERFLOW(num_send_requests, size_t, int) + H5_CHECK_OVERFLOW(num_msgs_incoming, size_t, int) + + /* + * Allocate receive buffer and MPI_Request arrays for non-blocking + * receives of incoming chunk messages + */ + if (num_msgs_incoming) { + if (NULL == (msg_recv_bufs = H5MM_malloc(num_msgs_incoming * sizeof(*msg_recv_bufs)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message buffer array") + + if (NULL == (recv_requests = H5MM_malloc(num_msgs_incoming * sizeof(*recv_requests)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive requests array") + } + + /* Process any incoming messages until everyone is done */ + do { + MPI_Status status; + int msg_flag; + + /* Probe for an incoming message from any rank */ + if (MPI_SUCCESS != (mpi_code = MPI_Iprobe(MPI_ANY_SOURCE, H5D_CHUNK_MOD_DATA_TAG, io_info->comm, + &msg_flag, &status))) + HMPI_GOTO_ERROR(FAIL, "MPI_Iprobe failed", mpi_code) + + /* + * If a message was found, allocate a buffer for the message and + * post a non-blocking receive to receive it + */ + if (msg_flag) { +#if MPI_VERSION >= 3 + MPI_Count msg_size = 0; + + if (MPI_SUCCESS != (mpi_code = MPI_Get_elements_x(&status, MPI_BYTE, &msg_size))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements_x failed", mpi_code) + + H5_CHECK_OVERFLOW(msg_size, MPI_Count, int) +#else + int msg_size = 0; + + if (MPI_SUCCESS != (mpi_code = MPI_Get_elements(&status, MPI_BYTE, &msg_size))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements failed", mpi_code) +#endif + + if (msg_size <= 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid chunk modification message size") + + HDassert((num_recv_requests + 1) <= num_msgs_incoming); + if (NULL == + (msg_recv_bufs[num_recv_requests] = H5MM_malloc((size_t)msg_size * sizeof(unsigned char)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message receive buffer") + + if (MPI_SUCCESS != (mpi_code = MPI_Irecv(msg_recv_bufs[num_recv_requests], (int)msg_size, + MPI_BYTE, status.MPI_SOURCE, H5D_CHUNK_MOD_DATA_TAG, + io_info->comm, &recv_requests[num_recv_requests]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Irecv failed", mpi_code) + + num_recv_requests++; + } + + if (ibarrier_posted) { + int ibarrier_completed; + + if (MPI_SUCCESS != (mpi_code = MPI_Test(&ibarrier, &ibarrier_completed, MPI_STATUS_IGNORE))) + HMPI_GOTO_ERROR(FAIL, "MPI_Test failed", mpi_code) + + if (ibarrier_completed) + break; + } + else { + int all_sends_completed; + + /* Determine if all send requests have completed */ + if (MPI_SUCCESS != (mpi_code = MPI_Testall((int)num_send_requests, send_requests, + &all_sends_completed, MPI_STATUSES_IGNORE))) + HMPI_GOTO_ERROR(FAIL, "MPI_Testall failed", mpi_code) + + if (all_sends_completed) { + /* Post non-blocking barrier */ + if (MPI_SUCCESS != (mpi_code = MPI_Ibarrier(io_info->comm, &ibarrier))) + HMPI_GOTO_ERROR(FAIL, "MPI_Ibarrier failed", mpi_code) + ibarrier_posted = TRUE; + + /* + * Now that all send requests have completed, free up the + * send buffers used in the non-blocking operations + */ + if (msg_send_bufs) { + for (i = 0; i < num_send_requests; i++) { + if (msg_send_bufs[i]) + H5MM_free(msg_send_bufs[i]); + } + + msg_send_bufs = H5MM_xfree(msg_send_bufs); + } + } + } + } while (1); + + /* + * Ensure all receive requests have completed before moving on. + * For linked-chunk I/O, more overlap with computation could + * theoretically be achieved by returning the receive requests + * array and postponing this wait until during chunk updating + * when the data is really needed. However, multi-chunk I/O + * only updates a chunk at a time and the messages may not come + * in the order that chunks are processed. So, the safest way to + * support both I/O modes is to simply make sure all messages + * are available. + */ + if (MPI_SUCCESS != (mpi_code = MPI_Waitall((int)num_recv_requests, recv_requests, MPI_STATUSES_IGNORE))) + HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) + + /* Set the new number of locally-selected chunks */ + *chunk_list_num_entries = last_assigned_idx; + + /* Return chunk message buffers if any were received */ + *chunk_hash_table = chunk_table; + *chunk_msg_bufs = msg_recv_bufs; + *chunk_msg_bufs_len = (int)num_recv_requests; + +done: + if (ret_value < 0) { + /* If this rank failed, make sure to participate in collective barrier */ + if (!ibarrier_posted) { + if (MPI_SUCCESS != (mpi_code = MPI_Ibarrier(io_info->comm, &ibarrier))) + HMPI_GOTO_ERROR(FAIL, "MPI_Ibarrier failed", mpi_code) + } + + if (num_send_requests) { + for (i = 0; i < num_send_requests; i++) { + MPI_Cancel(&send_requests[i]); + } + } + + if (recv_requests) { + for (i = 0; i < num_recv_requests; i++) { + MPI_Cancel(&recv_requests[i]); + } + } + + if (msg_recv_bufs) { + for (i = 0; i < num_recv_requests; i++) { + H5MM_free(msg_recv_bufs[i]); + } + + H5MM_free(msg_recv_bufs); + } + + HASH_CLEAR(hh, chunk_table); + } + + if (recv_requests) + H5MM_free(recv_requests); + if (send_requests) + H5MM_free(send_requests); + + if (msg_send_bufs) { + for (i = 0; i < num_send_requests; i++) { + if (msg_send_bufs[i]) + H5MM_free(msg_send_bufs[i]); + } + + H5MM_free(msg_send_bufs); + } + + if (mem_iter) { + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release dataspace selection iterator") + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +#else + FUNC_ENTER_STATIC + HERROR( + H5E_DATASET, H5E_WRITEERROR, + "unable to send chunk modification data between MPI ranks - MPI version < 3 (MPI_Ibarrier missing)") + FUNC_LEAVE_NOAPI(FAIL) +#endif +} /* end H5D__mpio_share_chunk_modification_data() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_common_io + * + * Purpose: This routine performs the common part of collective I/O + * when reading or writing filtered chunks collectively. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_common_io(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_size) +{ + H5D_io_info_t coll_io_info; + H5D_storage_t ctg_store; + MPI_Datatype file_type = MPI_DATATYPE_NULL; + MPI_Datatype mem_type = MPI_DATATYPE_NULL; + hbool_t mem_type_is_derived = FALSE; + hbool_t file_type_is_derived = FALSE; + hsize_t mpi_buf_count; + haddr_t base_read_offset = HADDR_UNDEF; + size_t num_chunks; + size_t i; + char fake_buf; /* Used as a fake buffer for ranks with no chunks, thus a NULL buf pointer */ + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(type_info); + + /* Initialize temporary I/O info */ + coll_io_info = *io_info; + + /* + * Construct MPI derived datatype for collective I/O on chunks + */ + if (H5D__mpio_collective_filtered_io_type(chunk_list, chunk_list_num_entries, io_info->op_type, &mem_type, + &mem_type_is_derived, &file_type, &file_type_is_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "couldn't create MPI I/O type for chunk I/O") + + /* + * For reads, determine how many chunks are actually being read. + * Note that if this is a read during a write operation + * (read chunk -> unfilter -> modify -> write back), some + * chunks may not need to be read if they're being fully + * overwritten during a write operation. + */ + if (io_info->op_type == H5D_IO_OP_READ) { + for (i = 0, num_chunks = 0; i < chunk_list_num_entries; i++) { + HDassert(chunk_list[i].buf); + + if (chunk_list[i].need_read) { + if (!H5F_addr_defined(base_read_offset)) + base_read_offset = chunk_list[i].chunk_current.offset; + + num_chunks++; + } + } + } + else + num_chunks = chunk_list_num_entries; + + /* + * If this rank doesn't have a selection, it can + * skip I/O if independent I/O was requested at + * the low level, or if the MPI communicator size + * is 1. + * + * Otherwise, this rank has to participate in + * collective I/O, but probably has a NULL buf + * pointer, so override to a fake buffer since our + * write/read function expects one. + */ + if (num_chunks == 0) { + H5FD_mpio_collective_opt_t coll_opt_mode; + + /* Get the collective_opt property to check whether the application wants to do IO individually. */ + if (H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O collective_opt property") + + if ((mpi_size == 1) || (H5FD_MPIO_INDIVIDUAL_IO == coll_opt_mode)) { + HGOTO_DONE(SUCCEED) + } + else { + if (io_info->op_type == H5D_IO_OP_WRITE) + coll_io_info.u.wbuf = &fake_buf; + else + coll_io_info.u.rbuf = &fake_buf; + } + } + + /* + * Setup for I/O operation + */ + + mpi_buf_count = (num_chunks) ? 1 : 0; + + if (num_chunks) { + /* + * Setup the base storage address for this operation + * to be the first chunk's file address + */ + if (io_info->op_type == H5D_IO_OP_WRITE) + ctg_store.contig.dset_addr = chunk_list[0].chunk_new.offset; + else + ctg_store.contig.dset_addr = base_read_offset; + } + else + ctg_store.contig.dset_addr = 0; + + ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size; + coll_io_info.store = &ctg_store; + + /* Perform I/O */ + if (H5D__final_collective_io(&coll_io_info, type_info, mpi_buf_count, file_type, mem_type) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish MPI I/O") + +done: + /* Free the MPI buf and file types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_common_io() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_read + * + * Purpose: This routine coordinates a collective read across all ranks + * of the chunks they have selected. Each rank will then go + * and + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_read(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, int mpi_size) +{ + H5D_fill_buf_info_t fb_info; + H5D_chunk_info_t * chunk_info = NULL; + H5D_io_info_t coll_io_info; + H5Z_EDC_t err_detect; /* Error detection info */ + H5Z_cb_t filter_cb; /* I/O filter callback function */ + hsize_t file_chunk_size = 0; + hsize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */ + hbool_t should_fill = FALSE; + hbool_t fb_info_init = FALSE; + hbool_t index_empty = FALSE; + size_t i; + H5S_t * fill_space = NULL; + void * base_read_buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(type_info); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Filtered collective chunk read"); +#else + (void)mpi_rank; +#endif + + /* Initialize temporary I/O info */ + coll_io_info = *io_info; + coll_io_info.u.rbuf = NULL; + + if (chunk_list_num_entries) { + /* Retrieve filter settings from API context */ + if (H5CX_get_err_detect(&err_detect) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get error detection info") + if (H5CX_get_filter_cb(&filter_cb) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get I/O filter callback function") + + /* Set size of full chunks in dataset */ + file_chunk_size = io_info->dset->shared->layout.u.chunk.size; + + /* Determine if fill values should be "read" for unallocated chunks */ + should_fill = (io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_ALLOC) || + ((io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_IFSET) && + io_info->dset->shared->dcpl_cache.fill.fill_defined); + } + + /* + * Allocate memory buffers for all chunks being read. Chunk data buffers are of + * the largest size between the chunk's current filtered size and the chunk's true + * size, as calculated by the number of elements in the chunk's file space extent + * multiplied by the datatype size. This tries to ensure that: + * + * * If we're reading the chunk and the filter normally reduces the chunk size, + * the unfiltering operation won't need to grow the buffer. + * * If we're reading the chunk and the filter normally grows the chunk size, + * we make sure to read into a buffer of size equal to the filtered chunk's + * size; reading into a (smaller) buffer of size equal to the unfiltered + * chunk size would of course be bad. + */ + for (i = 0; i < chunk_list_num_entries; i++) { + HDassert(chunk_list[i].need_read); + + chunk_list[i].chunk_buf_size = MAX(chunk_list[i].chunk_current.length, file_chunk_size); + + if (NULL == (chunk_list[i].buf = H5MM_malloc(chunk_list[i].chunk_buf_size))) { + /* Push an error, but participate in collective read */ + HDONE_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer") + break; + } + + /* + * Check if chunk is currently allocated. If not, don't try to + * read it from the file. Instead, just fill the chunk buffer + * with the fill value if necessary. + */ + if (H5F_addr_defined(chunk_list[i].chunk_current.offset)) { + /* Set first read buffer */ + if (!base_read_buf) + base_read_buf = chunk_list[i].buf; + + /* Set chunk's new length for eventual filter pipeline calls */ + if (chunk_list[i].skip_filter_pline) + chunk_list[i].chunk_new.length = file_chunk_size; + else + chunk_list[i].chunk_new.length = chunk_list[i].chunk_current.length; + } + else { + chunk_list[i].need_read = FALSE; + + /* Set chunk's new length for eventual filter pipeline calls */ + chunk_list[i].chunk_new.length = file_chunk_size; + + if (should_fill) { + /* Initialize fill value buffer if not already initialized */ + if (!fb_info_init) { + hsize_t chunk_dims[H5S_MAX_RANK]; + + HDassert(io_info->dset->shared->ndims == io_info->dset->shared->layout.u.chunk.ndims - 1); + for (size_t j = 0; j < io_info->dset->shared->layout.u.chunk.ndims - 1; j++) + chunk_dims[j] = (hsize_t)io_info->dset->shared->layout.u.chunk.dim[j]; + + /* Get a dataspace for filling chunk memory buffers */ + if (NULL == (fill_space = H5S_create_simple( + io_info->dset->shared->layout.u.chunk.ndims - 1, chunk_dims, NULL))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create chunk fill dataspace") + + /* Initialize fill value buffer */ + if (H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_mem_alloc, + (void *)&io_info->dset->shared->dcpl_cache.pline, + (H5MM_free_t)H5D__chunk_mem_free, + (void *)&io_info->dset->shared->dcpl_cache.pline, + &io_info->dset->shared->dcpl_cache.fill, io_info->dset->shared->type, + io_info->dset->shared->type_id, 0, file_chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill value buffer") + + fb_info_init = TRUE; + } + + /* Write fill value to memory buffer */ + HDassert(fb_info.fill_buf); + if (H5D__fill(fb_info.fill_buf, io_info->dset->shared->type, chunk_list[i].buf, + type_info->mem_type, fill_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't fill chunk buffer with fill value") + } + } + } + + /* + * If dataset is incrementally allocated and hasn't been written to + * yet, the chunk index should be empty. In this case, a collective + * read of chunks is essentially a no-op, so avoid it here. + */ + index_empty = FALSE; + if (io_info->dset->shared->dcpl_cache.fill.alloc_time == H5D_ALLOC_TIME_INCR) + if (H5D__chunk_index_empty(io_info->dset, &index_empty) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "couldn't determine if chunk index is empty") + + if (!index_empty) { + /* + * Override the read buffer to point to the address of + * the first chunk data buffer being read into + */ + if (base_read_buf) + coll_io_info.u.rbuf = base_read_buf; + + /* Perform collective chunk read */ + if (H5D__mpio_collective_filtered_chunk_common_io(chunk_list, chunk_list_num_entries, &coll_io_info, + type_info, mpi_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish collective filtered chunk read") + } + + /* + * Iterate through all the read chunks, unfiltering them and scattering their + * data out to the application's read buffer. + */ + for (i = 0; i < chunk_list_num_entries; i++) { + chunk_info = chunk_list[i].chunk_info; + + /* Unfilter the chunk, unless we didn't read it from the file */ + if (chunk_list[i].need_read && !chunk_list[i].skip_filter_pline) { + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, + &(chunk_list[i].index_info.filter_mask), err_detect, filter_cb, + (size_t *)&chunk_list[i].chunk_new.length, &chunk_list[i].chunk_buf_size, + &chunk_list[i].buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying") + } + + /* Scatter the chunk data to the read buffer */ + iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + + if (H5D_select_io_mem(io_info->u.rbuf, chunk_info->mspace, chunk_list[i].buf, chunk_info->fspace, + type_info->src_type_size, (size_t)iter_nelmts) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't copy chunk data to read buffer") + } + +done: + /* Free all resources used by entries in the chunk list */ + for (i = 0; i < chunk_list_num_entries; i++) { + if (chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + + /* Release the fill buffer info, if it's been initialized */ + if (fb_info_init && H5D__fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") + if (fill_space && (H5S_close(fill_space) < 0)) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close fill space") + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_read() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_update + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, all ranks must update their owned chunks + * with their own modification data and data from other ranks. + * This routine is responsible for coordinating that process. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_update(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + H5D_filtered_collective_io_info_t *chunk_hash_table, + unsigned char **chunk_msg_bufs, int chunk_msg_bufs_len, + const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + int mpi_rank, int mpi_size) +{ + H5D_fill_buf_info_t fb_info; + H5D_chunk_info_t * chunk_info = NULL; + H5S_sel_iter_t * sel_iter = NULL; /* Dataspace selection iterator for H5D__scatter_mem */ + H5D_io_info_t coll_io_info; + H5Z_EDC_t err_detect; /* Error detection info */ + H5Z_cb_t filter_cb; /* I/O filter callback function */ + hsize_t file_chunk_size = 0; + hsize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */ + hbool_t should_fill = FALSE; + hbool_t fb_info_init = FALSE; + hbool_t sel_iter_init = FALSE; + hbool_t index_empty = FALSE; + size_t i; + H5S_t * dataspace = NULL; + H5S_t * fill_space = NULL; + void * base_read_buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert((chunk_msg_bufs && chunk_hash_table) || 0 == chunk_msg_bufs_len); + HDassert(io_info); + HDassert(type_info); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Filtered collective chunk update"); +#endif + + if (chunk_list_num_entries) { + /* Retrieve filter settings from API context */ + if (H5CX_get_err_detect(&err_detect) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get error detection info") + if (H5CX_get_filter_cb(&filter_cb) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get I/O filter callback function") + + /* Set size of full chunks in dataset */ + file_chunk_size = io_info->dset->shared->layout.u.chunk.size; + + /* Determine if fill values should be written to chunks */ + should_fill = (io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_ALLOC) || + ((io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_IFSET) && + io_info->dset->shared->dcpl_cache.fill.fill_defined); + } + + /* + * Allocate memory buffers for all owned chunks. Chunk data buffers are of the + * largest size between the chunk's current filtered size and the chunk's true + * size, as calculated by the number of elements in the chunk's file space extent + * multiplied by the datatype size. This tries to ensure that: + * + * * If we're fully overwriting the chunk and the filter normally reduces the + * chunk size, we simply have the exact buffer size required to hold the + * unfiltered chunk data. + * * If we're fully overwriting the chunk and the filter normally grows the + * chunk size (e.g., fletcher32 filter), the final filtering operation + * (hopefully) won't need to grow the buffer. + * * If we're reading the chunk and the filter normally reduces the chunk size, + * the unfiltering operation won't need to grow the buffer. + * * If we're reading the chunk and the filter normally grows the chunk size, + * we make sure to read into a buffer of size equal to the filtered chunk's + * size; reading into a (smaller) buffer of size equal to the unfiltered + * chunk size would of course be bad. + */ + for (i = 0; i < chunk_list_num_entries; i++) { + HDassert(mpi_rank == chunk_list[i].new_owner); + + chunk_list[i].chunk_buf_size = MAX(chunk_list[i].chunk_current.length, file_chunk_size); + + /* + * If this chunk hasn't been allocated yet and we aren't writing + * out fill values to it, make sure to 0-fill its memory buffer + * so we don't use uninitialized memory. + */ + if (!H5F_addr_defined(chunk_list[i].chunk_current.offset) && !should_fill) + chunk_list[i].buf = H5MM_calloc(chunk_list[i].chunk_buf_size); + else + chunk_list[i].buf = H5MM_malloc(chunk_list[i].chunk_buf_size); + + if (NULL == chunk_list[i].buf) { + /* Push an error, but participate in collective read */ + HDONE_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer") + break; + } + + /* Set chunk's new length for eventual filter pipeline calls */ + if (chunk_list[i].need_read) { + /* + * Check if chunk is currently allocated. If not, don't try to + * read it from the file. Instead, just fill the chunk buffer + * with the fill value if fill values are to be written. + */ + if (H5F_addr_defined(chunk_list[i].chunk_current.offset)) { + /* Set first read buffer */ + if (!base_read_buf) + base_read_buf = chunk_list[i].buf; + + /* Set chunk's new length for eventual filter pipeline calls */ + if (chunk_list[i].skip_filter_pline) + chunk_list[i].chunk_new.length = file_chunk_size; + else + chunk_list[i].chunk_new.length = chunk_list[i].chunk_current.length; + } + else { + chunk_list[i].need_read = FALSE; + + /* Set chunk's new length for eventual filter pipeline calls */ + chunk_list[i].chunk_new.length = file_chunk_size; + + if (should_fill) { + /* Initialize fill value buffer if not already initialized */ + if (!fb_info_init) { + hsize_t chunk_dims[H5S_MAX_RANK]; + + HDassert(io_info->dset->shared->ndims == + io_info->dset->shared->layout.u.chunk.ndims - 1); + for (size_t j = 0; j < io_info->dset->shared->layout.u.chunk.ndims - 1; j++) + chunk_dims[j] = (hsize_t)io_info->dset->shared->layout.u.chunk.dim[j]; + + /* Get a dataspace for filling chunk memory buffers */ + if (NULL == (fill_space = H5S_create_simple( + io_info->dset->shared->layout.u.chunk.ndims - 1, chunk_dims, NULL))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to create chunk fill dataspace") + + /* Initialize fill value buffer */ + if (H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_mem_alloc, + (void *)&io_info->dset->shared->dcpl_cache.pline, + (H5MM_free_t)H5D__chunk_mem_free, + (void *)&io_info->dset->shared->dcpl_cache.pline, + &io_info->dset->shared->dcpl_cache.fill, + io_info->dset->shared->type, io_info->dset->shared->type_id, 0, + file_chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill value buffer") + + fb_info_init = TRUE; + } + + /* Write fill value to memory buffer */ + HDassert(fb_info.fill_buf); + if (H5D__fill(fb_info.fill_buf, io_info->dset->shared->type, chunk_list[i].buf, + type_info->mem_type, fill_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "couldn't fill chunk buffer with fill value") + } + } + } + else + chunk_list[i].chunk_new.length = file_chunk_size; + } + + /* + * If dataset is incrementally allocated and hasn't been written to + * yet, the chunk index should be empty. In this case, a collective + * read of chunks is essentially a no-op, so avoid it here. + */ + index_empty = FALSE; + if (io_info->dset->shared->dcpl_cache.fill.alloc_time == H5D_ALLOC_TIME_INCR) + if (H5D__chunk_index_empty(io_info->dset, &index_empty) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "couldn't determine if chunk index is empty") + + if (!index_empty) { + /* + * Setup for I/O operation + */ + + /* Initialize temporary I/O info */ + coll_io_info = *io_info; + coll_io_info.op_type = H5D_IO_OP_READ; + + /* Override the read buffer to point to the address of the first + * chunk data buffer being read into + */ + if (base_read_buf) + coll_io_info.u.rbuf = base_read_buf; + + /* Read all chunks that need to be read from the file */ + if (H5D__mpio_collective_filtered_chunk_common_io(chunk_list, chunk_list_num_entries, &coll_io_info, + type_info, mpi_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish collective filtered chunk read") + } + + /* + * Now that all owned chunks have been read, update the chunks + * with modification data from the owning rank and other ranks. + */ + + /* Process all chunks with data from the owning rank first */ + for (i = 0; i < chunk_list_num_entries; i++) { + HDassert(mpi_rank == chunk_list[i].new_owner); + + chunk_info = chunk_list[i].chunk_info; + + /* + * If this chunk wasn't being fully overwritten, we read it from + * the file, so we need to unfilter it + */ + if (chunk_list[i].need_read && !chunk_list[i].skip_filter_pline) { + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, + &(chunk_list[i].index_info.filter_mask), err_detect, filter_cb, + (size_t *)&chunk_list[i].chunk_new.length, &chunk_list[i].chunk_buf_size, + &chunk_list[i].buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying") + } + + iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + + if (H5D_select_io_mem(chunk_list[i].buf, chunk_info->fspace, io_info->u.wbuf, chunk_info->mspace, + type_info->dst_type_size, (size_t)iter_nelmts) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't copy chunk data to write buffer") + } + + /* Allocate iterator for memory selection */ + if (NULL == (sel_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + + /* Now process all received chunk message buffers */ + for (i = 0; i < (size_t)chunk_msg_bufs_len; i++) { + H5D_filtered_collective_io_info_t *chunk_entry = NULL; + const unsigned char * msg_ptr = chunk_msg_bufs[i]; + hsize_t chunk_idx; + + if (msg_ptr) { + /* Retrieve the chunk's index value */ + HDmemcpy(&chunk_idx, msg_ptr, sizeof(hsize_t)); + msg_ptr += sizeof(hsize_t); + + /* Find the chunk entry according to its chunk index */ + HASH_FIND(hh, chunk_hash_table, &chunk_idx, sizeof(hsize_t), chunk_entry); + HDassert(chunk_entry); + HDassert(mpi_rank == chunk_entry->new_owner); + + /* + * Only process the chunk if its data buffer is allocated. + * In the case of multi-chunk I/O, we're only working on + * a chunk at a time, so we need to skip over messages + * that aren't for the chunk we're currently working on. + */ + if (!chunk_entry->buf) + continue; + else { + /* Decode the chunk file dataspace from the message */ + if (NULL == (dataspace = H5S_decode(&msg_ptr))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, FAIL, "unable to decode dataspace") + + if (H5S_select_iter_init(sel_iter, dataspace, type_info->dst_type_size, + H5S_SEL_ITER_SHARE_WITH_DATASPACE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + sel_iter_init = TRUE; + + iter_nelmts = H5S_GET_SELECT_NPOINTS(dataspace); + + /* Update the chunk data with the received modification data */ + if (H5D__scatter_mem(msg_ptr, sel_iter, (size_t)iter_nelmts, chunk_entry->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't scatter to write buffer") + + if (H5S_SELECT_ITER_RELEASE(sel_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + sel_iter_init = FALSE; + + if (dataspace) { + if (H5S_close(dataspace) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") + dataspace = NULL; + } + + H5MM_free(chunk_msg_bufs[i]); + chunk_msg_bufs[i] = NULL; + } + } + } + + /* Finally, filter all the chunks */ + for (i = 0; i < chunk_list_num_entries; i++) { + if (!chunk_list[i].skip_filter_pline) { + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, 0, + &(chunk_list[i].index_info.filter_mask), err_detect, filter_cb, + (size_t *)&chunk_list[i].chunk_new.length, &chunk_list[i].chunk_buf_size, + &chunk_list[i].buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") + } + +#if H5_SIZEOF_SIZE_T > 4 + /* Check for the chunk expanding too much to encode in a 32-bit value */ + if (chunk_list[i].chunk_new.length > ((size_t)0xffffffff)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length") +#endif + } + +done: + if (sel_iter) { + if (sel_iter_init && H5S_SELECT_ITER_RELEASE(sel_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + sel_iter = H5FL_FREE(H5S_sel_iter_t, sel_iter); + } + if (dataspace && (H5S_close(dataspace) < 0)) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") + if (fill_space && (H5S_close(fill_space) < 0)) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close fill space") + + /* Release the fill buffer info, if it's been initialized */ + if (fb_info_init && H5D__fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") + + /* On failure, try to free all resources used by entries in the chunk list */ + if (ret_value < 0) { + for (i = 0; i < chunk_list_num_entries; i++) { + if (chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_update() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_reallocate + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, all ranks must eventually get together and + * perform a collective reallocation of space in the file for + * all chunks that were modified on all ranks. This routine is + * responsible for coordinating that process. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_reallocate(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, size_t *num_chunks_assigned_map, + H5D_io_info_t *io_info, H5D_chk_idx_info_t *idx_info, + int mpi_rank, int mpi_size) +{ + H5D_chunk_alloc_info_t *collective_list = NULL; + MPI_Datatype send_type; + MPI_Datatype recv_type; + hbool_t send_type_derived = FALSE; + hbool_t recv_type_derived = FALSE; + hbool_t need_sort = FALSE; + size_t collective_num_entries = 0; + size_t num_local_chunks_processed = 0; + size_t i; + void * gathered_array = NULL; + int * counts_disps_array = NULL; + int * counts_ptr = NULL; + int * displacements_ptr = NULL; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(idx_info); + HDassert(idx_info->storage->idx_type != H5D_CHUNK_IDX_NONE); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Reallocation of chunk file space"); +#endif + + /* + * Make sure it's safe to cast this rank's number + * of chunks to be sent into an int for MPI + */ + H5_CHECK_OVERFLOW(chunk_list_num_entries, size_t, int); + + /* Create derived datatypes for the chunk file space info needed */ + if (H5D__mpio_get_chunk_alloc_info_types(&recv_type, &recv_type_derived, &send_type, &send_type_derived) < + 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't create derived datatypes for chunk file space info") + + /* + * Gather the new chunk sizes to all ranks for a collective reallocation + * of the chunks in the file. + */ + if (num_chunks_assigned_map) { + /* + * If a mapping between rank value -> number of assigned chunks has + * been provided (usually during linked-chunk I/O), we can use this + * to optimize MPI overhead a bit since MPI ranks won't need to + * first inform each other about how many chunks they're contributing. + */ + if (NULL == (counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*counts_disps_array)))) { + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + else { + /* Set the receive counts from the assigned chunks map */ + counts_ptr = counts_disps_array; + + for (i = 0; i < (size_t)mpi_size; i++) + H5_CHECKED_ASSIGN(counts_ptr[i], int, num_chunks_assigned_map[i], size_t); + + /* Set the displacements into the receive buffer for the gather operation */ + displacements_ptr = &counts_disps_array[mpi_size]; + + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + counts_ptr[i - 1]; + } + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(chunk_list, (int)chunk_list_num_entries, send_type, counts_ptr, + displacements_ptr, recv_type, TRUE, 0, io_info->comm, mpi_rank, mpi_size, + &gathered_array, &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "can't gather chunk file space info to/from ranks") + } + else { + /* + * If no mapping between rank value -> number of assigned chunks has + * been provided (usually during multi-chunk I/O), all MPI ranks will + * need to first inform other ranks about how many chunks they're + * contributing before performing the actual gather operation. Use + * the 'simple' MPI_Allgatherv wrapper for this. + */ + if (H5_mpio_gatherv_alloc_simple(chunk_list, (int)chunk_list_num_entries, send_type, recv_type, TRUE, + 0, io_info->comm, mpi_rank, mpi_size, &gathered_array, + &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "can't gather chunk file space info to/from ranks") + } + + /* Collectively re-allocate the modified chunks (from each rank) in the file */ + collective_list = (H5D_chunk_alloc_info_t *)gathered_array; + for (i = 0, num_local_chunks_processed = 0; i < collective_num_entries; i++) { + H5D_chunk_alloc_info_t *coll_entry = &collective_list[i]; + hbool_t need_insert; + hbool_t update_local_chunk; + + if (H5D__chunk_file_alloc(idx_info, &coll_entry->chunk_current, &coll_entry->chunk_new, &need_insert, + NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") + + /* + * If we just re-allocated a chunk that is local to this + * rank, make sure to update the chunk entry in the local + * chunk list + */ + update_local_chunk = + (num_local_chunks_processed < chunk_list_num_entries) && + (coll_entry->chunk_idx == chunk_list[num_local_chunks_processed].index_info.chunk_idx); + + if (update_local_chunk) { + H5D_filtered_collective_io_info_t *local_chunk; + + local_chunk = &chunk_list[num_local_chunks_processed]; + + /* Sanity check that this chunk is actually local */ + HDassert(mpi_rank == local_chunk->orig_owner); + HDassert(mpi_rank == local_chunk->new_owner); + + local_chunk->chunk_new = coll_entry->chunk_new; + local_chunk->index_info.need_insert = need_insert; + + /* + * Since chunk reallocation can move chunks around, check if + * the local chunk list is still in ascending offset of order + * in the file + */ + if (num_local_chunks_processed) { + haddr_t curr_chunk_offset = local_chunk->chunk_new.offset; + haddr_t prev_chunk_offset = chunk_list[num_local_chunks_processed - 1].chunk_new.offset; + + HDassert(H5F_addr_defined(prev_chunk_offset) && H5F_addr_defined(curr_chunk_offset)); + if (curr_chunk_offset < prev_chunk_offset) + need_sort = TRUE; + } + + num_local_chunks_processed++; + } + } + + HDassert(chunk_list_num_entries == num_local_chunks_processed); + + /* + * Ensure this rank's local chunk list is sorted in + * ascending order of offset in the file + */ + if (need_sort) + HDqsort(chunk_list, chunk_list_num_entries, sizeof(H5D_filtered_collective_io_info_t), + H5D__cmp_filtered_collective_io_info_entry); + +done: + H5MM_free(gathered_array); + H5MM_free(counts_disps_array); + + if (send_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&send_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (recv_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&recv_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__mpio_collective_filtered_chunk_reallocate() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_reinsert + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, all ranks must eventually get together and + * perform a collective reinsertion into the dataset's chunk + * index of chunks that were modified. This routine is + * responsible for coordinating that process. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_reinsert(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, size_t *num_chunks_assigned_map, + H5D_io_info_t *io_info, H5D_chk_idx_info_t *idx_info, + int mpi_rank, int mpi_size) +{ + H5D_chunk_ud_t chunk_ud; + MPI_Datatype send_type; + MPI_Datatype recv_type; + hbool_t send_type_derived = FALSE; + hbool_t recv_type_derived = FALSE; + hsize_t scaled_coords[H5O_LAYOUT_NDIMS]; + size_t collective_num_entries = 0; + size_t i; + void * gathered_array = NULL; + int * counts_disps_array = NULL; + int * counts_ptr = NULL; + int * displacements_ptr = NULL; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(idx_info); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Reinsertion of modified chunks into chunk index"); +#endif - num_writers++; - } while (++i < shared_chunks_info_array_num_entries && - shared_chunks_info_array[i].chunk_states.chunk_current.offset == last_seen_addr); + /* Only re-insert chunks if index has an insert method */ + if (!idx_info->storage->ops->insert) + HGOTO_DONE(SUCCEED); - /* Set all of the chunk entries' "new_owner" fields */ - for (; set_begin_index < i; set_begin_index++) { - shared_chunks_info_array[set_begin_index].owners.new_owner = new_chunk_owner; - shared_chunks_info_array[set_begin_index].num_writers = num_writers; - } /* end for */ + /* + * Make sure it's safe to cast this rank's number + * of chunks to be sent into an int for MPI + */ + H5_CHECK_OVERFLOW(chunk_list_num_entries, size_t, int); - num_assigned_chunks_array[new_chunk_owner]++; - } /* end for */ + /* Create derived datatypes for the chunk re-insertion info needed */ + if (H5D__mpio_get_chunk_insert_info_types(&recv_type, &recv_type_derived, &send_type, + &send_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't create derived datatypes for chunk re-insertion info") - /* Sort the new list in order of previous owner so that each original owner of a chunk - * entry gets that entry back, with the possibly newly-modified "new_owner" field + /* + * Gather information to all ranks for a collective re-insertion + * of the modified chunks into the chunk index + */ + if (num_chunks_assigned_map) { + /* + * If a mapping between rank value -> number of assigned chunks has + * been provided (usually during linked-chunk I/O), we can use this + * to optimize MPI overhead a bit since MPI ranks won't need to + * first inform each other about how many chunks they're contributing. */ - if (shared_chunks_info_array_num_entries > 1) - HDqsort(shared_chunks_info_array, shared_chunks_info_array_num_entries, - sizeof(H5D_filtered_collective_io_info_t), - H5D__cmp_filtered_collective_io_info_entry_owner); - - send_displacements[0] = 0; - for (i = 1; i < (size_t)mpi_size; i++) - send_displacements[i] = send_displacements[i - 1] + send_counts[i - 1]; - } /* end if */ + if (NULL == (counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*counts_disps_array)))) { + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + else { + /* Set the receive counts from the assigned chunks map */ + counts_ptr = counts_disps_array; - /* Scatter the segments of the list back to each process */ - H5_CHECKED_ASSIGN(scatter_recvcount_int, int, - *local_chunk_array_num_entries * sizeof(H5D_filtered_collective_io_info_t), size_t); - if (MPI_SUCCESS != - (mpi_code = MPI_Scatterv(shared_chunks_info_array, send_counts, send_displacements, MPI_BYTE, - local_chunk_array, scatter_recvcount_int, MPI_BYTE, 0, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "unable to scatter shared chunks info buffer", mpi_code) + for (i = 0; i < (size_t)mpi_size; i++) + H5_CHECKED_ASSIGN(counts_ptr[i], int, num_chunks_assigned_map[i], size_t); - if (shared_chunks_info_array) { - H5MM_free(shared_chunks_info_array); - shared_chunks_info_array = NULL; - } /* end if */ + /* Set the displacements into the receive buffer for the gather operation */ + displacements_ptr = &counts_disps_array[mpi_size]; - /* Now that the chunks have been redistributed, each process must send its modification data - * to the new owners of any of the chunks it previously possessed. Accordingly, each process - * must also issue asynchronous receives for any messages it may receive for each of the - * chunks it is assigned, in order to avoid potential deadlocking issues. - */ - if (*local_chunk_array_num_entries) - if (NULL == (mod_data = (unsigned char **)H5MM_malloc(*local_chunk_array_num_entries * - sizeof(unsigned char *)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data buffer array") - - /* Perform all the sends on the chunks that this rank doesn't own */ - /* (Sends and recvs must be two separate loops, to avoid deadlock) */ - for (i = 0, last_assigned_idx = 0; i < *local_chunk_array_num_entries; i++) { - H5D_filtered_collective_io_info_t *chunk_entry = &local_chunk_array[i]; - - if (mpi_rank != chunk_entry->owners.new_owner) { - H5D_chunk_info_t *chunk_info = NULL; - unsigned char * mod_data_p = NULL; - hsize_t iter_nelmts; - size_t mod_data_size; + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + counts_ptr[i - 1]; + } + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(chunk_list, (int)chunk_list_num_entries, send_type, counts_ptr, + displacements_ptr, recv_type, TRUE, 0, io_info->comm, mpi_rank, mpi_size, + &gathered_array, &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, + "can't gather chunk index re-insertion info to/from ranks") + } + else { + /* + * If no mapping between rank value -> number of assigned chunks has + * been provided (usually during multi-chunk I/O), all MPI ranks will + * need to first inform other ranks about how many chunks they're + * contributing before performing the actual gather operation. Use + * the 'simple' MPI_Allgatherv wrapper for this. + */ + if (H5_mpio_gatherv_alloc_simple(chunk_list, (int)chunk_list_num_entries, send_type, recv_type, TRUE, + 0, io_info->comm, mpi_rank, mpi_size, &gathered_array, + &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, + "can't gather chunk index re-insertion info to/from ranks") + } + + /* Initialize static chunk udata fields from chunk index info */ + H5D_MPIO_INIT_CHUNK_UD_INFO(chunk_ud, idx_info); - /* Look up the chunk and get its file and memory dataspaces */ - if (NULL == (chunk_info = (H5D_chunk_info_t *)H5SL_search(fm->sel_chunks, &chunk_entry->index))) - HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list") + for (i = 0; i < collective_num_entries; i++) { + H5D_chunk_insert_info_t *coll_entry = &((H5D_chunk_insert_info_t *)gathered_array)[i]; - /* Determine size of serialized chunk file dataspace, plus the size of - * the data being written + /* + * We only need to reinsert this chunk if we had to actually + * allocate or reallocate space in the file for it + */ + if (!coll_entry->index_info.need_insert) + continue; + + chunk_ud.chunk_block = coll_entry->chunk_block; + chunk_ud.chunk_idx = coll_entry->index_info.chunk_idx; + chunk_ud.filter_mask = coll_entry->index_info.filter_mask; + chunk_ud.common.scaled = scaled_coords; + + /* Calculate scaled coordinates for the chunk */ + if (idx_info->layout->idx_type == H5D_CHUNK_IDX_EARRAY && idx_info->layout->u.earray.unlim_dim > 0) { + /* + * Extensible arrays where the unlimited dimension is not + * the slowest-changing dimension "swizzle" the coordinates + * to move the unlimited dimension value to offset 0. Therefore, + * we use the "swizzled" down chunks to calculate the "swizzled" + * scaled coordinates and then we undo the "swizzle" operation. + * + * TODO: In the future, this is something that should be handled + * by the particular chunk index rather than manually + * here. Likely, the chunk index ops should get a new + * callback that accepts a chunk index and provides the + * caller with the scaled coordinates for that chunk. */ - if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to get encoded dataspace size") + H5VM_array_calc_pre(chunk_ud.chunk_idx, io_info->dset->shared->ndims, + idx_info->layout->u.earray.swizzled_down_chunks, scaled_coords); - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + H5VM_unswizzle_coords(hsize_t, scaled_coords, idx_info->layout->u.earray.unlim_dim); + } + else { + H5VM_array_calc_pre(chunk_ud.chunk_idx, io_info->dset->shared->ndims, + io_info->dset->shared->layout.u.chunk.down_chunks, scaled_coords); + } - H5_CHECK_OVERFLOW(iter_nelmts, hsize_t, size_t); - mod_data_size += (size_t)iter_nelmts * type_info->src_type_size; + scaled_coords[io_info->dset->shared->ndims] = 0; - if (NULL == (mod_data[num_send_requests] = (unsigned char *)H5MM_malloc(mod_data_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "couldn't allocate chunk modification send buffer") +#ifndef NDEBUG + /* + * If a matching local chunk entry is found, the + * `chunk_info` structure (which contains the chunk's + * pre-computed scaled coordinates) will be valid + * for this rank. Compare those coordinates against + * the calculated coordinates above to make sure + * they match. + */ + for (size_t dbg_idx = 0; dbg_idx < chunk_list_num_entries; dbg_idx++) { + if (coll_entry->index_info.chunk_idx == chunk_list[dbg_idx].index_info.chunk_idx) { + hbool_t coords_match = !HDmemcmp(scaled_coords, chunk_list[dbg_idx].chunk_info->scaled, + io_info->dset->shared->ndims * sizeof(hsize_t)); - /* Serialize the chunk's file dataspace into the buffer */ - mod_data_p = mod_data[num_send_requests]; - if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to encode dataspace") + HDassert(coords_match && "Calculated scaled coordinates for chunk didn't match " + "chunk's actual scaled coordinates!"); + break; + } + } +#endif - /* Initialize iterator for memory selection */ - if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize memory selection information") - mem_iter_init = TRUE; + if ((idx_info->storage->ops->insert)(idx_info, &chunk_ud, io_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index") + } - /* Collect the modification data into the buffer */ - if (0 == H5D__gather_mem(io_info->u.wbuf, mem_iter, (size_t)iter_nelmts, mod_data_p)) - HGOTO_ERROR(H5E_IO, H5E_CANTGATHER, FAIL, "couldn't gather from write buffer") +done: + H5MM_free(gathered_array); + H5MM_free(counts_disps_array); - /* Send modification data to new owner */ - H5_CHECK_OVERFLOW(mod_data_size, size_t, int) - H5_CHECK_OVERFLOW(chunk_entry->index, hsize_t, int) - if (MPI_SUCCESS != - (mpi_code = MPI_Isend(mod_data[num_send_requests], (int)mod_data_size, MPI_BYTE, - chunk_entry->owners.new_owner, (int)chunk_entry->index, io_info->comm, - &send_requests[num_send_requests]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Isend failed", mpi_code) + if (send_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&send_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (recv_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&recv_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } - if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release memory selection iterator") - mem_iter_init = FALSE; +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif - num_send_requests++; - } /* end if */ - } /* end for */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_reinsert() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_get_chunk_redistribute_info_types + * + * Purpose: Constructs MPI derived datatypes for communicating the + * info from a H5D_filtered_collective_io_info_t structure + * that is necessary for redistributing shared chunks during a + * collective write of filtered chunks. + * + * The datatype returned through `contig_type` has an extent + * equal to the size of an H5D_chunk_redistribute_info_t + * structure and is suitable for communicating that structure + * type. + * + * The datatype returned through `resized_type` has an extent + * equal to the size of an H5D_filtered_collective_io_info_t + * structure. This makes it suitable for sending an array of + * those structures, while extracting out just the info + * necessary for the chunk redistribution operation during + * communication. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_get_chunk_redistribute_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived) +{ + MPI_Datatype struct_type = MPI_DATATYPE_NULL; + hbool_t struct_type_derived = FALSE; + MPI_Datatype chunk_block_type = MPI_DATATYPE_NULL; + hbool_t chunk_block_type_derived = FALSE; + MPI_Datatype types[5]; + MPI_Aint displacements[5]; + int block_lengths[5]; + int field_count; + int mpi_code; + herr_t ret_value = SUCCEED; - /* Perform all the recvs on the chunks this rank owns */ - for (i = 0, last_assigned_idx = 0; i < *local_chunk_array_num_entries; i++) { - H5D_filtered_collective_io_info_t *chunk_entry = &local_chunk_array[i]; + FUNC_ENTER_STATIC - if (mpi_rank == chunk_entry->owners.new_owner) { - /* Allocate all necessary buffers for an asynchronous receive operation */ - if (chunk_entry->num_writers > 1) { - MPI_Message message; - MPI_Status status; - size_t j; + HDassert(contig_type); + HDassert(contig_type_derived); + HDassert(resized_type); + HDassert(resized_type_derived); - chunk_entry->async_info.num_receive_requests = (int)chunk_entry->num_writers - 1; - if (NULL == (chunk_entry->async_info.receive_requests_array = (MPI_Request *)H5MM_malloc( - (size_t)chunk_entry->async_info.num_receive_requests * sizeof(MPI_Request)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async requests array") + *contig_type_derived = FALSE; + *resized_type_derived = FALSE; - if (NULL == - (chunk_entry->async_info.receive_buffer_array = (unsigned char **)H5MM_malloc( - (size_t)chunk_entry->async_info.num_receive_requests * sizeof(unsigned char *)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async receive buffers") + /* Create struct type for the inner H5F_block_t structure */ + if (H5F_mpi_get_file_block_type(FALSE, &chunk_block_type, &chunk_block_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't create derived type for chunk file description") - for (j = 0; j < chunk_entry->num_writers - 1; j++) { - int count = 0; + field_count = 5; + HDassert(field_count == (sizeof(types) / sizeof(MPI_Datatype))); - /* Probe for a particular message from any process, removing that message - * from the receive queue in the process and allocating that much memory - * for the asynchronous receive - */ - if (MPI_SUCCESS != (mpi_code = MPI_Mprobe(MPI_ANY_SOURCE, (int)chunk_entry->index, - io_info->comm, &message, &status))) - HMPI_GOTO_ERROR(FAIL, "MPI_Mprobe failed", mpi_code) - - if (MPI_SUCCESS != (mpi_code = MPI_Get_count(&status, MPI_BYTE, &count))) - HMPI_GOTO_ERROR(FAIL, "MPI_Get_count failed", mpi_code) - - HDassert(count >= 0); - if (NULL == (chunk_entry->async_info.receive_buffer_array[j] = - (unsigned char *)H5MM_malloc((size_t)count * sizeof(char *)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "unable to allocate modification data receive buffer") - - if (MPI_SUCCESS != (mpi_code = MPI_Imrecv( - chunk_entry->async_info.receive_buffer_array[j], count, MPI_BYTE, - &message, &chunk_entry->async_info.receive_requests_array[j]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Imrecv failed", mpi_code) - } /* end for */ - } /* end if */ - - local_chunk_array[last_assigned_idx++] = local_chunk_array[i]; - } /* end else */ - } /* end for */ + /* + * Create structure type to pack chunk H5F_block_t structure + * next to chunk_idx, orig_owner, new_owner and num_writers + * fields + */ + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + block_lengths[3] = 1; + block_lengths[4] = 1; + displacements[0] = offsetof(H5D_chunk_redistribute_info_t, chunk_block); + displacements[1] = offsetof(H5D_chunk_redistribute_info_t, chunk_idx); + displacements[2] = offsetof(H5D_chunk_redistribute_info_t, orig_owner); + displacements[3] = offsetof(H5D_chunk_redistribute_info_t, new_owner); + displacements[4] = offsetof(H5D_chunk_redistribute_info_t, num_writers); + types[0] = chunk_block_type; + types[1] = HSIZE_AS_MPI_TYPE; + types[2] = MPI_INT; + types[3] = MPI_INT; + types[4] = MPI_INT; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + *contig_type_derived = TRUE; - *local_chunk_array_num_entries = last_assigned_idx; + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + + /* Create struct type to extract the chunk_current, chunk_idx, orig_owner, + * new_owner and num_writers fields from a H5D_filtered_collective_io_info_t + * structure + */ + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + block_lengths[3] = 1; + block_lengths[4] = 1; + displacements[0] = offsetof(H5D_filtered_collective_io_info_t, chunk_current); + displacements[1] = offsetof(H5D_filtered_collective_io_info_t, index_info.chunk_idx); + displacements[2] = offsetof(H5D_filtered_collective_io_info_t, orig_owner); + displacements[3] = offsetof(H5D_filtered_collective_io_info_t, new_owner); + displacements[4] = offsetof(H5D_filtered_collective_io_info_t, num_writers); + types[0] = chunk_block_type; + types[1] = HSIZE_AS_MPI_TYPE; + types[2] = MPI_INT; + types[3] = MPI_INT; + types[4] = MPI_INT; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; - /* Wait for all async send requests to complete before returning */ - if (num_send_requests) { - if (NULL == (send_statuses = (MPI_Status *)H5MM_malloc(num_send_requests * sizeof(MPI_Status)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send statuses buffer") + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized( + struct_type, 0, sizeof(H5D_filtered_collective_io_info_t), resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *resized_type_derived = TRUE; - H5_CHECK_OVERFLOW(num_send_requests, size_t, int); - if (MPI_SUCCESS != (mpi_code = MPI_Waitall((int)num_send_requests, send_requests, send_statuses))) - HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) - } /* end if */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) done: - /* Now that all async send requests have completed, free up the send - * buffers used in the async operations - */ - for (i = 0; i < num_send_requests; i++) { - if (mod_data[i]) - H5MM_free(mod_data[i]); - } /* end for */ + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (chunk_block_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&chunk_block_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } - if (send_requests) - H5MM_free(send_requests); - if (send_statuses) - H5MM_free(send_statuses); - if (send_counts) - H5MM_free(send_counts); - if (send_displacements) - H5MM_free(send_displacements); - if (mod_data) - H5MM_free(mod_data); - if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - if (mem_iter) - H5MM_free(mem_iter); - if (num_assigned_chunks_array) - H5MM_free(num_assigned_chunks_array); - if (shared_chunks_info_array) - H5MM_free(shared_chunks_info_array); + if (ret_value < 0) { + if (*resized_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(resized_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *resized_type_derived = FALSE; + } + if (*contig_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(contig_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *contig_type_derived = FALSE; + } + } FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__chunk_redistribute_shared_chunks() */ -#endif +} /* end H5D__mpio_get_chunk_redistribute_info_types() */ /*------------------------------------------------------------------------- - * Function: H5D__mpio_filtered_collective_write_type + * Function: H5D__mpio_get_chunk_alloc_info_types * - * Purpose: Constructs a MPI derived datatype for both the memory and - * the file for a collective write of filtered chunks. The - * datatype contains the offsets in the file and the locations - * of the filtered chunk data buffers. + * Purpose: Constructs MPI derived datatypes for communicating the info + * from a H5D_filtered_collective_io_info_t structure that is + * necessary for re-allocating file space during a collective + * write of filtered chunks. * - * Return: Non-negative on success/Negative on failure + * The datatype returned through `contig_type` has an extent + * equal to the size of an H5D_chunk_alloc_info_t structure + * and is suitable for communicating that structure type. + * + * The datatype returned through `resized_type` has an extent + * equal to the size of an H5D_filtered_collective_io_info_t + * structure. This makes it suitable for sending an array of + * those structures, while extracting out just the info + * necessary for the chunk file space reallocation operation + * during communication. * - * Programmer: Jordan Henderson - * Tuesday, November 22, 2016 + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ static herr_t -H5D__mpio_filtered_collective_write_type(H5D_filtered_collective_io_info_t *chunk_list, size_t num_entries, - MPI_Datatype *new_mem_type, hbool_t *mem_type_derived, - MPI_Datatype *new_file_type, hbool_t *file_type_derived) +H5D__mpio_get_chunk_alloc_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived) { - MPI_Aint *write_buf_array = NULL; /* Relative displacements of filtered chunk data buffers */ - MPI_Aint *file_offset_array = NULL; /* Chunk offsets in the file */ - int * length_array = NULL; /* Filtered Chunk lengths */ - herr_t ret_value = SUCCEED; + MPI_Datatype struct_type = MPI_DATATYPE_NULL; + hbool_t struct_type_derived = FALSE; + MPI_Datatype chunk_block_type = MPI_DATATYPE_NULL; + hbool_t chunk_block_type_derived = FALSE; + MPI_Datatype types[3]; + MPI_Aint displacements[3]; + int block_lengths[3]; + int field_count; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC - HDassert(chunk_list); - HDassert(new_mem_type); - HDassert(mem_type_derived); - HDassert(new_file_type); - HDassert(file_type_derived); + HDassert(contig_type); + HDassert(contig_type_derived); + HDassert(resized_type); + HDassert(resized_type_derived); - if (num_entries > 0) { - size_t i; - int mpi_code; - void * base_buf; - - H5_CHECK_OVERFLOW(num_entries, size_t, int); - - /* Allocate arrays */ - if (NULL == (length_array = (int *)H5MM_malloc((size_t)num_entries * sizeof(int)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for filtered collective write length array") - if (NULL == (write_buf_array = (MPI_Aint *)H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for filtered collective write buf length array") - if (NULL == (file_offset_array = (MPI_Aint *)H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for collective write offset array") - - /* Ensure the list is sorted in ascending order of offset in the file */ - HDqsort(chunk_list, num_entries, sizeof(H5D_filtered_collective_io_info_t), - H5D__cmp_filtered_collective_io_info_entry); + *contig_type_derived = FALSE; + *resized_type_derived = FALSE; - base_buf = chunk_list[0].buf; - for (i = 0; i < num_entries; i++) { - /* Set up the offset in the file, the length of the chunk data, and the relative - * displacement of the chunk data write buffer - */ - file_offset_array[i] = (MPI_Aint)chunk_list[i].chunk_states.new_chunk.offset; - length_array[i] = (int)chunk_list[i].chunk_states.new_chunk.length; - write_buf_array[i] = (MPI_Aint)chunk_list[i].buf - (MPI_Aint)base_buf; - } /* end for */ + /* Create struct type for the inner H5F_block_t structure */ + if (H5F_mpi_get_file_block_type(FALSE, &chunk_block_type, &chunk_block_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't create derived type for chunk file description") - /* Create memory MPI type */ - if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_entries, length_array, - write_buf_array, MPI_BYTE, new_mem_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - *mem_type_derived = TRUE; - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_mem_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - - /* Create file MPI type */ - if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_entries, length_array, - file_offset_array, MPI_BYTE, new_file_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - *file_type_derived = TRUE; - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - } /* end if */ + field_count = 3; + HDassert(field_count == (sizeof(types) / sizeof(MPI_Datatype))); + + /* + * Create structure type to pack both chunk H5F_block_t structures + * next to chunk_idx field + */ + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + displacements[0] = offsetof(H5D_chunk_alloc_info_t, chunk_current); + displacements[1] = offsetof(H5D_chunk_alloc_info_t, chunk_new); + displacements[2] = offsetof(H5D_chunk_alloc_info_t, chunk_idx); + types[0] = chunk_block_type; + types[1] = chunk_block_type; + types[2] = HSIZE_AS_MPI_TYPE; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + *contig_type_derived = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + + /* + * Create struct type to extract the chunk_current, chunk_new and chunk_idx + * fields from a H5D_filtered_collective_io_info_t structure + */ + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + displacements[0] = offsetof(H5D_filtered_collective_io_info_t, chunk_current); + displacements[1] = offsetof(H5D_filtered_collective_io_info_t, chunk_new); + displacements[2] = offsetof(H5D_filtered_collective_io_info_t, index_info.chunk_idx); + types[0] = chunk_block_type; + types[1] = chunk_block_type; + types[2] = HSIZE_AS_MPI_TYPE; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized( + struct_type, 0, sizeof(H5D_filtered_collective_io_info_t), resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *resized_type_derived = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) done: - if (write_buf_array) - H5MM_free(write_buf_array); - if (file_offset_array) - H5MM_free(file_offset_array); - if (length_array) - H5MM_free(length_array); + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (chunk_block_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&chunk_block_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + + if (ret_value < 0) { + if (*resized_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(resized_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *resized_type_derived = FALSE; + } + if (*contig_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(contig_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *contig_type_derived = FALSE; + } + } FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__mpio_filtered_collective_write_type() */ +} /* end H5D__mpio_get_chunk_alloc_info_types() */ /*------------------------------------------------------------------------- - * Function: H5D__filtered_collective_chunk_entry_io - * - * Purpose: Given an entry for a filtered chunk, performs the necessary - * steps for updating the chunk data during a collective - * write, or for reading the chunk from file during a - * collective read. + * Function: H5D__mpio_get_chunk_insert_info_types + * + * Purpose: Constructs MPI derived datatypes for communicating the + * information necessary when reinserting chunks into a + * dataset's chunk index. This includes the chunk's new offset + * and size (H5F_block_t) and the inner `index_info` structure + * of a H5D_filtered_collective_io_info_t structure. + * + * The datatype returned through `contig_type` has an extent + * equal to the size of an H5D_chunk_insert_info_t structure + * and is suitable for communicating that structure type. + * + * The datatype returned through `resized_type` has an extent + * equal to the size of the encompassing + * H5D_filtered_collective_io_info_t structure. This makes it + * suitable for sending an array of + * H5D_filtered_collective_io_info_t structures, while + * extracting out just the information needed during + * communication. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Wednesday, January 18, 2017 - * *------------------------------------------------------------------------- */ static herr_t -H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry, - const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5D_chunk_map_t *fm) +H5D__mpio_get_chunk_insert_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived) { - H5D_chunk_info_t *chunk_info = NULL; - H5S_sel_iter_t * mem_iter = NULL; /* Memory iterator for H5D__scatter_mem/H5D__gather_mem */ - H5S_sel_iter_t * file_iter = NULL; - H5Z_EDC_t err_detect; /* Error detection info */ - H5Z_cb_t filter_cb; /* I/O filter callback function */ - unsigned filter_mask = 0; - hsize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */ - hssize_t extent_npoints; - hsize_t true_chunk_size; - hbool_t mem_iter_init = FALSE; - hbool_t file_iter_init = FALSE; - size_t buf_size; - size_t i; - H5S_t * dataspace = NULL; /* Other process' dataspace for the chunk */ - void * tmp_gath_buf = NULL; /* Temporary gather buffer to gather into from application buffer - before scattering out to the chunk data buffer (when writing data), - or vice versa (when reading data) */ - int mpi_code; - herr_t ret_value = SUCCEED; + MPI_Datatype struct_type = MPI_DATATYPE_NULL; + hbool_t struct_type_derived = FALSE; + MPI_Datatype chunk_block_type = MPI_DATATYPE_NULL; + hbool_t chunk_block_type_derived = FALSE; + MPI_Aint contig_type_extent; + MPI_Datatype types[4]; + MPI_Aint displacements[4]; + int block_lengths[4]; + int field_count; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC - HDassert(chunk_entry); - HDassert(io_info); - HDassert(type_info); - HDassert(fm); + HDassert(contig_type); + HDassert(contig_type_derived); + HDassert(resized_type); + HDassert(resized_type_derived); - /* Retrieve filter settings from API context */ - if (H5CX_get_err_detect(&err_detect) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get error detection info") - if (H5CX_get_filter_cb(&filter_cb) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get I/O filter callback function") + *contig_type_derived = FALSE; + *resized_type_derived = FALSE; - /* Look up the chunk and get its file and memory dataspaces */ - if (NULL == (chunk_info = (H5D_chunk_info_t *)H5SL_search(fm->sel_chunks, &chunk_entry->index))) - HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list") + /* Create struct type for an H5F_block_t structure */ + if (H5F_mpi_get_file_block_type(FALSE, &chunk_block_type, &chunk_block_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't create derived type for chunk file description") - if ((extent_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") - true_chunk_size = (hsize_t)extent_npoints * type_info->src_type_size; + field_count = 4; + HDassert(field_count == (sizeof(types) / sizeof(MPI_Datatype))); - /* If the size of the filtered chunk is larger than the number of points in the - * chunk file space extent times the datatype size, allocate enough space to hold the - * whole filtered chunk. Otherwise, allocate a buffer equal to the size of the - * chunk so that the unfiltering operation doesn't have to grow the buffer. + /* + * Create struct type to pack information into memory as follows: + * + * Chunk's new Offset/Size (H5F_block_t) -> + * Chunk Index Info (H5D_chunk_index_info_t) */ - buf_size = MAX(chunk_entry->chunk_states.chunk_current.length, true_chunk_size); + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + block_lengths[3] = 1; + displacements[0] = offsetof(H5D_chunk_insert_info_t, chunk_block); + displacements[1] = offsetof(H5D_chunk_insert_info_t, index_info.chunk_idx); + displacements[2] = offsetof(H5D_chunk_insert_info_t, index_info.filter_mask); + displacements[3] = offsetof(H5D_chunk_insert_info_t, index_info.need_insert); + types[0] = chunk_block_type; + types[1] = HSIZE_AS_MPI_TYPE; + types[2] = MPI_UNSIGNED; + types[3] = MPI_C_BOOL; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; + + contig_type_extent = (MPI_Aint)(sizeof(H5F_block_t) + sizeof(H5D_chunk_index_info_t)); - if (NULL == (chunk_entry->buf = H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer") + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized(struct_type, 0, contig_type_extent, contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *contig_type_derived = TRUE; - /* If this is not a full chunk overwrite or this is a read operation, the chunk must be - * read from the file and unfiltered. + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + + struct_type_derived = FALSE; + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + /* + * Create struct type to correctly extract all needed + * information from a H5D_filtered_collective_io_info_t + * structure. */ - if (!chunk_entry->full_overwrite || io_info->op_type == H5D_IO_OP_READ) { - H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request */ + displacements[0] = offsetof(H5D_filtered_collective_io_info_t, chunk_new); + displacements[1] = offsetof(H5D_filtered_collective_io_info_t, index_info.chunk_idx); + displacements[2] = offsetof(H5D_filtered_collective_io_info_t, index_info.filter_mask); + displacements[3] = offsetof(H5D_filtered_collective_io_info_t, index_info.need_insert); + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; - chunk_entry->chunk_states.new_chunk.length = chunk_entry->chunk_states.chunk_current.length; + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized( + struct_type, 0, sizeof(H5D_filtered_collective_io_info_t), resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *resized_type_derived = TRUE; - /* Currently, these chunk reads are done independently and will likely - * cause issues with collective metadata reads enabled. In the future, - * this should be refactored to use collective chunk reads - JTH */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - /* Get the original state of parallel I/O transfer mode */ - if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") +done: + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (chunk_block_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&chunk_block_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } - /* Change the xfer_mode to independent for handling the I/O */ - if (H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode") + if (ret_value < 0) { + if (*resized_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(resized_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *resized_type_derived = FALSE; + } + if (*contig_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(contig_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *contig_type_derived = FALSE; + } + } - if (H5F_shared_block_read(io_info->f_sh, H5FD_MEM_DRAW, - chunk_entry->chunk_states.chunk_current.offset, - chunk_entry->chunk_states.new_chunk.length, chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read raw data chunk") + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_get_chunk_insert_info_types() */ - /* Return to the original I/O transfer mode setting */ - if (H5CX_set_io_xfer_mode(xfer_mode) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode") +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_io_type + * + * Purpose: Constructs a MPI derived datatype for both the memory and + * the file for a collective I/O operation on filtered chunks. + * The datatype contains the chunk offsets and lengths in the + * file and the locations of the chunk data buffers to read + * into/write from. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_io_type(H5D_filtered_collective_io_info_t *chunk_list, size_t num_entries, + H5D_io_op_type_t op_type, MPI_Datatype *new_mem_type, + hbool_t *mem_type_derived, MPI_Datatype *new_file_type, + hbool_t *file_type_derived) +{ + MPI_Aint *io_buf_array = NULL; /* Relative displacements of filtered chunk data buffers */ + MPI_Aint *file_offset_array = NULL; /* Chunk offsets in the file */ + int * length_array = NULL; /* Filtered Chunk lengths */ + int mpi_code; + herr_t ret_value = SUCCEED; - if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, &filter_mask, err_detect, - filter_cb, (size_t *)&chunk_entry->chunk_states.new_chunk.length, &buf_size, - &chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying") - } /* end if */ - else { - chunk_entry->chunk_states.new_chunk.length = true_chunk_size; - } /* end else */ + FUNC_ENTER_STATIC - /* Initialize iterator for memory selection */ - if (NULL == (mem_iter = (H5S_sel_iter_t *)H5MM_malloc(sizeof(H5S_sel_iter_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + HDassert(chunk_list || 0 == num_entries); + HDassert(new_mem_type); + HDassert(mem_type_derived); + HDassert(new_file_type); + HDassert(file_type_derived); - if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") - mem_iter_init = TRUE; + *mem_type_derived = FALSE; + *file_type_derived = FALSE; + *new_mem_type = MPI_BYTE; + *new_file_type = MPI_BYTE; - /* If this is a read operation, scatter the read chunk data to the user's buffer. - * - * If this is a write operation, update the chunk data buffer with the modifications - * from the current process, then apply any modifications from other processes. Finally, - * filter the newly-updated chunk. - */ - switch (io_info->op_type) { - case H5D_IO_OP_READ: - if (NULL == (file_iter = (H5S_sel_iter_t *)H5MM_malloc(sizeof(H5S_sel_iter_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file iterator") + if (num_entries > 0) { + H5F_block_t *chunk_block; + size_t last_valid_idx = 0; + size_t i; + int chunk_count; - if (H5S_select_iter_init(file_iter, chunk_info->fspace, type_info->src_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize memory selection information") - file_iter_init = TRUE; + /* + * Determine number of chunks for I/O operation and + * setup for derived datatype creation if I/O operation + * includes multiple chunks + */ + if (num_entries == 1) { + /* Set last valid index to 0 for contiguous datatype creation */ + last_valid_idx = 0; - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + if (op_type == H5D_IO_OP_WRITE) + chunk_count = 1; + else + chunk_count = chunk_list[0].need_read ? 1 : 0; + } + else { + MPI_Aint chunk_buf; + MPI_Aint base_buf; + haddr_t base_offset = HADDR_UNDEF; + + H5_CHECK_OVERFLOW(num_entries, size_t, int); + + /* Allocate arrays */ + if (NULL == (length_array = H5MM_malloc((size_t)num_entries * sizeof(int)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for filtered collective I/O length array") + if (NULL == (io_buf_array = H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for filtered collective I/O buf length array") + if (NULL == (file_offset_array = H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for filtered collective I/O offset array") + + /* + * If doing a write, we can set the base chunk offset + * and base chunk data buffer right away. + * + * If doing a read, some chunks may be skipped over + * for reading if they aren't yet allocated in the + * file. Therefore, we have to find the first chunk + * actually being read in order to set the base chunk + * offset and base chunk data buffer. + */ + if (op_type == H5D_IO_OP_WRITE) { +#if MPI_VERSION >= 3 + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(chunk_list[0].buf, &base_buf))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) +#else + base_buf = (MPI_Aint)chunk_list[0].buf; +#endif - if (NULL == (tmp_gath_buf = H5MM_malloc(iter_nelmts * type_info->src_type_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate temporary gather buffer") + base_offset = chunk_list[0].chunk_new.offset; + } - if (!H5D__gather_mem(chunk_entry->buf, file_iter, (size_t)iter_nelmts, tmp_gath_buf)) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't gather from chunk buffer") + for (i = 0, chunk_count = 0; i < num_entries; i++) { + if (op_type == H5D_IO_OP_READ) { + /* + * If this chunk isn't being read, don't add it + * to the MPI type we're building up for I/O + */ + if (!chunk_list[i].need_read) + continue; - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + /* + * If this chunk is being read, go ahead and + * set the base chunk offset and base chunk + * data buffer if we haven't already + */ + if (!H5F_addr_defined(base_offset)) { +#if MPI_VERSION >= 3 + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(chunk_list[i].buf, &base_buf))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) +#else + base_buf = (MPI_Aint)chunk_list[i].buf; +#endif - if (H5D__scatter_mem(tmp_gath_buf, mem_iter, (size_t)iter_nelmts, io_info->u.rbuf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to read buffer") + base_offset = chunk_list[i].chunk_current.offset; + } + } - break; + /* Set convenience pointer for current chunk block */ + chunk_block = + (op_type == H5D_IO_OP_READ) ? &chunk_list[i].chunk_current : &chunk_list[i].chunk_new; - case H5D_IO_OP_WRITE: - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + /* + * Set the current chunk entry's offset in the file, relative to + * the first chunk entry + */ + HDassert(H5F_addr_defined(chunk_block->offset)); + file_offset_array[chunk_count] = (MPI_Aint)(chunk_block->offset - base_offset); - if (NULL == (tmp_gath_buf = H5MM_malloc(iter_nelmts * type_info->src_type_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate temporary gather buffer") + /* + * Ensure the chunk list is sorted in ascending ordering of + * offset in the file + */ + if (chunk_count) + HDassert(file_offset_array[chunk_count] > file_offset_array[chunk_count - 1]); - /* Gather modification data from the application write buffer into a temporary buffer */ - if (0 == H5D__gather_mem(io_info->u.wbuf, mem_iter, (size_t)iter_nelmts, tmp_gath_buf)) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't gather from write buffer") + /* Set the current chunk entry's size for the I/O operation */ + H5_CHECK_OVERFLOW(chunk_block->length, hsize_t, int); + length_array[chunk_count] = (int)chunk_block->length; - if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - mem_iter_init = FALSE; + /* + * Set the displacement of the chunk entry's chunk data buffer, + * relative to the first entry's data buffer + */ +#if MPI_VERSION >= 3 && MPI_SUBVERSION >= 1 + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(chunk_list[i].buf, &chunk_buf))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) - /* Initialize iterator for file selection */ - if (H5S_select_iter_init(mem_iter, chunk_info->fspace, type_info->dst_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize file selection information") - mem_iter_init = TRUE; + io_buf_array[chunk_count] = MPI_Aint_diff(chunk_buf, base_buf); +#else + chunk_buf = (MPI_Aint)chunk_list[i].buf; + io_buf_array[chunk_count] = chunk_buf - base_buf; +#endif - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + /* + * Set last valid index in case only a single chunk will + * be involved in the I/O operation + */ + last_valid_idx = i; - /* Scatter the owner's modification data into the chunk data buffer according to - * the file space. - */ - if (H5D__scatter_mem(tmp_gath_buf, mem_iter, (size_t)iter_nelmts, chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to chunk data buffer") + chunk_count++; + } /* end for */ + } - if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - mem_iter_init = FALSE; + /* + * Create derived datatypes for the chunk list if this + * rank has any chunks to work on + */ + if (chunk_count > 0) { + if (chunk_count == 1) { + int chunk_len; - if (MPI_SUCCESS != - (mpi_code = MPI_Waitall(chunk_entry->async_info.num_receive_requests, - chunk_entry->async_info.receive_requests_array, MPI_STATUSES_IGNORE))) - HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) + /* Single chunk - use a contiguous type for both memory and file */ - /* For each asynchronous receive call previously posted, receive the chunk modification - * buffer from another rank and update the chunk data - */ - for (i = 0; i < (size_t)chunk_entry->async_info.num_receive_requests; i++) { - const unsigned char *mod_data_p; + /* Ensure that we can cast chunk size to an int for MPI */ + chunk_block = (op_type == H5D_IO_OP_READ) ? &chunk_list[last_valid_idx].chunk_current + : &chunk_list[last_valid_idx].chunk_new; + H5_CHECKED_ASSIGN(chunk_len, int, chunk_block->length, hsize_t); - /* Decode the process' chunk file dataspace */ - mod_data_p = chunk_entry->async_info.receive_buffer_array[i]; - if (NULL == (dataspace = H5S_decode(&mod_data_p))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, FAIL, "unable to decode dataspace") + if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(chunk_len, MPI_BYTE, new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + *new_mem_type = *new_file_type; - if (H5S_select_iter_init(mem_iter, dataspace, type_info->dst_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize memory selection information") - mem_iter_init = TRUE; + /* + * Since we use the same datatype for both memory and file, only + * mark the file type as derived so the caller doesn't try to + * free the same type twice + */ + *mem_type_derived = FALSE; + *file_type_derived = TRUE; - iter_nelmts = H5S_GET_SELECT_NPOINTS(dataspace); + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } + else { + HDassert(file_offset_array); + HDassert(length_array); + HDassert(io_buf_array); - /* Update the chunk data with the received modification data */ - if (H5D__scatter_mem(mod_data_p, mem_iter, (size_t)iter_nelmts, chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't scatter to write buffer") + /* Multiple chunks - use an hindexed type for both memory and file */ - if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - mem_iter_init = FALSE; - if (dataspace) { - if (H5S_close(dataspace) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") - dataspace = NULL; - } - H5MM_free(chunk_entry->async_info.receive_buffer_array[i]); - } /* end for */ + /* Create memory MPI type */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed( + chunk_count, length_array, io_buf_array, MPI_BYTE, new_mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + *mem_type_derived = TRUE; - /* Filter the chunk */ - if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, 0, &filter_mask, err_detect, filter_cb, - (size_t *)&chunk_entry->chunk_states.new_chunk.length, &buf_size, - &chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) -#if H5_SIZEOF_SIZE_T > 4 - /* Check for the chunk expanding too much to encode in a 32-bit value */ - if (chunk_entry->chunk_states.new_chunk.length > ((size_t)0xffffffff)) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length") -#endif - break; + /* Create file MPI type */ + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_hindexed(chunk_count, length_array, file_offset_array, + MPI_BYTE, new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + *file_type_derived = TRUE; - default: - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid I/O operation") - } /* end switch */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } + } + } /* end if */ done: - if (chunk_entry->async_info.receive_buffer_array) - H5MM_free(chunk_entry->async_info.receive_buffer_array); - if (chunk_entry->async_info.receive_requests_array) - H5MM_free(chunk_entry->async_info.receive_requests_array); - if (tmp_gath_buf) - H5MM_free(tmp_gath_buf); - if (file_iter_init && H5S_SELECT_ITER_RELEASE(file_iter) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - if (file_iter) - H5MM_free(file_iter); - if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - if (mem_iter) - H5MM_free(mem_iter); - if (dataspace) - if (H5S_close(dataspace) < 0) - HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") + if (file_offset_array) + H5MM_free(file_offset_array); + if (io_buf_array) + H5MM_free(io_buf_array); + if (length_array) + H5MM_free(length_array); + + if (ret_value < 0) { + if (*file_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(new_file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *file_type_derived = FALSE; + } + if (*mem_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(new_mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *mem_type_derived = FALSE; + } + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_io_type() */ + +#ifdef H5Dmpio_DEBUG + +static herr_t +H5D__mpio_dump_collective_filtered_chunk_list(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, int mpi_rank) +{ + H5D_filtered_collective_io_info_t *chunk_entry; + size_t i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC_NOERR + + H5D_MPIO_DEBUG(mpi_rank, "CHUNK LIST: ["); + for (i = 0; i < chunk_list_num_entries; i++) { + unsigned chunk_rank; + + chunk_entry = &chunk_list[i]; + + HDassert(chunk_entry->chunk_info); + chunk_rank = (unsigned)H5S_GET_EXTENT_NDIMS(chunk_entry->chunk_info->fspace); + + H5D_MPIO_DEBUG(mpi_rank, " {"); + H5D_MPIO_DEBUG_VA(mpi_rank, " - Entry %zu -", i); + + H5D_MPIO_DEBUG(mpi_rank, " - Chunk Fspace Info -"); + H5D_MPIO_DEBUG_VA(mpi_rank, + " Chunk Current Info: { Offset: %" PRIuHADDR ", Length: %" PRIuHADDR " }", + chunk_entry->chunk_current.offset, chunk_entry->chunk_current.length); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk New Info: { Offset: %" PRIuHADDR ", Length: %" PRIuHADDR " }", + chunk_entry->chunk_new.offset, chunk_entry->chunk_new.length); + + H5D_MPIO_DEBUG(mpi_rank, " - Chunk Insert Info -"); + H5D_MPIO_DEBUG_VA(mpi_rank, + " Chunk Scaled Coords (4-d): { %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + ", %" PRIuHSIZE " }", + chunk_rank < 1 ? 0 : chunk_entry->chunk_info->scaled[0], + chunk_rank < 2 ? 0 : chunk_entry->chunk_info->scaled[1], + chunk_rank < 3 ? 0 : chunk_entry->chunk_info->scaled[2], + chunk_rank < 4 ? 0 : chunk_entry->chunk_info->scaled[3]); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Index: %" PRIuHSIZE, chunk_entry->index_info.chunk_idx); + H5D_MPIO_DEBUG_VA(mpi_rank, " Filter Mask: %u", chunk_entry->index_info.filter_mask); + H5D_MPIO_DEBUG_VA(mpi_rank, " Need Insert: %s", + chunk_entry->index_info.need_insert ? "YES" : "NO"); + + H5D_MPIO_DEBUG(mpi_rank, " - Other Info -"); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Info Ptr: %p", (void *)chunk_entry->chunk_info); + H5D_MPIO_DEBUG_VA(mpi_rank, " Need Read: %s", chunk_entry->need_read ? "YES" : "NO"); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk I/O Size: %zu", chunk_entry->io_size); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Buffer Size: %zu", chunk_entry->chunk_buf_size); + H5D_MPIO_DEBUG_VA(mpi_rank, " Original Owner: %d", chunk_entry->orig_owner); + H5D_MPIO_DEBUG_VA(mpi_rank, " New Owner: %d", chunk_entry->new_owner); + H5D_MPIO_DEBUG_VA(mpi_rank, " # of Writers: %d", chunk_entry->num_writers); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Data Buffer Ptr: %p", (void *)chunk_entry->buf); + + H5D_MPIO_DEBUG(mpi_rank, " }"); + } + H5D_MPIO_DEBUG(mpi_rank, "]"); FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__filtered_collective_chunk_entry_io() */ +} /* end H5D__mpio_dump_collective_filtered_chunk_list() */ + +#endif + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 92ea120..5114d6a 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -123,13 +123,12 @@ typedef hbool_t (*H5D_layout_is_space_alloc_func_t)(const H5O_storage_t *storage typedef hbool_t (*H5D_layout_is_data_cached_func_t)(const H5D_shared_t *shared_dset); typedef herr_t (*H5D_layout_io_init_func_t)(const struct H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, - struct H5D_chunk_map_t *cm); + H5S_t *file_space, H5S_t *mem_space, struct H5D_chunk_map_t *cm); typedef herr_t (*H5D_layout_read_func_t)(struct H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, struct H5D_chunk_map_t *fm); typedef herr_t (*H5D_layout_write_func_t)(struct H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, struct H5D_chunk_map_t *fm); typedef ssize_t (*H5D_layout_readvv_func_t)(const struct H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], @@ -170,10 +169,10 @@ typedef struct H5D_layout_ops_t { /* Function pointers for either multiple or single block I/O access */ typedef herr_t (*H5D_io_single_read_func_t)(const struct H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space); + H5S_t *file_space, H5S_t *mem_space); typedef herr_t (*H5D_io_single_write_func_t)(const struct H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space); + H5S_t *file_space, H5S_t *mem_space); /* Typedef for raw data I/O framework info */ typedef struct H5D_io_ops_t { @@ -347,10 +346,10 @@ typedef struct H5D_chunk_map_t { H5O_layout_t *layout; /* Dataset layout information*/ hsize_t nelmts; /* Number of elements selected in file & memory dataspaces */ - const H5S_t *file_space; /* Pointer to the file dataspace */ - unsigned f_ndims; /* Number of dimensions for file dataspace */ + H5S_t * file_space; /* Pointer to the file dataspace */ + unsigned f_ndims; /* Number of dimensions for file dataspace */ - const H5S_t * mem_space; /* Pointer to the memory dataspace */ + H5S_t * mem_space; /* Pointer to the memory dataspace */ H5S_t * mchunk_tmpl; /* Dataspace template for new memory chunks */ H5S_sel_iter_t mem_iter; /* Iterator for elements in memory selection */ unsigned m_ndims; /* Number of dimensions for memory dataspace */ @@ -435,7 +434,7 @@ typedef struct H5D_rdcdc_t { /* * A dataset is made of two layers, an H5D_t struct that is unique to - * each instance of an opened datset, and a shared struct that is only + * each instance of an opened dataset, and a shared struct that is only * created once for a given dataset. Thus, if a dataset is opened twice, * there will be two IDs and two H5D_t structs, both sharing one H5D_shared_t. */ @@ -561,6 +560,7 @@ H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t hbool_t full_overwrite, hsize_t old_dim[]); H5_DLL herr_t H5D__get_storage_size(const H5D_t *dset, hsize_t *storage_size); H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset, hsize_t *storage_size); +H5_DLL herr_t H5D__chunk_index_empty(const H5D_t *dset, hbool_t *empty); H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks); H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t chk_idx, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size); @@ -582,25 +582,29 @@ H5_DLL herr_t H5D__refresh(H5D_t *dataset, hid_t dset_id); H5_DLL herr_t H5D__format_convert(H5D_t *dataset); /* Internal I/O routines */ -H5_DLL herr_t H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, +H5_DLL herr_t H5D__read(H5D_t *dataset, hid_t mem_type_id, H5S_t *mem_space, H5S_t *file_space, void *buf /*out*/); -H5_DLL herr_t H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, +H5_DLL herr_t H5D__write(H5D_t *dataset, hid_t mem_type_id, H5S_t *mem_space, H5S_t *file_space, const void *buf); /* Functions that perform direct serial I/O operations */ H5_DLL herr_t H5D__select_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space); + H5S_t *file_space, H5S_t *mem_space); H5_DLL herr_t H5D__select_write(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); + +/* Functions that perform direct copying between memory buffers */ +H5_DLL herr_t H5D_select_io_mem(void *dst_buf, const H5S_t *dst_space, const void *src_buf, + const H5S_t *src_space, size_t elmt_size, size_t nelmts); /* Functions that perform scatter-gather serial I/O operations */ H5_DLL herr_t H5D__scatter_mem(const void *_tscat_buf, H5S_sel_iter_t *iter, size_t nelmts, void *_buf); H5_DLL size_t H5D__gather_mem(const void *_buf, H5S_sel_iter_t *iter, size_t nelmts, void *_tgath_buf /*out*/); H5_DLL herr_t H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); H5_DLL herr_t H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); /* Functions that operate on dataset's layout information */ H5_DLL herr_t H5D__layout_set_io_ops(const H5D_t *dataset); @@ -618,9 +622,9 @@ H5_DLL hbool_t H5D__contig_is_space_alloc(const H5O_storage_t *storage); H5_DLL hbool_t H5D__contig_is_data_cached(const H5D_shared_t *shared_dset); H5_DLL herr_t H5D__contig_fill(const H5D_io_info_t *io_info); H5_DLL herr_t H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm); + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); H5_DLL herr_t H5D__contig_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm); + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); H5_DLL herr_t H5D__contig_copy(H5F_t *f_src, const H5O_storage_contig_t *storage_src, H5F_t *f_dst, H5O_storage_contig_t *storage_dst, H5T_t *src_dtype, H5O_copy_t *cpy_info); H5_DLL herr_t H5D__contig_delete(H5F_t *f, const H5O_storage_t *store); @@ -637,7 +641,13 @@ H5_DLL herr_t H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_ov const hsize_t old_dim[]); H5_DLL herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert, const hsize_t *scaled); +H5_DLL void * H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline); +H5_DLL void H5D__chunk_mem_free(void *chk, const void *_pline); +H5_DLL void * H5D__chunk_mem_xfree(void *chk, const void *pline); +H5_DLL void * H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline); H5_DLL herr_t H5D__chunk_update_old_edge_chunks(H5D_t *dset, hsize_t old_dim[]); +H5_DLL hbool_t H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, + const hsize_t *chunk_scaled, const hsize_t *dset_dims); H5_DLL herr_t H5D__chunk_prune_by_extent(H5D_t *dset, const hsize_t *old_dim); H5_DLL herr_t H5D__chunk_set_sizes(H5D_t *dset); #ifdef H5_HAVE_PARALLEL @@ -687,7 +697,7 @@ H5_DLL herr_t H5D__efl_bh_info(H5F_t *f, H5O_efl_t *efl, hsize_t *heap_size); /* Functions that perform fill value operations on datasets */ H5_DLL herr_t H5D__fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_type, - const H5S_t *space); + H5S_t *space); H5_DLL herr_t H5D__fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, H5MM_allocate_t alloc_func, void *alloc_info, H5MM_free_t free_func, void *free_info, const H5O_fill_t *fill, const H5T_t *dset_type, hid_t dset_type_id, size_t nelmts, size_t min_buf_size); @@ -696,39 +706,41 @@ H5_DLL herr_t H5D__fill_term(H5D_fill_buf_info_t *fb_info); #ifdef H5_HAVE_PARALLEL -#ifdef H5S_DEBUG +#ifdef H5D_DEBUG #ifndef H5Dmpio_DEBUG #define H5Dmpio_DEBUG #endif /*H5Dmpio_DEBUG*/ -#endif /*H5S_DEBUG*/ +#endif /*H5D_DEBUG*/ /* MPI-IO function to read, it will select either regular or irregular read */ H5_DLL herr_t H5D__mpio_select_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); /* MPI-IO function to write, it will select either regular or irregular read */ H5_DLL herr_t H5D__mpio_select_write(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); /* MPI-IO functions to handle contiguous collective IO */ H5_DLL herr_t H5D__contig_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); H5_DLL herr_t H5D__contig_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); /* MPI-IO functions to handle chunked collective IO */ H5_DLL herr_t H5D__chunk_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); H5_DLL herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); /* MPI-IO function to check if a direct I/O transfer is possible between * memory and the file */ H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, const H5S_t *mem_space, const H5D_type_info_t *type_info); +H5_DLL herr_t H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, + char *global_cause, size_t global_cause_len); #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index c496414..02644ed 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -77,7 +77,7 @@ typedef enum H5D_chunk_index_t { */ typedef enum H5D_alloc_time_t { H5D_ALLOC_TIME_ERROR = -1, /**< Error */ - H5D_ALLOC_TIME_DEFAULT = 0, /**< \todo Define this! */ + H5D_ALLOC_TIME_DEFAULT = 0, /**< Default (layout dependent) */ H5D_ALLOC_TIME_EARLY = 1, /**< Allocate on creation */ H5D_ALLOC_TIME_LATE = 2, /**< Allocate on first write */ H5D_ALLOC_TIME_INCR = 3 /**< Allocate incrementally (by chunk) */ @@ -91,9 +91,9 @@ typedef enum H5D_alloc_time_t { typedef enum H5D_space_status_t { H5D_SPACE_STATUS_ERROR = -1, /**< Error */ H5D_SPACE_STATUS_NOT_ALLOCATED = 0, /**< Space has not been allocated for this dataset. */ - H5D_SPACE_STATUS_PART_ALLOCATED = 1, /**< Space has been allocated for this dataset. */ - H5D_SPACE_STATUS_ALLOCATED = 2 /**< Space has been partially allocated for this dataset. (Used only for - datasets with chunked storage.) */ + H5D_SPACE_STATUS_PART_ALLOCATED = 1, /**< Space has been partially allocated for this dataset. + (Used only for datasets with chunked storage.) */ + H5D_SPACE_STATUS_ALLOCATED = 2 /**< Space has been allocated for this dataset. */ } H5D_space_status_t; //! @@ -127,8 +127,8 @@ typedef enum H5D_fill_value_t { */ typedef enum H5D_vds_view_t { H5D_VDS_ERROR = -1, /**< Error */ - H5D_VDS_FIRST_MISSING = 0, /**< \todo Define this! */ - H5D_VDS_LAST_AVAILABLE = 1 /**< \todo Define this! */ + H5D_VDS_FIRST_MISSING = 0, /**< Include all data before the first missing mapped data */ + H5D_VDS_LAST_AVAILABLE = 1 /**< Include all available mapped data */ } H5D_vds_view_t; //! @@ -300,7 +300,7 @@ extern "C" { * caller may derive new datatypes, dataspaces, and creation and * access properties from the old ones and reuse them in calls to * create additional datasets. Once created, the dataset can be - * read from or written to. Reading data from a datatset that was + * read from or written to. Reading data from a dataset that was * not previously written, the HDF5 library will return default * or user-defined fill values. * @@ -682,8 +682,7 @@ H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, u * Iterate over all chunked datasets and chunks in a file. * \snippet H5D_examples.c H5Ovisit_cb * - * \version 1.?.? - * \todo When was this function introduced? + * \since 1.13.0 * */ H5_DLL herr_t H5Dchunk_iter(hid_t dset_id, hid_t dxpl_id, H5D_chunk_iter_op_t cb, void *op_data); diff --git a/src/H5Dscatgath.c b/src/H5Dscatgath.c index 971ddfb..500ea9e 100644 --- a/src/H5Dscatgath.c +++ b/src/H5Dscatgath.c @@ -437,7 +437,7 @@ done: */ herr_t H5D__scatgath_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space) + H5S_t *file_space, H5S_t *mem_space) { void * buf = io_info->u.rbuf; /* Local pointer to application buffer */ H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info*/ @@ -577,7 +577,7 @@ done: */ herr_t H5D__scatgath_write(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space) + H5S_t *file_space, H5S_t *mem_space) { const void * buf = io_info->u.wbuf; /* Local pointer to application buffer */ H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info*/ diff --git a/src/H5Dselect.c b/src/H5Dselect.c index 7d2ead1..f464ca5 100644 --- a/src/H5Dselect.c +++ b/src/H5Dselect.c @@ -44,8 +44,8 @@ /* Local Prototypes */ /********************/ -static herr_t H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space); +static herr_t H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, H5S_t *file_space, + H5S_t *mem_space); /*********************/ /* Package Variables */ @@ -77,8 +77,8 @@ H5FL_EXTERN(H5S_sel_iter_t); *------------------------------------------------------------------------- */ static herr_t -H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, const H5S_t *file_space, - const H5S_t *mem_space) +H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, H5S_t *file_space, + H5S_t *mem_space) { H5S_sel_iter_t *mem_iter = NULL; /* Memory selection iteration info */ hbool_t mem_iter_init = FALSE; /* Memory selection iteration info has been initialized */ @@ -105,6 +105,9 @@ H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, co HDassert(io_info->store); HDassert(io_info->u.rbuf); + if (elmt_size == 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid elmt_size of 0") + /* Check for only one element in selection */ if (nelmts == 1) { hsize_t single_mem_off; /* Offset in memory */ @@ -226,8 +229,6 @@ H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, co /* Decrement number of elements left to process */ HDassert(((size_t)tmp_file_len % elmt_size) == 0); - if (elmt_size == 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "Resulted in division by zero") nelmts -= ((size_t)tmp_file_len / elmt_size); } /* end while */ } /* end else */ @@ -257,6 +258,188 @@ done: } /* end H5D__select_io() */ /*------------------------------------------------------------------------- + * Function: H5D_select_io_mem + * + * Purpose: Perform memory copies directly between two memory buffers + * according to the selections in the `dst_space` and + * `src_space` dataspaces. + * + * Note: This routine is [basically] the same as H5D__select_io, + * with the only difference being that the readvv/writevv + * calls are exchanged for H5VM_memcpyvv calls. Changes should + * be made to both routines. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_select_io_mem(void *dst_buf, const H5S_t *dst_space, const void *src_buf, const H5S_t *src_space, + size_t elmt_size, size_t nelmts) +{ + H5S_sel_iter_t *dst_sel_iter = NULL; /* Destination dataspace iteration info */ + H5S_sel_iter_t *src_sel_iter = NULL; /* Source dataspace iteration info */ + hbool_t dst_sel_iter_init = FALSE; /* Destination dataspace selection iterator initialized? */ + hbool_t src_sel_iter_init = FALSE; /* Source dataspace selection iterator initialized? */ + hsize_t * dst_off = NULL; /* Pointer to sequence offsets in destination buffer */ + hsize_t * src_off = NULL; /* Pointer to sequence offsets in source buffer */ + size_t * dst_len = NULL; /* Pointer to sequence lengths in destination buffer */ + size_t * src_len = NULL; /* Pointer to sequence lengths in source buffer */ + size_t curr_dst_seq; /* Current destination buffer sequence to operate on */ + size_t curr_src_seq; /* Current source buffer sequence to operate on */ + size_t dst_nseq; /* Number of sequences generated for destination buffer */ + size_t src_nseq; /* Number of sequences generated for source buffer */ + size_t dxpl_vec_size; /* Vector length from API context's DXPL */ + size_t vec_size; /* Vector length */ + ssize_t bytes_copied; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(dst_buf); + HDassert(dst_space); + HDassert(src_buf); + HDassert(src_space); + + if (elmt_size == 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid elmt_size of 0") + + /* Check for only one element in selection */ + if (nelmts == 1) { + hsize_t single_dst_off; /* Offset in dst_space */ + hsize_t single_src_off; /* Offset in src_space */ + size_t single_dst_len; /* Length in dst_space */ + size_t single_src_len; /* Length in src_space */ + + /* Get offset of first element in selections */ + if (H5S_SELECT_OFFSET(dst_space, &single_dst_off) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve destination selection offset") + if (H5S_SELECT_OFFSET(src_space, &single_src_off) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve source selection offset") + + /* Set up necessary information for I/O operation */ + dst_nseq = src_nseq = 1; + curr_dst_seq = curr_src_seq = 0; + single_dst_off *= elmt_size; + single_src_off *= elmt_size; + single_dst_len = single_src_len = elmt_size; + + /* Perform vectorized memcpy from src_buf to dst_buf */ + if ((bytes_copied = + H5VM_memcpyvv(dst_buf, dst_nseq, &curr_dst_seq, &single_dst_len, &single_dst_off, src_buf, + src_nseq, &curr_src_seq, &single_src_len, &single_src_off)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + + HDassert(((size_t)bytes_copied % elmt_size) == 0); + } + else { + unsigned sel_iter_flags = H5S_SEL_ITER_GET_SEQ_LIST_SORTED | H5S_SEL_ITER_SHARE_WITH_DATASPACE; + size_t dst_nelem; /* Number of elements used in destination buffer sequences */ + size_t src_nelem; /* Number of elements used in source buffer sequences */ + + /* Get info from API context */ + if (H5CX_get_vec_size(&dxpl_vec_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "can't retrieve I/O vector size") + + /* Allocate the vector I/O arrays */ + if (dxpl_vec_size > H5D_IO_VECTOR_SIZE) + vec_size = dxpl_vec_size; + else + vec_size = H5D_IO_VECTOR_SIZE; + + if (NULL == (dst_len = H5FL_SEQ_MALLOC(size_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O length vector array") + if (NULL == (dst_off = H5FL_SEQ_MALLOC(hsize_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O offset vector array") + if (NULL == (src_len = H5FL_SEQ_MALLOC(size_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O length vector array") + if (NULL == (src_off = H5FL_SEQ_MALLOC(hsize_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O offset vector array") + + /* Allocate the dataspace selection iterators */ + if (NULL == (dst_sel_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate destination selection iterator") + if (NULL == (src_sel_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate source selection iterator") + + /* Initialize destination selection iterator */ + if (H5S_select_iter_init(dst_sel_iter, dst_space, elmt_size, sel_iter_flags) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + dst_sel_iter_init = TRUE; /* Destination selection iteration info has been initialized */ + + /* Initialize source selection iterator */ + if (H5S_select_iter_init(src_sel_iter, src_space, elmt_size, H5S_SEL_ITER_SHARE_WITH_DATASPACE) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + src_sel_iter_init = TRUE; /* Source selection iteration info has been initialized */ + + /* Initialize sequence counts */ + curr_dst_seq = curr_src_seq = 0; + dst_nseq = src_nseq = 0; + + /* Loop, until all bytes are processed */ + while (nelmts > 0) { + /* Check if more destination buffer sequences are needed */ + if (curr_dst_seq >= dst_nseq) { + /* Get sequences for destination selection */ + if (H5S_SELECT_ITER_GET_SEQ_LIST(dst_sel_iter, vec_size, nelmts, &dst_nseq, &dst_nelem, + dst_off, dst_len) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed") + + /* Start at the beginning of the sequences again */ + curr_dst_seq = 0; + } + + /* Check if more source buffer sequences are needed */ + if (curr_src_seq >= src_nseq) { + /* Get sequences for source selection */ + if (H5S_SELECT_ITER_GET_SEQ_LIST(src_sel_iter, vec_size, nelmts, &src_nseq, &src_nelem, + src_off, src_len) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed") + + /* Start at the beginning of the sequences again */ + curr_src_seq = 0; + } /* end if */ + + /* Perform vectorized memcpy from src_buf to dst_buf */ + if ((bytes_copied = H5VM_memcpyvv(dst_buf, dst_nseq, &curr_dst_seq, dst_len, dst_off, src_buf, + src_nseq, &curr_src_seq, src_len, src_off)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + + /* Decrement number of elements left to process */ + HDassert(((size_t)bytes_copied % elmt_size) == 0); + nelmts -= ((size_t)bytes_copied / elmt_size); + } + } + +done: + /* Release selection iterators */ + if (src_sel_iter) { + if (src_sel_iter_init && H5S_SELECT_ITER_RELEASE(src_sel_iter) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator") + + src_sel_iter = H5FL_FREE(H5S_sel_iter_t, src_sel_iter); + } + if (dst_sel_iter) { + if (dst_sel_iter_init && H5S_SELECT_ITER_RELEASE(dst_sel_iter) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator") + + dst_sel_iter = H5FL_FREE(H5S_sel_iter_t, dst_sel_iter); + } + + /* Release vector arrays, if allocated */ + if (src_off) + src_off = H5FL_SEQ_FREE(hsize_t, src_off); + if (src_len) + src_len = H5FL_SEQ_FREE(size_t, src_len); + if (dst_off) + dst_off = H5FL_SEQ_FREE(hsize_t, dst_off); + if (dst_len) + dst_len = H5FL_SEQ_FREE(size_t, dst_len); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_select_io_mem() */ + +/*------------------------------------------------------------------------- * Function: H5D__select_read * * Purpose: Reads directly from file into application memory. @@ -270,7 +453,7 @@ done: */ herr_t H5D__select_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space) + H5S_t *file_space, H5S_t *mem_space) { herr_t ret_value = SUCCEED; /* Return value */ @@ -299,7 +482,7 @@ done: */ herr_t H5D__select_write(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space) + H5S_t *file_space, H5S_t *mem_space) { herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5Dsingle.c b/src/H5Dsingle.c index eba569e..a843d88 100644 --- a/src/H5Dsingle.c +++ b/src/H5Dsingle.c @@ -126,8 +126,14 @@ H5D__single_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t H5_ATTR_UNU HDassert(idx_info->layout); HDassert(idx_info->storage); - if (idx_info->pline->nused) + if (idx_info->pline->nused) { idx_info->layout->flags |= H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER; + + if (!H5F_addr_defined(idx_info->storage->idx_addr)) { + idx_info->storage->u.single.nbytes = 0; + idx_info->storage->u.single.filter_mask = 0; + } + } else idx_info->layout->flags = 0; diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index d5375c1..7158e05 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -84,9 +84,9 @@ /* Layout operation callbacks */ static hbool_t H5D__virtual_is_data_cached(const H5D_shared_t *shared_dset); static herr_t H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm); + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); static herr_t H5D__virtual_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm); + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); static herr_t H5D__virtual_flush(H5D_t *dset); /* Other functions */ @@ -103,26 +103,37 @@ static herr_t H5D__virtual_build_source_name(char * size_t static_strlen, size_t nsubs, hsize_t blockno, char **built_name); static herr_t H5D__virtual_init_all(const H5D_t *dset); -static herr_t H5D__virtual_pre_io(H5D_io_info_t *io_info, H5O_storage_virtual_t *storage, - const H5S_t *file_space, const H5S_t *mem_space, hsize_t *tot_nelmts); +static herr_t H5D__virtual_pre_io(H5D_io_info_t *io_info, H5O_storage_virtual_t *storage, H5S_t *file_space, + H5S_t *mem_space, hsize_t *tot_nelmts); static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage); static herr_t H5D__virtual_read_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset); + H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset); static herr_t H5D__virtual_write_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset); + H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset); /*********************/ /* Package Variables */ /*********************/ /* Contiguous storage layout I/O ops */ -const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1] = {{NULL, H5D__virtual_init, H5D__virtual_is_space_alloc, - H5D__virtual_is_data_cached, NULL, H5D__virtual_read, - H5D__virtual_write, +const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1] = {{ + NULL, /* construct */ + H5D__virtual_init, /* init */ + H5D__virtual_is_space_alloc, /* is_space_alloc */ + H5D__virtual_is_data_cached, /* is_data_cached */ + NULL, /* io_init */ + H5D__virtual_read, /* ser_read */ + H5D__virtual_write, /* ser_write */ #ifdef H5_HAVE_PARALLEL - NULL, NULL, -#endif /* H5_HAVE_PARALLEL */ - NULL, NULL, H5D__virtual_flush, NULL, NULL}}; + NULL, /* par_read */ + NULL, /* par_write */ +#endif + NULL, /* readvv */ + NULL, /* writevv */ + H5D__virtual_flush, /* flush */ + NULL, /* io_term */ + NULL /* dest */ +}}; /*******************/ /* Local Variables */ @@ -195,7 +206,7 @@ H5D_virtual_check_mapping_pre(const H5S_t *vspace, const H5S_t *src_space, "can't get number of elements in non-unlimited dimension") if (nenu_vs != nenu_ss) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, - "numbers of elemnts in the non-unlimited dimensions is different for source and " + "numbers of elements in the non-unlimited dimensions is different for source and " "virtual spaces") } /* end if */ /* We will handle the printf case after parsing the source names */ @@ -1310,7 +1321,7 @@ H5D_virtual_free_parsed_name(H5O_storage_virtual_name_seg_t *name_seg) H5O_storage_virtual_name_seg_t *next_seg; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Walk name segments, freeing them */ while (name_seg) { @@ -1320,7 +1331,6 @@ H5D_virtual_free_parsed_name(H5O_storage_virtual_name_seg_t *name_seg) name_seg = next_seg; } /* end while */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_virtual_free_parsed_name() */ @@ -1488,7 +1498,7 @@ H5D__virtual_set_extent_unlim(const H5D_t *dset) storage->list[i].source_dset.dset->shared->space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") - /* Get source space dimenstions */ + /* Get source space dimensions */ if (H5S_get_simple_extent_dims(storage->list[i].source_select, curr_dims, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get source space dimensions") @@ -1966,7 +1976,7 @@ H5D__virtual_init_all(const H5D_t *dset) storage->list[i].source_dset.dset->shared->space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") - /* Get source space dimenstions */ + /* Get source space dimensions */ if (H5S_get_simple_extent_dims(storage->list[i].source_select, source_dims, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get source space dimensions") @@ -2243,10 +2253,38 @@ H5D__virtual_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id) storage->printf_gap = (hsize_t)0; /* Retrieve VDS file FAPL to layout */ - if (storage->source_fapl <= 0) + if (storage->source_fapl <= 0) { + H5P_genplist_t * source_fapl = NULL; /* Source file FAPL */ + H5F_close_degree_t close_degree = H5F_CLOSE_WEAK; /* Close degree for source files */ + if ((storage->source_fapl = H5F_get_access_plist(f, FALSE)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get fapl") + /* Get property list pointer */ + if (NULL == (source_fapl = (H5P_genplist_t *)H5I_object(storage->source_fapl))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, H5I_INVALID_HID, "not a property list") + + /* Source files must always be opened with H5F_CLOSE_WEAK close degree */ + if (H5P_set(source_fapl, H5F_ACS_CLOSE_DEGREE_NAME, &close_degree) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file close degree") + } /* end if */ +#ifndef NDEBUG + else { + H5P_genplist_t * source_fapl = NULL; /* Source file FAPL */ + H5F_close_degree_t close_degree; /* Close degree for source files */ + + /* Get property list pointer */ + if (NULL == (source_fapl = (H5P_genplist_t *)H5I_object(storage->source_fapl))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, H5I_INVALID_HID, "not a property list") + + /* Verify H5F_CLOSE_WEAK close degree is set */ + if (H5P_get(source_fapl, H5F_ACS_CLOSE_DEGREE_NAME, &close_degree) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file close degree") + + HDassert(close_degree == H5F_CLOSE_WEAK); + } /* end else */ +#endif /* NDEBUG */ + /* Copy DAPL to layout */ if (storage->source_dapl <= 0) if ((storage->source_dapl = H5P_copy_plist(dapl, FALSE)) < 0) @@ -2322,7 +2360,7 @@ H5D__virtual_is_data_cached(const H5D_shared_t *shared_dset) if (storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { /* Iterate over sub-source dsets */ for (j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) - /* Check for cahced data in source dset */ + /* Check for cached data in source dset */ if (storage->list[i].sub_dset[j].dset && storage->list[i].sub_dset[j].dset->shared->layout.ops->is_data_cached && storage->list[i].sub_dset[j].dset->shared->layout.ops->is_data_cached( @@ -2355,8 +2393,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_pre_io(H5D_io_info_t *io_info, H5O_storage_virtual_t *storage, const H5S_t *file_space, - const H5S_t *mem_space, hsize_t *tot_nelmts) +H5D__virtual_pre_io(H5D_io_info_t *io_info, H5O_storage_virtual_t *storage, H5S_t *file_space, + H5S_t *mem_space, hsize_t *tot_nelmts) { hssize_t select_nelmts; /* Number of elements in selection */ hsize_t bounds_start[H5S_MAX_RANK]; /* Selection bounds start */ @@ -2656,7 +2694,7 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) /*------------------------------------------------------------------------- * Function: H5D__virtual_read_one * - * Purpose: Read from a singe source dataset in a virtual dataset. + * Purpose: Read from a single source dataset in a virtual dataset. * * Return: Non-negative on success/Negative on failure * @@ -2666,7 +2704,7 @@ H5D__virtual_post_io(H5O_storage_virtual_t *storage) *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_read_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5S_t *file_space, +H5D__virtual_read_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset) { H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ @@ -2726,8 +2764,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) +H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, H5S_t *file_space, + H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ @@ -2846,7 +2884,7 @@ done: /*------------------------------------------------------------------------- * Function: H5D__virtual_write_one * - * Purpose: Write to a singe source dataset in a virtual dataset. + * Purpose: Write to a single source dataset in a virtual dataset. * * Return: Non-negative on success/Negative on failure * @@ -2856,7 +2894,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__virtual_write_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5S_t *file_space, +H5D__virtual_write_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset) { H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ @@ -2919,7 +2957,7 @@ done: */ static herr_t H5D__virtual_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, - const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) + H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ diff --git a/src/H5E.c b/src/H5E.c index a5a9a2e..d203248 100644 --- a/src/H5E.c +++ b/src/H5E.c @@ -92,9 +92,6 @@ static herr_t H5E__append_stack(H5E_t *dst_estack, const H5E_t *src_stack); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -141,46 +138,19 @@ static const H5I_class_t H5I_ERRSTK_CLS[1] = {{ * * Purpose: Initialize the interface from some other layer. * - * Return: SUCCEED/FAIL - * - * Programmer: Quincey Koziol - * Tuesday, June 29, 2004 - * + * Return: Success: non-negative + * Failure: negative *------------------------------------------------------------------------- */ herr_t H5E_init(void) { - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5E_init() */ - -/*-------------------------------------------------------------------------- - * Function: H5E__init_package - * - * Purpose: Initialize interface-specific information - * - * Return: SUCCEED/FAIL - * - * Programmer: Raymond Lu - * Friday, July 11, 2003 - * - *-------------------------------------------------------------------------- - */ -herr_t -H5E__init_package(void) -{ H5E_cls_t *cls; /* Pointer to error class */ H5E_msg_t *msg; /* Pointer to new error message */ char lib_vers[128]; /* Buffer to constructu library version within */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_NOAPI(FAIL) /* Initialize the ID group for the error class IDs */ if (H5I_register_type(H5I_ERRCLS_CLS) < 0) @@ -213,7 +183,7 @@ H5E__init_package(void) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5E__init_package() */ +} /*------------------------------------------------------------------------- * Function: H5E_term_package @@ -237,66 +207,61 @@ H5E_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - int64_t ncls, nmsg, nstk; - - /* Check if there are any open error stacks, classes or messages */ - ncls = H5I_nmembers(H5I_ERROR_CLASS); - nmsg = H5I_nmembers(H5I_ERROR_MSG); - nstk = H5I_nmembers(H5I_ERROR_STACK); - - if ((ncls + nmsg + nstk) > 0) { - /* Clear the default error stack. Note that - * the following H5I_clear_type calls do not - * force the clears and will not be able to - * clear any error message IDs that are still - * in use by the default error stack unless we - * clear that stack manually. - * - * Error message IDs will typically still be - * in use by the default error stack when the - * application does H5E_BEGIN/END_TRY cleanup - * at the very end. - */ - H5E_clear_stack(NULL); - - /* Clear any outstanding error stacks */ - if (nstk > 0) - (void)H5I_clear_type(H5I_ERROR_STACK, FALSE, FALSE); - - /* Clear all the error classes */ - if (ncls > 0) { - (void)H5I_clear_type(H5I_ERROR_CLASS, FALSE, FALSE); - - /* Reset the HDF5 error class, if its been closed */ - if (H5I_nmembers(H5I_ERROR_CLASS) == 0) - H5E_ERR_CLS_g = -1; - } /* end if */ + int64_t ncls, nmsg, nstk; + + /* Check if there are any open error stacks, classes or messages */ + ncls = H5I_nmembers(H5I_ERROR_CLASS); + nmsg = H5I_nmembers(H5I_ERROR_MSG); + nstk = H5I_nmembers(H5I_ERROR_STACK); + + if ((ncls + nmsg + nstk) > 0) { + /* Clear the default error stack. Note that + * the following H5I_clear_type calls do not + * force the clears and will not be able to + * clear any error message IDs that are still + * in use by the default error stack unless we + * clear that stack manually. + * + * Error message IDs will typically still be + * in use by the default error stack when the + * application does H5E_BEGIN/END_TRY cleanup + * at the very end. + */ + H5E_clear_stack(NULL); + + /* Clear any outstanding error stacks */ + if (nstk > 0) + (void)H5I_clear_type(H5I_ERROR_STACK, FALSE, FALSE); + + /* Clear all the error classes */ + if (ncls > 0) { + (void)H5I_clear_type(H5I_ERROR_CLASS, FALSE, FALSE); - /* Clear all the error messages */ - if (nmsg > 0) { - (void)H5I_clear_type(H5I_ERROR_MSG, FALSE, FALSE); + /* Reset the HDF5 error class, if its been closed */ + if (H5I_nmembers(H5I_ERROR_CLASS) == 0) + H5E_ERR_CLS_g = -1; + } /* end if */ - /* Reset the HDF5 error messages, if they've been closed */ - if (H5I_nmembers(H5I_ERROR_MSG) == 0) { + /* Clear all the error messages */ + if (nmsg > 0) { + (void)H5I_clear_type(H5I_ERROR_MSG, FALSE, FALSE); + + /* Reset the HDF5 error messages, if they've been closed */ + if (H5I_nmembers(H5I_ERROR_MSG) == 0) { /* Include the automatically generated error code termination */ #include "H5Eterm.h" - } /* end if */ - } /* end if */ - - n++; /*H5I*/ - } /* end if */ - else { - /* Destroy the error class, message, and stack id groups */ - n += (H5I_dec_type_ref(H5I_ERROR_STACK) > 0); - n += (H5I_dec_type_ref(H5I_ERROR_CLASS) > 0); - n += (H5I_dec_type_ref(H5I_ERROR_MSG) > 0); - - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ + } /* end if */ + } /* end if */ + + n++; /*H5I*/ + } /* end if */ + else { + /* Destroy the error class, message, and stack id groups */ + n += (H5I_dec_type_ref(H5I_ERROR_STACK) > 0); + n += (H5I_dec_type_ref(H5I_ERROR_CLASS) > 0); + n += (H5I_dec_type_ref(H5I_ERROR_MSG) > 0); + + } /* end else */ FUNC_LEAVE_NOAPI(n) } /* end H5E_term_package() */ @@ -721,7 +686,7 @@ done: /*------------------------------------------------------------------------- * Function: H5E__close_msg * - * Purpose: Private function to close an error messge. + * Purpose: Private function to close an error message. * * Return: SUCCEED/FAIL * diff --git a/src/H5EA.c b/src/H5EA.c index 8f9a4b6..66eb8cc 100644 --- a/src/H5EA.c +++ b/src/H5EA.c @@ -75,9 +75,6 @@ static H5EA_t *H5EA__new(H5F_t *f, haddr_t ea_addr, hbool_t from_open, void *ctx /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Extensible array client ID to class mapping */ /* Remember to add client ID to H5EA_cls_id_t in H5EAprivate.h when adding a new diff --git a/src/H5EAdbg.c b/src/H5EAdbg.c index b0e564c..b377422 100644 --- a/src/H5EAdbg.c +++ b/src/H5EAdbg.c @@ -237,7 +237,7 @@ H5EA__iblock_debug(H5F_t *f, haddr_t H5_ATTR_UNUSED addr, FILE *stream, int inde HDfprintf(stream, "%*sData Block Addresses in Index Block:\n", indent, ""); for (u = 0; u < iblock->ndblk_addrs; u++) { /* Print address */ - HDsprintf(temp_str, "Address #%u:", u); + HDsnprintf(temp_str, sizeof(temp_str), "Address #%u:", u); HDfprintf(stream, "%*s%-*s %" PRIuHADDR "\n", (indent + 3), "", MAX(0, (fwidth - 3)), temp_str, iblock->dblk_addrs[u]); } /* end for */ @@ -252,7 +252,7 @@ H5EA__iblock_debug(H5F_t *f, haddr_t H5_ATTR_UNUSED addr, FILE *stream, int inde HDfprintf(stream, "%*sSuper Block Addresses in Index Block:\n", indent, ""); for (u = 0; u < iblock->nsblk_addrs; u++) { /* Print address */ - HDsprintf(temp_str, "Address #%u:", u); + HDsnprintf(temp_str, sizeof(temp_str), "Address #%u:", u); HDfprintf(stream, "%*s%-*s %" PRIuHADDR "\n", (indent + 3), "", MAX(0, (fwidth - 3)), temp_str, iblock->sblk_addrs[u]); } /* end for */ @@ -341,7 +341,7 @@ H5EA__sblock_debug(H5F_t *f, haddr_t addr, FILE *stream, int indent, int fwidth, HDfprintf(stream, "%*sData Block Addresses in Super Block:\n", indent, ""); for (u = 0; u < sblock->ndblks; u++) { /* Print address */ - HDsprintf(temp_str, "Address #%u:", u); + HDsnprintf(temp_str, sizeof(temp_str), "Address #%u:", u); HDfprintf(stream, "%*s%-*s %" PRIuHADDR "\n", (indent + 3), "", MAX(0, (fwidth - 3)), temp_str, sblock->dblk_addrs[u]); } /* end for */ diff --git a/src/H5EApkg.h b/src/H5EApkg.h index bfa8588..2212ccb 100644 --- a/src/H5EApkg.h +++ b/src/H5EApkg.h @@ -301,7 +301,7 @@ typedef struct H5EA_dblock_t { /* Computed/cached values (not stored) */ size_t nelmts; /* Number of elements in block */ - size_t npages; /* Nummber of pages in a block (zero if not paged) */ + size_t npages; /* Number of pages in a block (zero if not paged) */ } H5EA_dblock_t; /* The extensible array data block page information */ diff --git a/src/H5EAtest.c b/src/H5EAtest.c index 7924eaa..24efbc2 100644 --- a/src/H5EAtest.c +++ b/src/H5EAtest.c @@ -322,7 +322,7 @@ H5EA__test_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const void * HDassert(elmt); /* Print element */ - HDsprintf(temp_str, "Element #%llu:", (unsigned long long)idx); + HDsnprintf(temp_str, sizeof(temp_str), "Element #%llu:", (unsigned long long)idx); HDfprintf(stream, "%*s%-*s %llu\n", indent, "", fwidth, temp_str, (unsigned long long)*(const uint64_t *)elmt); diff --git a/src/H5ES.c b/src/H5ES.c index ccc0dd8..ad42000 100644 --- a/src/H5ES.c +++ b/src/H5ES.c @@ -236,6 +236,61 @@ done: } /* end H5ESget_op_counter() */ /*------------------------------------------------------------------------- + * Function: H5ESget_requests + * + * Purpose: Retrieve the requests in an event set. Up to *count + * requests are stored in the provided requests array, and + * the connector ids corresponding to these requests are + * stored in the provided connector_ids array. Either or + * both of these arrays may be NULL, in which case this + * information is not returned. If these arrays are + * non-NULL, they must be large enough to contain *count + * entries. On exit, *count is set to the total number of + * events in the event set. + * + * Events are returned in the order they were added to the + * event set. If order is H5_ITER_INC or H5_ITER_NATIVE, + * events will be returned starting from the oldest. If order + * is H5_ITER_DEC, events will be returned starting with the + * newest/most recent. + * + * Return: SUCCEED / FAIL + * + * Programmer: Neil Fortner + * Tuesday, November 23, 2021 + * + *------------------------------------------------------------------------- + */ +herr_t +H5ESget_requests(hid_t es_id, H5_iter_order_t order, hid_t *connector_ids, void **requests, size_t array_len, + size_t *count /*out*/) +{ + H5ES_t *es; /* Event set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "iIo*i**xzx", es_id, order, connector_ids, requests, array_len, count); + + /* Check arguments */ + if (NULL == (es = H5I_object_verify(es_id, H5I_EVENTSET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid event set identifier") + if (order <= H5_ITER_UNKNOWN || order >= H5_ITER_N) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid iteration order specified") + + /* Call internal routine */ + if (array_len > 0 && (requests || connector_ids)) + if (H5ES__get_requests(es, order, connector_ids, requests, array_len) < 0) + HGOTO_ERROR(H5E_EVENTSET, H5E_CANTGET, FAIL, "can't get requests") + + /* Retrieve the count, if non-NULL */ + if (count) + *count = H5ES__list_count(&es->active); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5ESget_requests() */ + +/*------------------------------------------------------------------------- * Function: H5ESwait * * Purpose: Wait (with timeout) for operations in event set to complete diff --git a/src/H5ESdevelop.h b/src/H5ESdevelop.h index 5a0f2b4..2fb9aeb 100644 --- a/src/H5ESdevelop.h +++ b/src/H5ESdevelop.h @@ -42,6 +42,8 @@ extern "C" { #endif H5_DLL herr_t H5ESinsert_request(hid_t es_id, hid_t connector_id, void *request); +H5_DLL herr_t H5ESget_requests(hid_t es_id, H5_iter_order_t order, hid_t *connector_ids, void **requests, + size_t array_len, size_t *count); #ifdef __cplusplus } diff --git a/src/H5ESint.c b/src/H5ESint.c index 6f9efe9..7eb5909 100644 --- a/src/H5ESint.c +++ b/src/H5ESint.c @@ -50,6 +50,14 @@ /* Local Typedefs */ /******************/ +/* Callback context for get events operations */ +typedef struct H5ES_get_requests_ctx_t { + hid_t *connector_ids; /* Output buffer for list of connector IDs that match the above requests */ + void **requests; /* Output buffer for list of requests in event set */ + size_t array_len; /* Length of the above output buffers */ + size_t i; /* Number of elements filled in output buffers */ +} H5ES_get_requests_ctx_t; + /* Callback context for wait operations */ typedef struct H5ES_wait_ctx_t { H5ES_t * es; /* Event set being operated on */ @@ -84,6 +92,7 @@ static herr_t H5ES__close(H5ES_t *es); static herr_t H5ES__close_cb(void *es, void **request_token); static herr_t H5ES__insert(H5ES_t *es, H5VL_t *connector, void *request_token, const char *app_file, const char *app_func, unsigned app_line, const char *caller, const char *api_args); +static int H5ES__get_requests_cb(H5ES_event_t *ev, void *_ctx); static herr_t H5ES__handle_fail(H5ES_t *es, H5ES_event_t *ev); static herr_t H5ES__op_complete(H5ES_t *es, H5ES_event_t *ev, H5VL_request_status_t ev_status); static int H5ES__wait_cb(H5ES_event_t *ev, void *_ctx); @@ -95,9 +104,6 @@ static int H5ES__close_failed_cb(H5ES_event_t *ev, void *_ctx); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -118,23 +124,20 @@ static const H5I_class_t H5I_EVENTSET_CLS[1] = {{ H5FL_DEFINE_STATIC(H5ES_t); /*------------------------------------------------------------------------- - * Function: H5ES__init_package + * Function: H5ES_init * - * Purpose: Initializes any interface-specific data or routines. - * - * Return: Non-negative on success / Negative on failure - * - * Programmer: Quincey Koziol - * Monday, April 6, 2020 + * Purpose: Initialize the interface from some other layer. * + * Return: Success: non-negative + * Failure: negative *------------------------------------------------------------------------- */ herr_t -H5ES__init_package(void) +H5ES_init(void) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_NOAPI(FAIL) /* Initialize the ID group for the event set IDs */ if (H5I_register_type(H5I_EVENTSET_CLS) < 0) @@ -142,7 +145,7 @@ H5ES__init_package(void) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5ES__init_package() */ +} /*------------------------------------------------------------------------- * Function: H5ES_term_package @@ -165,14 +168,8 @@ H5ES_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Destroy the event set ID group */ - n += (H5I_dec_type_ref(H5I_EVENTSET) > 0); - - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Destroy the event set ID group */ + n += (H5I_dec_type_ref(H5I_EVENTSET) > 0); FUNC_LEAVE_NOAPI(n) } /* end H5ES_term_package() */ @@ -294,7 +291,8 @@ H5ES__insert(H5ES_t *es, H5VL_t *connector, void *request_token, const char *app * there's no need to duplicate it. */ ev->op_info.api_name = caller; - if (NULL == (ev->op_info.api_args = H5MM_xstrdup(api_args))) + HDassert(ev->op_info.api_args == NULL); + if (api_args && NULL == (ev->op_info.api_args = H5MM_xstrdup(api_args))) HGOTO_ERROR(H5E_EVENTSET, H5E_CANTALLOC, FAIL, "can't copy API routine arguments") /* Append fully initialized event onto the event set's 'active' list */ @@ -431,6 +429,86 @@ done: } /* end H5ES__insert_request() */ /*------------------------------------------------------------------------- + * Function: H5ES__get_requests_cb + * + * Purpose: Iterator callback for H5ES__get_events - adds the event to + * the list. + * + * Return: SUCCEED / FAIL + * + * Programmer: Neil Fortner + * Tuesday, November 23, 2021 + * + *------------------------------------------------------------------------- + */ +static int +H5ES__get_requests_cb(H5ES_event_t *ev, void *_ctx) +{ + H5ES_get_requests_ctx_t *ctx = (H5ES_get_requests_ctx_t *)_ctx; /* Callback context */ + int ret_value = H5_ITER_CONT; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(ev); + HDassert(ctx); + HDassert(ctx->i < ctx->array_len); + + /* Get the connector ID for the event */ + if (ctx->connector_ids) + ctx->connector_ids[ctx->i] = ev->request->connector->id; + + /* Get the request for the event */ + if (ctx->requests) + ctx->requests[ctx->i] = ev->request->data; + + /* Check if we've run out of room in the arrays */ + if (++ctx->i == ctx->array_len) + ret_value = H5_ITER_STOP; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ES__get_requests_cb() */ + +/*------------------------------------------------------------------------- + * Function: H5ES__get_requests + * + * Purpose: Get all requests in an event set. + * + * Return: SUCCEED / FAIL + * + * Programmer: Neil Fortner + * Tuesday, November 23, 2021 + * + *------------------------------------------------------------------------- + */ +herr_t +H5ES__get_requests(H5ES_t *es, H5_iter_order_t order, hid_t *connector_ids, void **requests, size_t array_len) +{ + H5ES_get_requests_ctx_t ctx; /* Callback context */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(es); + HDassert(array_len > 0); + HDassert(requests || connector_ids); + + /* Set up context for iterator callbacks */ + ctx.connector_ids = connector_ids; + ctx.requests = requests; + ctx.array_len = array_len; + ctx.i = 0; + + /* Iterate over the events in the set */ + if (H5ES__list_iterate(&es->active, order, H5ES__get_requests_cb, &ctx) < 0) + HGOTO_ERROR(H5E_EVENTSET, H5E_BADITER, FAIL, "iteration failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ES__get_requests() */ + +/*------------------------------------------------------------------------- * Function: H5ES__handle_fail * * Purpose: Handle a failed event @@ -673,7 +751,7 @@ H5ES__wait(H5ES_t *es, uint64_t timeout, size_t *num_in_progress, hbool_t *op_fa ctx.op_failed = op_failed; /* Iterate over the events in the set, waiting for them to complete */ - if (H5ES__list_iterate(&es->active, H5ES__wait_cb, &ctx) < 0) + if (H5ES__list_iterate(&es->active, H5_ITER_NATIVE, H5ES__wait_cb, &ctx) < 0) HGOTO_ERROR(H5E_EVENTSET, H5E_BADITER, FAIL, "iteration failed") done: @@ -781,7 +859,7 @@ H5ES__cancel(H5ES_t *es, size_t *num_not_canceled, hbool_t *op_failed) ctx.op_failed = op_failed; /* Iterate over the events in the set, attempting to cancel them */ - if (H5ES__list_iterate(&es->active, H5ES__cancel_cb, &ctx) < 0) + if (H5ES__list_iterate(&es->active, H5_ITER_NATIVE, H5ES__cancel_cb, &ctx) < 0) HGOTO_ERROR(H5E_EVENTSET, H5E_BADITER, FAIL, "iteration failed") done: @@ -818,13 +896,13 @@ H5ES__get_err_info_cb(H5ES_event_t *ev, void *_ctx) * so there's no need to duplicate them internally, but they are duplicated * here, when they are given back to the user. */ - if (NULL == (ctx->curr_err_info->api_name = H5MM_strdup(ev->op_info.api_name))) + if (NULL == (ctx->curr_err_info->api_name = H5MM_xstrdup(ev->op_info.api_name))) HGOTO_ERROR(H5E_EVENTSET, H5E_CANTALLOC, H5_ITER_ERROR, "can't copy HDF5 API routine name") - if (NULL == (ctx->curr_err_info->api_args = H5MM_strdup(ev->op_info.api_args))) + if (NULL == (ctx->curr_err_info->api_args = H5MM_xstrdup(ev->op_info.api_args))) HGOTO_ERROR(H5E_EVENTSET, H5E_CANTALLOC, H5_ITER_ERROR, "can't copy HDF5 API routine arguments") - if (NULL == (ctx->curr_err_info->app_file_name = H5MM_strdup(ev->op_info.app_file_name))) + if (NULL == (ctx->curr_err_info->app_file_name = H5MM_xstrdup(ev->op_info.app_file_name))) HGOTO_ERROR(H5E_EVENTSET, H5E_CANTALLOC, H5_ITER_ERROR, "can't copy HDF5 application file name") - if (NULL == (ctx->curr_err_info->app_func_name = H5MM_strdup(ev->op_info.app_func_name))) + if (NULL == (ctx->curr_err_info->app_func_name = H5MM_xstrdup(ev->op_info.app_func_name))) HGOTO_ERROR(H5E_EVENTSET, H5E_CANTALLOC, H5_ITER_ERROR, "can't copy HDF5 application function name") ctx->curr_err_info->app_line_num = ev->op_info.app_line_num; ctx->curr_err_info->op_ins_count = ev->op_info.op_ins_count; @@ -895,7 +973,7 @@ H5ES__get_err_info(H5ES_t *es, size_t num_err_info, H5ES_err_info_t err_info[], ctx.curr_err_info = &err_info[0]; /* Iterate over the failed events in the set, copying their error info */ - if (H5ES__list_iterate(&es->failed, H5ES__get_err_info_cb, &ctx) < 0) + if (H5ES__list_iterate(&es->failed, H5_ITER_NATIVE, H5ES__get_err_info_cb, &ctx) < 0) HGOTO_ERROR(H5E_EVENTSET, H5E_BADITER, FAIL, "iteration failed") /* Set # of failed events cleared from event set's failed list */ @@ -969,7 +1047,7 @@ H5ES__close(H5ES_t *es) "can't close event set while unfinished operations are present (i.e. wait on event set first)") /* Iterate over the failed events in the set, releasing them */ - if (H5ES__list_iterate(&es->failed, H5ES__close_failed_cb, (void *)es) < 0) + if (H5ES__list_iterate(&es->failed, H5_ITER_NATIVE, H5ES__close_failed_cb, (void *)es) < 0) HGOTO_ERROR(H5E_EVENTSET, H5E_BADITER, FAIL, "iteration failed") /* Release the event set */ diff --git a/src/H5ESlist.c b/src/H5ESlist.c index c0e24cc..61a9dd1 100644 --- a/src/H5ESlist.c +++ b/src/H5ESlist.c @@ -88,6 +88,8 @@ H5ES__list_append(H5ES_event_list_t *el, H5ES_event_t *ev) HDassert(el); HDassert(ev); + ev->next = NULL; + /* Append event onto the event list */ if (NULL == el->tail) el->head = el->tail = ev; @@ -133,7 +135,10 @@ H5ES__list_count(const H5ES_event_list_t *el) * each event. * * Note: Iteration is safe for deleting the current event. Modifying - * the list in other ways is likely unsafe. + * the list in other ways is likely unsafe. If order is + * H5_ITER_INC or H5_ITER_NATIVE events are visited starting + * with the oldest, otherwise they are visited starting with + * the newest. * * Return: SUCCEED / FAIL * @@ -143,7 +148,7 @@ H5ES__list_count(const H5ES_event_list_t *el) *------------------------------------------------------------------------- */ int -H5ES__list_iterate(H5ES_event_list_t *el, H5ES_list_iter_func_t cb, void *ctx) +H5ES__list_iterate(H5ES_event_list_t *el, H5_iter_order_t order, H5ES_list_iter_func_t cb, void *ctx) { H5ES_event_t *ev; /* Event in list */ int ret_value = H5_ITER_CONT; /* Return value */ @@ -155,12 +160,12 @@ H5ES__list_iterate(H5ES_event_list_t *el, H5ES_list_iter_func_t cb, void *ctx) HDassert(cb); /* Iterate over events in list */ - ev = el->head; + ev = (order == H5_ITER_DEC) ? el->tail : el->head; while (ev) { H5ES_event_t *tmp; /* Temporary event */ /* Get pointer to next node, so it's safe if this one is removed */ - tmp = ev->next; + tmp = (order == H5_ITER_DEC) ? ev->prev : ev->next; /* Perform iterator callback */ if ((ret_value = (*cb)(ev, ctx)) != H5_ITER_CONT) { diff --git a/src/H5ESpkg.h b/src/H5ESpkg.h index a7a8e20..6ee50fa 100644 --- a/src/H5ESpkg.h +++ b/src/H5ESpkg.h @@ -81,6 +81,8 @@ typedef int (*H5ES_list_iter_func_t)(H5ES_event_t *ev, void *ctx); H5_DLL H5ES_t *H5ES__create(void); H5_DLL herr_t H5ES__insert_request(H5ES_t *es, H5VL_t *connector, void *token); H5_DLL herr_t H5ES__wait(H5ES_t *es, uint64_t timeout, size_t *num_in_progress, hbool_t *op_failed); +H5_DLL herr_t H5ES__get_requests(H5ES_t *es, H5_iter_order_t order, hid_t *connector_ids, void **requests, + size_t array_len); H5_DLL herr_t H5ES__cancel(H5ES_t *es, size_t *num_not_canceled, hbool_t *op_failed); H5_DLL herr_t H5ES__get_err_info(H5ES_t *es, size_t num_err_info, H5ES_err_info_t err_info[], size_t *num_cleared); @@ -88,7 +90,8 @@ H5_DLL herr_t H5ES__get_err_info(H5ES_t *es, size_t num_err_info, H5ES_err_info /* Event list operations */ H5_DLL void H5ES__list_append(H5ES_event_list_t *el, H5ES_event_t *ev); H5_DLL size_t H5ES__list_count(const H5ES_event_list_t *el); -H5_DLL int H5ES__list_iterate(H5ES_event_list_t *el, H5ES_list_iter_func_t cb, void *ctx); +H5_DLL int H5ES__list_iterate(H5ES_event_list_t *el, H5_iter_order_t order, H5ES_list_iter_func_t cb, + void *ctx); H5_DLL void H5ES__list_remove(H5ES_event_list_t *el, const H5ES_event_t *ev); /* Event operations */ diff --git a/src/H5ESprivate.h b/src/H5ESprivate.h index 3d9ce9f..153e3e3 100644 --- a/src/H5ESprivate.h +++ b/src/H5ESprivate.h @@ -51,5 +51,6 @@ typedef struct H5ES_t H5ES_t; /***************************************/ herr_t H5ES_insert(hid_t es_id, H5VL_t *connector, void *token, const char *caller, const char *caller_args, ...); +H5_DLL herr_t H5ES_init(void); #endif /* H5ESprivate_H */ diff --git a/src/H5ESpublic.h b/src/H5ESpublic.h index c8696b3..c8d1c7b 100644 --- a/src/H5ESpublic.h +++ b/src/H5ESpublic.h @@ -200,7 +200,18 @@ H5_DLL herr_t H5ESget_count(hid_t es_id, size_t *count); /** * \ingroup H5ES * - * \todo Fill in the blanks! + * \brief Retrieves the next operation counter to be assigned in an event set + * + * \es_id + * \param[out] counter The next counter value to be assigned to an event + * \returns \herr_t + * + * \details H5ESget_op_counter() retrieves the \p counter that will be assigned + * to the next operation inserted into the event set \p es_id. + * + * \note This is designed for wrapper libraries mainly, to use as a mechanism + * for matching operations inserted into the event set with possible + * errors that occur. * * \since 1.13.0 * diff --git a/src/H5Eint.c b/src/H5Eint.c index 6438cd9..d744db3 100644 --- a/src/H5Eint.c +++ b/src/H5Eint.c @@ -930,7 +930,7 @@ H5E_dump_api_stack(hbool_t is_api) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Only dump the error stack during an API call */ if (is_api) { @@ -953,6 +953,5 @@ H5E_dump_api_stack(hbool_t is_api) #endif /* H5_NO_DEPRECATED_SYMBOLS */ } /* end if */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5E_dump_api_stack() */ diff --git a/src/H5Epkg.h b/src/H5Epkg.h index 30ff084..b11ee9e 100644 --- a/src/H5Epkg.h +++ b/src/H5Epkg.h @@ -48,7 +48,7 @@ * each thread individually. The association of stacks to threads will * be handled by the pthread library. * - * In order for this macro to work, H5E__get_my_stack() must be preceeded + * In order for this macro to work, H5E__get_my_stack() must be preceded * by "H5E_t *estack =". */ #define H5E__get_my_stack() H5E__get_stack() diff --git a/src/H5F.c b/src/H5F.c index 46c8c0e..c54e759 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -1306,7 +1306,7 @@ done: /*------------------------------------------------------------------------- * Function: H5Funmount * - * Purpose: Given a mount point, dissassociate the mount point's file + * Purpose: Given a mount point, disassociate the mount point's file * from the file mounted there. Do not close either file. * * The mount point can either be the group in the parent or the @@ -2262,7 +2262,7 @@ done: * 1) The file being opened has v3 superblock * 2) The file is opened with H5F_ACC_RDWR * 3) The file is not already marked for SWMR writing - * 4) Current implementaion for opened objects: + * 4) Current implementation for opened objects: * --only allow datasets and groups without attributes * --disallow named datatype with/without attributes * --disallow opened attributes attached to objects diff --git a/src/H5FA.c b/src/H5FA.c index ad69ee2..3e81080 100644 --- a/src/H5FA.c +++ b/src/H5FA.c @@ -64,9 +64,6 @@ static H5FA_t *H5FA__new(H5F_t *f, haddr_t fa_addr, hbool_t from_open, void *ctx /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Fixed array client ID to class mapping */ /* Remember to add client ID to H5FA_cls_id_t in H5FAprivate.h when adding a new @@ -95,7 +92,7 @@ H5FL_BLK_DEFINE(fa_native_elmt); /*------------------------------------------------------------------------- * Function: H5FA__new * - * Purpose: Allocate and initialize a new fixe array wrapper in memory + * Purpose: Allocate and initialize a new fixed array wrapper in memory * * Return: Pointer to farray wrapper success * NULL on failure diff --git a/src/H5FAcache.c b/src/H5FAcache.c index 6975d3d..05a8762 100644 --- a/src/H5FAcache.c +++ b/src/H5FAcache.c @@ -946,7 +946,7 @@ done: * file space block set. * * This function is needed when the data block is paged, as - * the datablock header and all its pages are allocted as a + * the datablock header and all its pages are allocated as a * single contiguous chunk of file space, and must be * deallocated the same way. * diff --git a/src/H5FAdblkpage.c b/src/H5FAdblkpage.c index 713bd67..f6a5aef 100644 --- a/src/H5FAdblkpage.c +++ b/src/H5FAdblkpage.c @@ -147,7 +147,7 @@ H5FA__dblk_page_create(H5FA_hdr_t *hdr, haddr_t addr, size_t nelmts) FUNC_ENTER_PACKAGE #ifdef H5FA_DEBUG - HDfprintf(stderr, "%s: Called, addr = %a\n", __func__, addr); + HDfprintf(stderr, "%s: Called, addr = %" PRIuHADDR "\n", __func__, addr); #endif /* H5FA_DEBUG */ /* Sanity check */ diff --git a/src/H5FApkg.h b/src/H5FApkg.h index c4bf934..15f6445 100644 --- a/src/H5FApkg.h +++ b/src/H5FApkg.h @@ -179,8 +179,8 @@ typedef struct H5FA_dblock_t { /* Computed/cached values (not stored) */ haddr_t addr; /* Address of this data block on disk */ hsize_t size; /* Size of data block on disk */ - size_t npages; /* Nummber of pages in data block (zero if not paged) */ - size_t last_page_nelmts; /* Nummber of elements in last page, if paged */ + size_t npages; /* Number of pages in data block (zero if not paged) */ + size_t last_page_nelmts; /* Number of elements in last page, if paged */ /* Fixed Array data block information (not stored) */ size_t dblk_page_nelmts; /* # of elements per data block page */ diff --git a/src/H5FAprivate.h b/src/H5FAprivate.h index 26057bf..59d3652 100644 --- a/src/H5FAprivate.h +++ b/src/H5FAprivate.h @@ -134,7 +134,7 @@ H5_DLL herr_t H5FA_patch_file(H5FA_t *fa, H5F_t *f); H5_DLL herr_t H5FA_get_stats(const H5FA_t *ea, H5FA_stat_t *stats); /* Debugging routines */ -#ifdef H5FA_DEBUGGING -#endif /* H5FA_DEBUGGING */ +#ifdef H5FA_DEBUG +#endif /* H5FA_DEBUG */ #endif /* H5FAprivate_H */ diff --git a/src/H5FAtest.c b/src/H5FAtest.c index 384a657..b57f562 100644 --- a/src/H5FAtest.c +++ b/src/H5FAtest.c @@ -303,7 +303,7 @@ H5FA__test_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const void * HDassert(elmt); /* Print element */ - HDsprintf(temp_str, "Element #%llu:", (unsigned long long)idx); + HDsnprintf(temp_str, sizeof(temp_str), "Element #%llu:", (unsigned long long)idx); HDfprintf(stream, "%*s%-*s %llu\n", indent, "", fwidth, temp_str, (unsigned long long)*(const uint64_t *)elmt); diff --git a/src/H5FD.c b/src/H5FD.c index 2cae1a8..091a280 100644 --- a/src/H5FD.c +++ b/src/H5FD.c @@ -67,9 +67,6 @@ static herr_t H5FD__query(const H5FD_t *f, unsigned long *flags /*out*/); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -103,20 +100,20 @@ static const H5I_class_t H5I_VFL_CLS[1] = {{ }}; /*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initialize the virtual file layer. + * Function: H5FD_init * - * Return: SUCCEED/FAIL + * Purpose: Initialize the interface from some other layer. * + * Return: Success: non-negative + * Failure: negative *------------------------------------------------------------------------- */ herr_t -H5FD__init_package(void) +H5FD_init(void) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_NOAPI(FAIL) if (H5I_register_type(H5I_VFL_CLS) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize interface") @@ -126,7 +123,7 @@ H5FD__init_package(void) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD__init_package() */ +} /*------------------------------------------------------------------------- * Function: H5FD_term_package @@ -150,20 +147,14 @@ H5FD_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - if (H5I_nmembers(H5I_VFL) > 0) { - (void)H5I_clear_type(H5I_VFL, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - else { - /* Destroy the VFL driver ID group */ - n += (H5I_dec_type_ref(H5I_VFL) > 0); - - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ + if (H5I_nmembers(H5I_VFL) > 0) { + (void)H5I_clear_type(H5I_VFL, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + else { + /* Destroy the VFL driver ID group */ + n += (H5I_dec_type_ref(H5I_VFL) > 0); + } /* end else */ FUNC_LEAVE_NOAPI(n) } /* end H5FD_term_package() */ @@ -173,7 +164,7 @@ H5FD_term_package(void) * * Purpose: Frees a file driver class struct and returns an indication of * success. This function is used as the free callback for the - * virtual file layer object identifiers (cf H5FD__init_package). + * virtual file layer object identifiers (cf H5FD_init). * * Return: SUCCEED/FAIL * @@ -309,6 +300,62 @@ done: } /* end H5FD_register() */ /*------------------------------------------------------------------------- + * Function: H5FDis_driver_registered_by_name + * + * Purpose: Tests whether a VFD class has been registered or not + * according to a supplied driver name. + * + * Return: >0 if a VFD with that name has been registered + * 0 if a VFD with that name has NOT been registered + * <0 on errors + * + *------------------------------------------------------------------------- + */ +htri_t +H5FDis_driver_registered_by_name(const char *driver_name) +{ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("t", "*s", driver_name); + + /* Check if driver with this name is registered */ + if ((ret_value = H5FD_is_driver_registered_by_name(driver_name, NULL)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't check if VFD is registered") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5FDis_driver_registered_by_name() */ + +/*------------------------------------------------------------------------- + * Function: H5FDis_driver_registered_by_value + * + * Purpose: Tests whether a VFD class has been registered or not + * according to a supplied driver value (ID). + * + * Return: >0 if a VFD with that value has been registered + * 0 if a VFD with that value hasn't been registered + * <0 on errors + * + *------------------------------------------------------------------------- + */ +htri_t +H5FDis_driver_registered_by_value(H5FD_class_value_t driver_value) +{ + htri_t ret_value = FALSE; + + FUNC_ENTER_API(FAIL) + H5TRACE1("t", "DV", driver_value); + + /* Check if driver with this value is registered */ + if ((ret_value = H5FD_is_driver_registered_by_value(driver_value, NULL)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't check if VFD is registered") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5FDis_driver_registered_by_value() */ + +/*------------------------------------------------------------------------- * Function: H5FDunregister * * Purpose: Removes a driver ID from the library. This in no way affects @@ -406,7 +453,7 @@ H5FD_sb_size(H5FD_t *file) { hsize_t ret_value = 0; - FUNC_ENTER_NOAPI(0) + FUNC_ENTER_NOAPI_NOERR /* Sanity checks */ HDassert(file); @@ -416,7 +463,6 @@ H5FD_sb_size(H5FD_t *file) if (file->cls->sb_size) ret_value = (file->cls->sb_size)(file); -done: FUNC_LEAVE_NOAPI(ret_value) } @@ -544,7 +590,7 @@ H5FD_fapl_get(H5FD_t *file) { void *ret_value = NULL; - FUNC_ENTER_NOAPI(NULL) + FUNC_ENTER_NOAPI_NOERR /* Sanity checks */ HDassert(file); @@ -554,7 +600,6 @@ H5FD_fapl_get(H5FD_t *file) if (file->cls->fapl_get) ret_value = (file->cls->fapl_get)(file); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_fapl_get() */ @@ -826,9 +871,6 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) /* Dispatch to file driver */ if (HADDR_UNDEF == maxaddr) maxaddr = driver->maxaddr; -#if 0 /* JRM */ - HDfprintf(stderr, "H5FD_open(): calling %s.open().\n", driver->name); -#endif /* JRM */ if (NULL == (file = (driver->open)(name, flags, fapl_id, maxaddr))) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed") @@ -875,7 +917,6 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) ret_value = file; done: - /* XXX We leak H5FD_t's on many error conditions. */ /* Can't cleanup 'file' information, since we don't know what type it is */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_open() */ @@ -1015,10 +1056,9 @@ H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2) { int ret_value = -1; /* Return value */ - FUNC_ENTER_NOAPI(-1) /* return value is arbitrary */ + FUNC_ENTER_NOAPI_NOERR /* return value is arbitrary */ - if ((!f1 || !f1->cls) && (!f2 || !f2->cls)) - HGOTO_DONE(0) + if ((!f1 || !f1->cls) && (!f2 || !f2->cls)) HGOTO_DONE(0) if (!f1 || !f1->cls) HGOTO_DONE(-1) if (!f2 || !f2->cls) @@ -1376,7 +1416,7 @@ H5FD_get_maxaddr(const H5FD_t *file) { haddr_t ret_value = HADDR_UNDEF; /* Return value */ - FUNC_ENTER_NOAPI(HADDR_UNDEF) + FUNC_ENTER_NOAPI_NOERR /* Sanity checks */ HDassert(file); @@ -1384,7 +1424,6 @@ H5FD_get_maxaddr(const H5FD_t *file) /* Set return value */ ret_value = file->maxaddr; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_get_maxaddr() */ @@ -1456,7 +1495,7 @@ H5FD_get_fs_type_map(const H5FD_t *file, H5FD_mem_t *type_map) HDassert(file->cls); HDassert(type_map); - /* Check for VFD class providing a type map retrieval rouine */ + /* Check for VFD class providing a type map retrieval routine */ if (file->cls->get_type_map) { /* Retrieve type mapping for this file */ if ((file->cls->get_type_map)(file, type_map) < 0) @@ -1820,6 +1859,111 @@ done: } /* end H5FD_unlock() */ /*------------------------------------------------------------------------- + * Function: H5FDctl + * + * Purpose: Perform a CTL operation. + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: JRM -- 8/3/21 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FDctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "*#ULUL*x**x", file, op_code, flags, input, output); + + /* Check arguments */ + if (!file) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file pointer cannot be NULL") + + if (!file->cls) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file class pointer cannot be NULL") + + /* Don't attempt to validate the op code. If appropriate, that will + * be done by the underlying VFD callback, along with the input and + * output parameters. + */ + + /* Call private function */ + if (H5FD_ctl(file, op_code, flags, input, output) < 0) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed") + +done: + + FUNC_LEAVE_API(ret_value) + +} /* end H5FDctl() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_ctl + * + * Purpose: Private version of H5FDctl() + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: JRM -- 8/3/21 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FD_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(file); + HDassert(file->cls); + + /* Dispatch to driver if the ctl function exists. + * + * If it doesn't, fail if the H5FD_CTL__FAIL_IF_UNKNOWN_FLAG is set. + * + * Otherwise, report success. + */ + if (file->cls->ctl) { + + if ((file->cls->ctl)(file, op_code, flags, input, output) < 0) + + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed") + } + else if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) { + + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, + "VFD ctl request failed (no ctl callback and fail if unknown flag is set)") + } + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ctl() */ + +/*------------------------------------------------------------------------- * Function: H5FD_get_fileno * * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value diff --git a/src/H5FDcore.c b/src/H5FDcore.c index 50288c4..a1750ee 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -149,8 +149,10 @@ static herr_t H5FD__core_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing static herr_t H5FD__core_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD__core_unlock(H5FD_t *_file); static herr_t H5FD__core_delete(const char *filename, hid_t fapl_id); +static inline const H5FD_core_fapl_t *H5FD__core_get_default_config(void); static const H5FD_class_t H5FD_core_g = { + H5FD_CORE_VALUE, /* value */ "core", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -183,9 +185,15 @@ static const H5FD_class_t H5FD_core_g = { H5FD__core_lock, /* lock */ H5FD__core_unlock, /* unlock */ H5FD__core_delete, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; +/* Default configurations, if none provided */ +static const H5FD_core_fapl_t H5FD_core_default_config_g = { + (size_t)H5_MB, TRUE, H5FD_CORE_WRITE_TRACKING_FLAG, H5FD_CORE_WRITE_TRACKING_PAGE_SIZE}; +static const H5FD_core_fapl_t H5FD_core_default_paged_config_g = {(size_t)H5_MB, TRUE, TRUE, (size_t)4096}; + /* Define a free list to manage the region type */ H5FL_DEFINE(H5FD_core_region_t); @@ -390,7 +398,7 @@ H5FD__core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size) "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, " "errno = %d, error message = '%s', ptr = %p, total write size = %llu, bytes this " "sub-write = %llu, bytes actually written = %llu, offset = %llu", - HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, + HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), (void *)ptr, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_wrote, (unsigned long long)offset); } /* end if */ @@ -408,37 +416,30 @@ done: } /* end H5FD__core_write_to_bstore() */ /*------------------------------------------------------------------------- - * Function: H5FD__init_package + * Function: H5FD__core_get_default_config * - * Purpose: Initializes any interface-specific data or routines. + * Purpose: Retrieves a default configuration for this VFD when no + * configuration information has been provided. * - * Return: Non-negative on success/Negative on failure + * Return: Valid Core VFD configuration information pointer (can't + * fail) * *------------------------------------------------------------------------- */ -static herr_t -H5FD__init_package(void) +static inline const H5FD_core_fapl_t * +H5FD__core_get_default_config(void) { - char * lock_env_var = NULL; /* Environment variable pointer */ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC + char *driver = HDgetenv(HDF5_DRIVER); - /* Check the use disabled file locks environment variable */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) - ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ - else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) - ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ - else - ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ - - if (H5FD_core_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize core VFD") + if (driver) { + if (!HDstrcmp(driver, "core")) + return &H5FD_core_default_config_g; + else if (!HDstrcmp(driver, "core_paged")) + return &H5FD_core_default_paged_config_g; + } -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ + return &H5FD_core_default_config_g; +} /* end H5FD__core_get_default_config() */ /*------------------------------------------------------------------------- * Function: H5FD_core_init @@ -457,9 +458,19 @@ done: hid_t H5FD_core_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + char *lock_env_var = NULL; /* Environment variable pointer */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ - FUNC_ENTER_NOAPI(H5I_INVALID_HID) + FUNC_ENTER_NOAPI_NOERR + + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv(HDF5_USE_FILE_LOCKING); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ if (H5I_VFL != H5I_get_type(H5FD_CORE_g)) H5FD_CORE_g = H5FD_register(&H5FD_core_g, sizeof(H5FD_class_t), FALSE); @@ -467,7 +478,6 @@ H5FD_core_init(void) /* Set return value */ ret_value = H5FD_CORE_g; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_core_init() */ @@ -528,7 +538,7 @@ H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size) if (H5FD_CORE != H5P_peek_driver(plist)) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") if (NULL == (old_fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") + old_fa = H5FD__core_get_default_config(); /* Set VFD info values */ HDmemset(&fa, 0, sizeof(H5FD_core_fapl_t)); @@ -538,7 +548,7 @@ H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size) fa.page_size = page_size; /* Set the property values & the driver for the FAPL */ - if (H5P_set_driver(plist, H5FD_CORE, &fa) < 0) + if (H5P_set_driver(plist, H5FD_CORE, &fa, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD as driver") done: @@ -622,7 +632,7 @@ H5Pset_fapl_core(hid_t fapl_id, size_t increment, hbool_t backing_store) fa.page_size = H5FD_CORE_WRITE_TRACKING_PAGE_SIZE; /* Set the property values & the driver for the FAPL */ - if (H5P_set_driver(plist, H5FD_CORE, &fa) < 0) + if (H5P_set_driver(plist, H5FD_CORE, &fa, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD as driver") done: @@ -747,7 +757,7 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") if (NULL == (fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") + fa = H5FD__core_get_default_config(); /* Build the open flags */ o_flags = (H5F_ACC_RDWR & flags) ? O_RDWR : O_RDONLY; @@ -918,8 +928,8 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr "file read failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, " "error message = '%s', file->mem = %p, total read size = %llu, bytes this " "sub-read = %llu, bytes actually read = %llu, offset = %llu", - HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), file->mem, - (unsigned long long)size, (unsigned long long)bytes_in, + HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), + (void *)file->mem, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_read, (unsigned long long)offset); } /* end if */ @@ -1512,7 +1522,7 @@ done: * Addendum -- 12/2/11 * For file images opened with the core file driver, it is * necessary that we avoid reallocating the core file driver's - * buffer uneccessarily. + * buffer unnecessarily. * * To this end, I have made the following functional changes * to this function. @@ -1734,7 +1744,7 @@ H5FD__core_delete(const char *filename, hid_t fapl_id) if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") if (NULL == (fa = (const H5FD_core_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") + fa = H5FD__core_get_default_config(); if (fa->backing_store) if (HDremove(filename) < 0) diff --git a/src/H5FDcore.h b/src/H5FDcore.h index d456c3e..4067980 100644 --- a/src/H5FDcore.h +++ b/src/H5FDcore.h @@ -20,7 +20,8 @@ #ifndef H5FDcore_H #define H5FDcore_H -#define H5FD_CORE (H5FD_core_init()) +#define H5FD_CORE (H5FDperform_init(H5FD_core_init)) +#define H5FD_CORE_VALUE H5_VFD_CORE #ifdef __cplusplus extern "C" { diff --git a/src/H5FDdevelop.h b/src/H5FDdevelop.h index a723a1d..fd62ad7 100644 --- a/src/H5FDdevelop.h +++ b/src/H5FDdevelop.h @@ -160,6 +160,7 @@ typedef struct H5FD_t H5FD_t; /* Class information for each file driver */ typedef struct H5FD_class_t { + H5FD_class_value_t value; const char * name; haddr_t maxaddr; H5F_close_degree_t fc_degree; @@ -192,6 +193,7 @@ typedef struct H5FD_class_t { herr_t (*lock)(H5FD_t *file, hbool_t rw); herr_t (*unlock)(H5FD_t *file); herr_t (*del)(const char *name, hid_t fapl); + herr_t (*ctl)(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output); H5FD_mem_t fl_map[H5FD_MEM_NTYPES]; } H5FD_class_t; @@ -237,7 +239,10 @@ struct H5FD_t { extern "C" { #endif +H5_DLL hid_t H5FDperform_init(hid_t (*)(void)); H5_DLL hid_t H5FDregister(const H5FD_class_t *cls); +H5_DLL htri_t H5FDis_driver_registered_by_name(const char *driver_name); +H5_DLL htri_t H5FDis_driver_registered_by_value(H5FD_class_value_t driver_value); H5_DLL herr_t H5FDunregister(hid_t driver_id); H5_DLL H5FD_t *H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); H5_DLL herr_t H5FDclose(H5FD_t *file); @@ -258,6 +263,7 @@ H5_DLL herr_t H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); H5_DLL herr_t H5FDlock(H5FD_t *file, hbool_t rw); H5_DLL herr_t H5FDunlock(H5FD_t *file); H5_DLL herr_t H5FDdelete(const char *name, hid_t fapl_id); +H5_DLL herr_t H5FDctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output); #ifdef __cplusplus } diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index a1b7b7e..7a43117 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -120,6 +120,8 @@ typedef struct H5FD_direct_t { /* Prototypes */ static herr_t H5FD__direct_term(void); +static herr_t H5FD__direct_populate_config(size_t boundary, size_t block_size, size_t cbuf_size, + H5FD_direct_fapl_t *fa_out); static void * H5FD__direct_fapl_get(H5FD_t *file); static void * H5FD__direct_fapl_copy(const void *_old_fa); static H5FD_t *H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); @@ -140,6 +142,7 @@ static herr_t H5FD__direct_unlock(H5FD_t *_file); static herr_t H5FD__direct_delete(const char *filename, hid_t fapl_id); static const H5FD_class_t H5FD_direct_g = { + H5FD_DIRECT_VALUE, /* value */ "direct", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -172,48 +175,13 @@ static const H5FD_class_t H5FD_direct_g = { H5FD__direct_lock, /* lock */ H5FD__direct_unlock, /* unlock */ H5FD__direct_delete, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; /* Declare a free list to manage the H5FD_direct_t struct */ H5FL_DEFINE_STATIC(H5FD_direct_t); -/*-------------------------------------------------------------------------- -NAME - H5FD__init_package -- Initialize interface-specific information -USAGE - herr_t H5FD__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. (Just calls - H5FD_direct_init currently). - ---------------------------------------------------------------------------*/ -static herr_t -H5FD__init_package(void) -{ - char * lock_env_var = NULL; /* Environment variable pointer */ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - /* Check the use disabled file locks environment variable */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) - ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ - else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) - ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ - else - ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ - - if (H5FD_direct_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize direct VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - /*------------------------------------------------------------------------- * Function: H5FD_direct_init * @@ -231,12 +199,25 @@ done: hid_t H5FD_direct_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + char *lock_env_var = NULL; /* Environment variable pointer */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ FUNC_ENTER_NOAPI(H5I_INVALID_HID) - if (H5I_VFL != H5I_get_type(H5FD_DIRECT_g)) + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv(HDF5_USE_FILE_LOCKING); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + + if (H5I_VFL != H5I_get_type(H5FD_DIRECT_g)) { H5FD_DIRECT_g = H5FD_register(&H5FD_direct_g, sizeof(H5FD_class_t), FALSE); + if (H5I_INVALID_HID == H5FD_DIRECT_g) + HGOTO_ERROR(H5E_ID, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to register direct"); + } /* Set return value */ ret_value = H5FD_DIRECT_g; @@ -295,28 +276,10 @@ H5Pset_fapl_direct(hid_t fapl_id, size_t boundary, size_t block_size, size_t cbu if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - HDmemset(&fa, 0, sizeof(H5FD_direct_fapl_t)); - if (boundary != 0) - fa.mboundary = boundary; - else - fa.mboundary = MBOUNDARY_DEF; - if (block_size != 0) - fa.fbsize = block_size; - else - fa.fbsize = FBSIZE_DEF; - if (cbuf_size != 0) - fa.cbsize = cbuf_size; - else - fa.cbsize = CBSIZE_DEF; - - /* Set the default to be true for data alignment */ - fa.must_align = TRUE; - - /* Copy buffer size must be a multiple of file block size */ - if (fa.cbsize % fa.fbsize != 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "copy buffer size must be a multiple of block size") + if (H5FD__direct_populate_config(boundary, block_size, cbuf_size, &fa) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't initialize driver configuration info") - ret_value = H5P_set_driver(plist, H5FD_DIRECT, &fa); + ret_value = H5P_set_driver(plist, H5FD_DIRECT, &fa, NULL); done: FUNC_LEAVE_API(ret_value) @@ -366,6 +329,53 @@ done: } /* end H5Pget_fapl_direct() */ /*------------------------------------------------------------------------- + * Function: H5FD__direct_populate_config + * + * Purpose: Populates a H5FD_direct_fapl_t structure with the provided + * values, supplying defaults where values are not provided. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__direct_populate_config(size_t boundary, size_t block_size, size_t cbuf_size, H5FD_direct_fapl_t *fa_out) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(fa_out); + + HDmemset(fa_out, 0, sizeof(H5FD_direct_fapl_t)); + + if (boundary != 0) + fa_out->mboundary = boundary; + else + fa_out->mboundary = MBOUNDARY_DEF; + + if (block_size != 0) + fa_out->fbsize = block_size; + else + fa_out->fbsize = FBSIZE_DEF; + + if (cbuf_size != 0) + fa_out->cbsize = cbuf_size; + else + fa_out->cbsize = CBSIZE_DEF; + + /* Set the default to be true for data alignment */ + fa_out->must_align = TRUE; + + /* Copy buffer size must be a multiple of file block size */ + if (fa_out->cbsize % fa_out->fbsize != 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "copy buffer size must be a multiple of block size") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__direct_populate_config() */ + +/*------------------------------------------------------------------------- * Function: H5FD__direct_fapl_get * * Purpose: Returns a file access property list which indicates how the @@ -449,6 +459,7 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad int fd = (-1); H5FD_direct_t * file = NULL; const H5FD_direct_fapl_t *fa; + H5FD_direct_fapl_t default_fa; #ifdef H5_HAVE_WIN32_API HFILE filehandle; struct _BY_HANDLE_FILE_INFORMATION fileinfo; @@ -497,8 +508,11 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad /* Get the driver specific information */ if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") + if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) { + if (H5FD__direct_populate_config(0, 0, 0, &default_fa) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, NULL, "can't initialize driver configuration info") + fa = &default_fa; + } file->fd = fd; H5_CHECKED_ASSIGN(file->eof, haddr_t, sb.st_size, h5_stat_size_t); diff --git a/src/H5FDdirect.h b/src/H5FDdirect.h index f06de7f..bfad16a 100644 --- a/src/H5FDdirect.h +++ b/src/H5FDdirect.h @@ -21,9 +21,11 @@ #define H5FDdirect_H #ifdef H5_HAVE_DIRECT -#define H5FD_DIRECT (H5FD_direct_init()) +#define H5FD_DIRECT (H5FDperform_init(H5FD_direct_init)) +#define H5FD_DIRECT_VALUE H5_VFD_DIRECT #else -#define H5FD_DIRECT (H5I_INVALID_HID) +#define H5FD_DIRECT (H5I_INVALID_HID) +#define H5FD_DIRECT_VALUE H5_VFD_INVALID #endif /* H5_HAVE_DIRECT */ #ifdef H5_HAVE_DIRECT diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index 8cf9f9e..4e54197 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -47,6 +47,9 @@ /* The size of the member name buffers */ #define H5FD_FAM_MEMB_NAME_BUF_SIZE 4096 +/* Default member size - 100 MiB */ +#define H5FD_FAM_DEF_MEM_SIZE ((hsize_t)(100 * H5_MB)) + /* The driver identification number, initialized at runtime */ static hid_t H5FD_FAMILY_g = 0; @@ -77,6 +80,10 @@ typedef struct H5FD_family_fapl_t { hid_t memb_fapl_id; /*file access property list of each memb*/ } H5FD_family_fapl_t; +/* Private routines */ +static herr_t H5FD__family_get_default_config(H5FD_family_fapl_t *fa_out); +static char * H5FD__family_get_default_printf_filename(const char *old_filename); + /* Callback prototypes */ static herr_t H5FD__family_term(void); static void * H5FD__family_fapl_get(H5FD_t *_file); @@ -105,6 +112,7 @@ static herr_t H5FD__family_delete(const char *filename, hid_t fapl_id); /* The class struct */ static const H5FD_class_t H5FD_family_g = { + H5FD_FAMILY_VALUE, /* value */ "family", /* name */ HADDR_MAX, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -137,34 +145,125 @@ static const H5FD_class_t H5FD_family_g = { H5FD__family_lock, /* lock */ H5FD__family_unlock, /* unlock */ H5FD__family_delete, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; -/*-------------------------------------------------------------------------- -NAME - H5FD__init_package -- Initialize interface-specific information -USAGE - herr_t H5FD__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. (Just calls - H5FD_family_init currently). - ---------------------------------------------------------------------------*/ +/*------------------------------------------------------------------------- + * Function: H5FD__family_get_default_config + * + * Purpose: Populates a H5FD_family_fapl_t structure with default + * values. + * + * Return: Non-negative on Success/Negative on Failure + * + *------------------------------------------------------------------------- + */ static herr_t -H5FD__init_package(void) +H5FD__family_get_default_config(H5FD_family_fapl_t *fa_out) +{ + H5P_genplist_t *def_plist; + H5P_genplist_t *plist; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(fa_out); + + fa_out->memb_size = H5FD_FAM_DEF_MEM_SIZE; + + /* Use copy of default file access property list for member FAPL ID. + * The Sec2 driver is explicitly set on the member FAPL ID, as the + * default driver might have been replaced with the Family VFD, which + * would cause recursion badness in the child members. + */ + if (NULL == (def_plist = (H5P_genplist_t *)H5I_object(H5P_FILE_ACCESS_DEFAULT))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if ((fa_out->memb_fapl_id = H5P_copy_plist(def_plist, FALSE)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "can't copy property list") + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fa_out->memb_fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5P_set_driver_by_value(plist, H5_VFD_SEC2, NULL, TRUE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't set default driver on member FAPL") + +done: + if (ret_value < 0 && fa_out->memb_fapl_id >= 0) { + if (H5I_dec_ref(fa_out->memb_fapl_id) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't decrement ref. count on member FAPL ID") + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__family_get_default_config() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__family_get_default_printf_filename + * + * Purpose: Given a filename, allocates and returns a new filename + * buffer that contains the given filename modified into this + * VFD's printf-style format. For example, the filename + * "file1.h5" would be modified to "file1-%06d.h5". This would + * allow for member filenames such as "file1-000000.h5", + * "file1-000001.h5", etc. The caller is responsible for + * freeing the returned buffer. + * + * Return: Non-negative on Success/Negative on Failure + * + *------------------------------------------------------------------------- + */ +static char * +H5FD__family_get_default_printf_filename(const char *old_filename) { - herr_t ret_value = SUCCEED; + const char *suffix = "-%06d"; + size_t old_filename_len = 0; + size_t new_filename_len = 0; + char * file_extension = NULL; + char * tmp_buffer = NULL; + char * ret_value = NULL; FUNC_ENTER_STATIC - if (H5FD_family_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize family VFD") + HDassert(old_filename); + + old_filename_len = HDstrlen(old_filename); + if (0 == old_filename_len) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid filename") + + new_filename_len = old_filename_len + HDstrlen(suffix) + 1; + if (NULL == (tmp_buffer = H5MM_malloc(new_filename_len))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "can't allocate new filename buffer") + + /* Determine if filename contains a ".h5" extension. */ + if ((file_extension = strstr(old_filename, ".h5"))) { + /* Insert the printf format between the filename and ".h5" extension. */ + HDstrcpy(tmp_buffer, old_filename); + file_extension = strstr(tmp_buffer, ".h5"); + HDsprintf(file_extension, "%s%s", suffix, ".h5"); + } + else if ((file_extension = strrchr(old_filename, '.'))) { + char *new_extension_loc = NULL; + + /* If the filename doesn't contain a ".h5" extension, but contains + * AN extension, just insert the printf format before that extension. + */ + HDstrcpy(tmp_buffer, old_filename); + new_extension_loc = strrchr(tmp_buffer, '.'); + HDsprintf(new_extension_loc, "%s%s", suffix, file_extension); + } + else { + /* If the filename doesn't contain an extension at all, just insert + * the printf format at the end of the filename. + */ + HDsnprintf(tmp_buffer, new_filename_len, "%s%s", old_filename, suffix); + } + + ret_value = tmp_buffer; done: + if (!ret_value) + H5MM_xfree(tmp_buffer); + FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ +} /* end H5FD__family_get_default_printf_filename() */ /*------------------------------------------------------------------------- * Function: H5FD_family_init @@ -183,9 +282,9 @@ done: hid_t H5FD_family_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + hid_t ret_value = H5I_INVALID_HID; - FUNC_ENTER_NOAPI(H5I_INVALID_HID) + FUNC_ENTER_NOAPI_NOERR if (H5I_VFL != H5I_get_type(H5FD_FAMILY_g)) H5FD_FAMILY_g = H5FD_register(&H5FD_family_g, sizeof(H5FD_class_t), FALSE); @@ -193,7 +292,6 @@ H5FD_family_init(void) /* Set return value */ ret_value = H5FD_FAMILY_g; -done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_family_init() */ @@ -242,7 +340,7 @@ herr_t H5Pset_fapl_family(hid_t fapl_id, hsize_t msize, hid_t memb_fapl_id) { herr_t ret_value; - H5FD_family_fapl_t fa = {0, -1}; + H5FD_family_fapl_t fa = {0, H5I_INVALID_HID}; H5P_genplist_t * plist; /* Property list pointer */ FUNC_ENTER_API(FAIL) @@ -251,18 +349,22 @@ H5Pset_fapl_family(hid_t fapl_id, hsize_t msize, hid_t memb_fapl_id) /* Check arguments */ if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - if (H5P_DEFAULT == memb_fapl_id) - memb_fapl_id = H5P_FILE_ACCESS_DEFAULT; + if (H5P_DEFAULT == memb_fapl_id) { + /* Get default configuration for member FAPL */ + if (H5FD__family_get_default_config(&fa) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get default driver configuration info") + } else if (TRUE != H5P_isa_class(memb_fapl_id, H5P_FILE_ACCESS)) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") /* Initialize driver specific information. */ - fa.memb_size = msize; - fa.memb_fapl_id = memb_fapl_id; + fa.memb_size = msize; + if (H5P_DEFAULT != memb_fapl_id) + fa.memb_fapl_id = memb_fapl_id; if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - ret_value = H5P_set_driver(plist, H5FD_FAMILY, &fa); + ret_value = H5P_set_driver(plist, H5FD_FAMILY, &fa, NULL); done: FUNC_LEAVE_API(ret_value) @@ -588,9 +690,10 @@ H5FD__family_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad { H5FD_family_t *file = NULL; char * memb_name = NULL, *temp = NULL; - hsize_t eof = HADDR_UNDEF; - unsigned t_flags = flags & ~H5F_ACC_CREAT; - H5FD_t * ret_value = NULL; + hsize_t eof = HADDR_UNDEF; + hbool_t default_config = FALSE; + unsigned t_flags = flags & ~H5F_ACC_CREAT; + H5FD_t * ret_value = NULL; FUNC_ENTER_STATIC @@ -604,21 +707,32 @@ H5FD__family_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad if (NULL == (file = (H5FD_family_t *)H5MM_calloc(sizeof(H5FD_family_t)))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") if (H5P_FILE_ACCESS_DEFAULT == fapl_id) { - file->memb_fapl_id = H5P_FILE_ACCESS_DEFAULT; - if (H5I_inc_ref(file->memb_fapl_id, FALSE) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINC, NULL, "unable to increment ref count on VFL driver") - file->memb_size = 1024 * 1024 * 1024; /*1GB. Actual member size to be updated later */ - file->pmem_size = 1024 * 1024 * 1024; /*1GB. Member size passed in through property */ - file->mem_newsize = 0; /*New member size used by h5repart only */ - } /* end if */ + H5FD_family_fapl_t default_fa; + + /* Get default configuration */ + if (H5FD__family_get_default_config(&default_fa) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get default driver configuration info") + + file->memb_fapl_id = default_fa.memb_fapl_id; + file->memb_size = H5FD_FAM_DEF_MEM_SIZE; /* Actual member size to be updated later */ + file->pmem_size = H5FD_FAM_DEF_MEM_SIZE; /* Member size passed in through property */ + file->mem_newsize = 0; /*New member size used by h5repart only */ + + default_config = TRUE; + } /* end if */ else { H5P_genplist_t * plist; /* Property list pointer */ const H5FD_family_fapl_t *fa; + H5FD_family_fapl_t default_fa; if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if (NULL == (fa = (const H5FD_family_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") + if (NULL == (fa = (const H5FD_family_fapl_t *)H5P_peek_driver_info(plist))) { + if (H5FD__family_get_default_config(&default_fa) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get default family VFD configuration") + fa = &default_fa; + default_config = TRUE; + } /* Check for new family file size. It's used by h5repart only. */ if (H5P_exist_plist(plist, H5F_ACS_FAMILY_NEWSIZE_NAME) > 0) { @@ -642,7 +756,10 @@ H5FD__family_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad } /* end else */ file->memb_size = fa->memb_size; /* Actual member size to be updated later */ file->pmem_size = fa->memb_size; /* Member size passed in through property */ - } /* end else */ + + if (default_config && H5I_dec_ref(fa->memb_fapl_id) < 0) + HGOTO_ERROR(H5E_ID, H5E_CANTDEC, NULL, "can't decrement ref. count on member FAPL") + } /* end else */ file->name = H5MM_strdup(name); file->flags = flags; @@ -655,8 +772,16 @@ H5FD__family_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad /* Check that names are unique */ HDsnprintf(memb_name, H5FD_FAM_MEMB_NAME_BUF_SIZE, name, 0); HDsnprintf(temp, H5FD_FAM_MEMB_NAME_BUF_SIZE, name, 1); - if (!HDstrcmp(memb_name, temp)) - HGOTO_ERROR(H5E_FILE, H5E_FILEEXISTS, NULL, "file names not unique") + if (!HDstrcmp(memb_name, temp)) { + if (default_config) { + temp = H5MM_xfree(temp); + if (NULL == (temp = H5FD__family_get_default_printf_filename(name))) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get default printf-style filename") + name = temp; + } + else + HGOTO_ERROR(H5E_FILE, H5E_FILEEXISTS, NULL, "file names not unique") + } /* Open all the family members */ while (1) { @@ -1360,7 +1485,9 @@ H5FD__family_delete(const char *filename, hid_t fapl_id) { H5P_genplist_t * plist; const H5FD_family_fapl_t *fa; - hid_t memb_fapl_id = H5I_INVALID_HID; + H5FD_family_fapl_t default_fa = {0, H5I_INVALID_HID}; + hbool_t default_config = FALSE; + hid_t memb_fapl_id = H5I_INVALID_HID; unsigned current_member; char * member_name = NULL; char * temp = NULL; @@ -1374,13 +1501,21 @@ H5FD__family_delete(const char *filename, hid_t fapl_id) /* Get the driver info (for the member fapl) * The family_open call accepts H5P_DEFAULT, so we'll accept that here, too. */ - if (H5P_FILE_ACCESS_DEFAULT == fapl_id) - memb_fapl_id = H5P_FILE_ACCESS_DEFAULT; + if (H5P_FILE_ACCESS_DEFAULT == fapl_id) { + if (H5FD__family_get_default_config(&default_fa) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get default family VFD configuration") + memb_fapl_id = default_fa.memb_fapl_id; + default_config = TRUE; + } else { if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - if (NULL == (fa = (const H5FD_family_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad family VFD driver info") + if (NULL == (fa = (const H5FD_family_fapl_t *)H5P_peek_driver_info(plist))) { + if (H5FD__family_get_default_config(&default_fa) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get default family VFD configuration") + fa = &default_fa; + default_config = TRUE; + } memb_fapl_id = fa->memb_fapl_id; } @@ -1393,8 +1528,17 @@ H5FD__family_delete(const char *filename, hid_t fapl_id) /* Sanity check to make sure that generated names are unique */ HDsnprintf(member_name, H5FD_FAM_MEMB_NAME_BUF_SIZE, filename, 0); HDsnprintf(temp, H5FD_FAM_MEMB_NAME_BUF_SIZE, filename, 1); - if (!HDstrcmp(member_name, temp)) - HGOTO_ERROR(H5E_VFL, H5E_CANTDELETEFILE, FAIL, "provided file name cannot generate unique sub-files") + if (!HDstrcmp(member_name, temp)) { + if (default_config) { + temp = H5MM_xfree(temp); + if (NULL == (temp = H5FD__family_get_default_printf_filename(filename))) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get default printf-style filename") + filename = temp; + } + else + HGOTO_ERROR(H5E_VFL, H5E_CANTDELETEFILE, FAIL, + "provided file name cannot generate unique sub-files") + } /* Delete all the family members */ current_member = 0; @@ -1430,9 +1574,9 @@ done: if (temp) H5MM_xfree(temp); - /* Don't close memb_fapl_id - We didn't bump its reference count since we're - * only using it in this call. - */ + /* Only close memb_fapl_id if we created one from the default configuration */ + if (default_fa.memb_fapl_id >= 0 && H5I_dec_ref(default_fa.memb_fapl_id) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't decrement ref. count on member FAPL ID") FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__family_delete() */ diff --git a/src/H5FDfamily.h b/src/H5FDfamily.h index 20ef532..bd51f36 100644 --- a/src/H5FDfamily.h +++ b/src/H5FDfamily.h @@ -20,7 +20,8 @@ #ifndef H5FDfamily_H #define H5FDfamily_H -#define H5FD_FAMILY (H5FD_family_init()) +#define H5FD_FAMILY (H5FDperform_init(H5FD_family_init)) +#define H5FD_FAMILY_VALUE H5_VFD_FAMILY #ifdef __cplusplus extern "C" { diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c index 2c4bff6..4927a40 100644 --- a/src/H5FDhdfs.c +++ b/src/H5FDhdfs.c @@ -278,6 +278,7 @@ static herr_t H5FD__hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing static herr_t H5FD__hdfs_validate_config(const H5FD_hdfs_fapl_t *fa); static const H5FD_class_t H5FD_hdfs_g = { + H5FD_HDFS_VALUE, /* value */ "hdfs", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -310,6 +311,7 @@ static const H5FD_class_t H5FD_hdfs_g = { NULL, /* lock */ NULL, /* unlock */ NULL, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -317,29 +319,6 @@ static const H5FD_class_t H5FD_hdfs_g = { H5FL_DEFINE_STATIC(H5FD_hdfs_t); /*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initializes any interface-specific data or routines. - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__init_package(void) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - if (H5FD_hdfs_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize hdfs VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - -/*------------------------------------------------------------------------- * Function: H5FD_hdfs_init * * Purpose: Initialize this driver by registering the driver with the @@ -551,16 +530,16 @@ done: * Function: H5FD__hdfs_validate_config() * * Purpose: Test to see if the supplied instance of H5FD_hdfs_fapl_t - * contains internally consistant data. Return SUCCEED if so, + * contains internally consistent data. Return SUCCEED if so, * and FAIL otherwise. * - * Note the difference between internally consistant and + * Note the difference between internally consistent and * correct. As we will have to try to access the target * object to determine whether the supplied data is correct, - * we will settle for internal consistancy at this point + * we will settle for internal consistency at this point * * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally - * consistant data, FAIL otherwise. + * consistent data, FAIL otherwise. * * Programmer: Jacob Smith * 9/10/17 @@ -623,7 +602,7 @@ H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa) if (FAIL == H5FD__hdfs_validate_config(fa)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid hdfs config") - ret_value = H5P_set_driver(plist, H5FD_HDFS, (void *)fa); + ret_value = H5P_set_driver(plist, H5FD_HDFS, (void *)fa, NULL); done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h index 7c871a4..e401e81 100644 --- a/src/H5FDhdfs.h +++ b/src/H5FDhdfs.h @@ -22,9 +22,11 @@ #define H5FDhdfs_H #ifdef H5_HAVE_LIBHDFS -#define H5FD_HDFS (H5FD_hdfs_init()) +#define H5FD_HDFS (H5FDperform_init(H5FD_hdfs_init)) +#define H5FD_HDFS_VALUE H5_VFD_HDFS #else /* H5_HAVE_LIBHDFS */ -#define H5FD_HDFS (H5I_INVALID_HID) +#define H5FD_HDFS (H5I_INVALID_HID) +#define H5FD_HDFS_VALUE H5_VFD_INVALID #endif /* H5_HAVE_LIBHDFS */ #ifdef H5_HAVE_LIBHDFS diff --git a/src/H5FDint.c b/src/H5FDint.c index 23dfa4d..d09801f 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -35,6 +35,7 @@ #include "H5Fprivate.h" /* File access */ #include "H5FDpkg.h" /* File Drivers */ #include "H5Iprivate.h" /* IDs */ +#include "H5PLprivate.h" /* Plugins */ /****************/ /* Local Macros */ @@ -44,6 +45,20 @@ /* Local Typedefs */ /******************/ +/* Information needed for iterating over the registered VFD hid_t IDs. + * The name or value of the new VFD that is being registered is stored + * in the name (or value) field and the found_id field is initialized to + * H5I_INVALID_HID (-1). If we find a VFD with the same name / value, + * we set the found_id field to the existing ID for return to the function. + */ +typedef struct H5FD_get_driver_ud_t { + /* IN */ + H5PL_vfd_key_t key; + + /* OUT */ + hid_t found_id; /* The driver ID, if we found a match */ +} H5FD_get_driver_ud_t; + /********************/ /* Package Typedefs */ /********************/ @@ -51,6 +66,7 @@ /********************/ /* Local Prototypes */ /********************/ +static int H5FD__get_driver_cb(void *obj, hid_t id, void *_op_data); /*********************/ /* Package Variables */ @@ -166,21 +182,23 @@ H5FD_read(H5FD_t *file, H5FD_mem_t type, haddr_t addr, size_t size, void *buf /* HGOTO_DONE(SUCCEED) #endif /* H5_HAVE_PARALLEL */ - /* If the file is open for SWMR read access, allow access to data past - * the end of the allocated space (the 'eoa'). This is done because the - * eoa stored in the file's superblock might be out of sync with the - * objects being written within the file by the application performing - * SWMR write operations. - */ #if 0 /* JRM */ - if(!(file->access_flags & H5F_ACC_SWMR_READ)) { - haddr_t eoa; + /* If the file is open for SWMR read access, allow access to data past + * the end of the allocated space (the 'eoa'). This is done because the + * eoa stored in the file's superblock might be out of sync with the + * objects being written within the file by the application performing + * SWMR write operations. + */ + if (!(file->access_flags & H5F_ACC_SWMR_READ)) { + haddr_t eoa; - if(HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type))) + if (HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type))) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed") - if((addr + file->base_addr + size) > eoa) - HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size = %llu, eoa = %llu", (unsigned long long)(addr + file->base_addr), (unsigned long long)size, (unsigned long long)eoa) + if ((addr + file->base_addr + size) > eoa) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size = %llu, eoa = %llu", + (unsigned long long)(addr + file->base_addr), (unsigned long long)size, + (unsigned long long)eoa) } #endif /* JRM */ @@ -425,3 +443,357 @@ H5FD_delete(const char *filename, hid_t fapl_id) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_delete() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_check_plugin_load + * + * Purpose: Check if a VFD plugin matches the search criteria, and can + * be loaded. + * + * Note: Matching the driver's name / value, but the driver having + * an incompatible version is not an error, but means that the + * driver isn't a "match". Setting the SUCCEED value to FALSE + * and not failing for that case allows the plugin framework + * to keep looking for other DLLs that match and have a + * compatible version. + * + * Return: SUCCEED / FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5FD_check_plugin_load(const H5FD_class_t *cls, const H5PL_key_t *key, hbool_t *success) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(cls); + HDassert(key); + HDassert(success); + + /* Which kind of key are we looking for? */ + if (key->vfd.kind == H5FD_GET_DRIVER_BY_NAME) { + /* Check if plugin name matches VFD class name */ + if (cls->name && !HDstrcmp(cls->name, key->vfd.u.name)) + *success = TRUE; + } + else { + /* Sanity check */ + HDassert(key->vfd.kind == H5FD_GET_DRIVER_BY_VALUE); + + /* Check if plugin value matches VFD class value */ + if (cls->value == key->vfd.u.value) + *success = TRUE; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_check_plugin_load() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__get_driver_cb + * + * Purpose: Callback routine to search through registered VFDs + * + * Return: Success: H5_ITER_STOP if the class and op_data name + * members match. H5_ITER_CONT otherwise. + * Failure: Can't fail + * + *------------------------------------------------------------------------- + */ +static int +H5FD__get_driver_cb(void *obj, hid_t id, void *_op_data) +{ + H5FD_get_driver_ud_t *op_data = (H5FD_get_driver_ud_t *)_op_data; /* User data for callback */ + H5FD_class_t * cls = (H5FD_class_t *)obj; + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC_NOERR + + if (H5FD_GET_DRIVER_BY_NAME == op_data->key.kind) { + if (0 == HDstrcmp(cls->name, op_data->key.u.name)) { + op_data->found_id = id; + ret_value = H5_ITER_STOP; + } /* end if */ + } /* end if */ + else { + HDassert(H5FD_GET_DRIVER_BY_VALUE == op_data->key.kind); + if (cls->value == op_data->key.u.value) { + op_data->found_id = id; + ret_value = H5_ITER_STOP; + } /* end if */ + } /* end else */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__get_driver_cb() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_register_driver_by_name + * + * Purpose: Registers a new VFD as a member of the virtual file driver + * class. + * + * Return: Success: A VFD ID which is good until the library is + * closed. + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_register_driver_by_name(const char *name, hbool_t app_ref) +{ + htri_t driver_is_registered = FALSE; + hid_t driver_id = H5I_INVALID_HID; + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_NOAPI(H5I_INVALID_HID) + + /* Check if driver is already registered */ + if ((driver_is_registered = H5FD_is_driver_registered_by_name(name, &driver_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADITER, H5I_INVALID_HID, "can't check if driver is already registered") + + /* If driver is already registered, increment ref count on ID and return ID */ + if (driver_is_registered) { + HDassert(driver_id >= 0); + + if (H5I_inc_ref(driver_id, app_ref) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINC, H5I_INVALID_HID, "unable to increment ref count on VFD") + } /* end if */ + else { + H5PL_key_t key; + const H5FD_class_t *cls; + + /* Try loading the driver */ + key.vfd.kind = H5FD_GET_DRIVER_BY_NAME; + key.vfd.u.name = name; + if (NULL == (cls = (const H5FD_class_t *)H5PL_load(H5PL_TYPE_VFD, &key))) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, "unable to load VFD") + + /* Register the driver we loaded */ + if ((driver_id = H5FD_register(cls, sizeof(*cls), app_ref)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to register VFD ID") + } /* end else */ + + ret_value = driver_id; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_register_driver_by_name() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_register_driver_by_value + * + * Purpose: Registers a new VFD as a member of the virtual file driver + * class. + * + * Return: Success: A VFD ID which is good until the library is + * closed. + * + * Failure: H5I_INVALID_HID + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_register_driver_by_value(H5FD_class_value_t value, hbool_t app_ref) +{ + htri_t driver_is_registered = FALSE; + hid_t driver_id = H5I_INVALID_HID; + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_NOAPI(H5I_INVALID_HID) + + /* Check if driver is already registered */ + if ((driver_is_registered = H5FD_is_driver_registered_by_value(value, &driver_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADITER, H5I_INVALID_HID, "can't check if driver is already registered") + + /* If driver is already registered, increment ref count on ID and return ID */ + if (driver_is_registered) { + HDassert(driver_id >= 0); + + if (H5I_inc_ref(driver_id, app_ref) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINC, H5I_INVALID_HID, "unable to increment ref count on VFD") + } /* end if */ + else { + H5PL_key_t key; + const H5FD_class_t *cls; + + /* Try loading the driver */ + key.vfd.kind = H5FD_GET_DRIVER_BY_VALUE; + key.vfd.u.value = value; + if (NULL == (cls = (const H5FD_class_t *)H5PL_load(H5PL_TYPE_VFD, &key))) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, "unable to load VFD") + + /* Register the driver we loaded */ + if ((driver_id = H5FD_register(cls, sizeof(*cls), app_ref)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to register VFD ID") + } /* end else */ + + ret_value = driver_id; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_register_driver_by_value() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_is_driver_registered_by_name + * + * Purpose: Checks if a driver with a particular name is registered. + * If `registered_id` is non-NULL and a driver with the + * specified name has been registered, the driver's ID will be + * returned in `registered_id`. + * + * Return: >0 if a VFD with that name has been registered + * 0 if a VFD with that name has NOT been registered + * <0 on errors + * + *------------------------------------------------------------------------- + */ +htri_t +H5FD_is_driver_registered_by_name(const char *driver_name, hid_t *registered_id) +{ + H5FD_get_driver_ud_t op_data; /* Callback info for driver search */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Set up op data for iteration */ + op_data.key.kind = H5FD_GET_DRIVER_BY_NAME; + op_data.key.u.name = driver_name; + op_data.found_id = H5I_INVALID_HID; + + /* Find driver with name */ + if (H5I_iterate(H5I_VFL, H5FD__get_driver_cb, &op_data, FALSE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADITER, FAIL, "can't iterate over VFDs") + + /* Found a driver with that name */ + if (op_data.found_id != H5I_INVALID_HID) { + if (registered_id) + *registered_id = op_data.found_id; + ret_value = TRUE; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_is_driver_registered_by_name() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_is_driver_registered_by_value + * + * Purpose: Checks if a driver with a particular value (ID) is + * registered. If `registered_id` is non-NULL and a driver + * with the specified value has been registered, the driver's + * ID will be returned in `registered_id`. + * + * Return: >0 if a VFD with that value has been registered + * 0 if a VFD with that value has NOT been registered + * <0 on errors + * + *------------------------------------------------------------------------- + */ +htri_t +H5FD_is_driver_registered_by_value(H5FD_class_value_t driver_value, hid_t *registered_id) +{ + H5FD_get_driver_ud_t op_data; /* Callback info for driver search */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Set up op data for iteration */ + op_data.key.kind = H5FD_GET_DRIVER_BY_VALUE; + op_data.key.u.value = driver_value; + op_data.found_id = H5I_INVALID_HID; + + /* Find driver with value */ + if (H5I_iterate(H5I_VFL, H5FD__get_driver_cb, &op_data, FALSE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADITER, FAIL, "can't iterate over VFDs") + + /* Found a driver with that value */ + if (op_data.found_id != H5I_INVALID_HID) { + if (registered_id) + *registered_id = op_data.found_id; + ret_value = TRUE; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_is_driver_registered_by_value() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_get_driver_id_by_name + * + * Purpose: Retrieves the ID for a registered VFL driver. + * + * Return: Positive if the VFL driver has been registered + * Negative on error (if the driver is not a valid driver or + * is not registered) + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_get_driver_id_by_name(const char *name, hbool_t is_api) +{ + H5FD_get_driver_ud_t op_data; + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_NOAPI(H5I_INVALID_HID) + + /* Set up op data for iteration */ + op_data.key.kind = H5FD_GET_DRIVER_BY_NAME; + op_data.key.u.name = name; + op_data.found_id = H5I_INVALID_HID; + + /* Find driver with specified name */ + if (H5I_iterate(H5I_VFL, H5FD__get_driver_cb, &op_data, FALSE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADITER, H5I_INVALID_HID, "can't iterate over VFL drivers") + + /* Found a driver with that name */ + if (op_data.found_id != H5I_INVALID_HID) { + ret_value = op_data.found_id; + if (H5I_inc_ref(ret_value, is_api) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINC, H5I_INVALID_HID, "unable to increment ref count on VFL driver") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_get_driver_id_by_name() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_get_driver_id_by_value + * + * Purpose: Retrieves the ID for a registered VFL driver. + * + * Return: Positive if the VFL driver has been registered + * Negative on error (if the driver is not a valid driver or + * is not registered) + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_get_driver_id_by_value(H5FD_class_value_t value, hbool_t is_api) +{ + H5FD_get_driver_ud_t op_data; + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_NOAPI(H5I_INVALID_HID) + + /* Set up op data for iteration */ + op_data.key.kind = H5FD_GET_DRIVER_BY_VALUE; + op_data.key.u.value = value; + op_data.found_id = H5I_INVALID_HID; + + /* Find driver with specified value */ + if (H5I_iterate(H5I_VFL, H5FD__get_driver_cb, &op_data, FALSE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADITER, H5I_INVALID_HID, "can't iterate over VFL drivers") + + /* Found a driver with that value */ + if (op_data.found_id != H5I_INVALID_HID) { + ret_value = op_data.found_id; + if (H5I_inc_ref(ret_value, is_api) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINC, H5I_INVALID_HID, "unable to increment ref count on VFL driver") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_get_driver_id_by_value() */ diff --git a/src/H5FDlog.c b/src/H5FDlog.c index f996b9e..af34682 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -180,78 +180,50 @@ static herr_t H5FD__log_unlock(H5FD_t *_file); static herr_t H5FD__log_delete(const char *filename, hid_t fapl_id); static const H5FD_class_t H5FD_log_g = { - "log", /* name */ - MAXADDR, /* maxaddr */ - H5F_CLOSE_WEAK, /* fc_degree */ - H5FD__log_term, /* terminate */ - NULL, /* sb_size */ - NULL, /* sb_encode */ - NULL, /* sb_decode */ - sizeof(H5FD_log_fapl_t), /* fapl_size */ - H5FD__log_fapl_get, /* fapl_get */ - H5FD__log_fapl_copy, /* fapl_copy */ - H5FD__log_fapl_free, /* fapl_free */ - 0, /* dxpl_size */ - NULL, /* dxpl_copy */ - NULL, /* dxpl_free */ - H5FD__log_open, /* open */ - H5FD__log_close, /* close */ - H5FD__log_cmp, /* cmp */ - H5FD__log_query, /* query */ - NULL, /* get_type_map */ - H5FD__log_alloc, /* alloc */ - H5FD__log_free, /* free */ - H5FD__log_get_eoa, /* get_eoa */ - H5FD__log_set_eoa, /* set_eoa */ - H5FD__log_get_eof, /* get_eof */ - H5FD__log_get_handle, /* get_handle */ - H5FD__log_read, /* read */ - H5FD__log_write, /* write */ - NULL, /* flush */ - H5FD__log_truncate, /* truncate */ - H5FD__log_lock, /* lock */ - H5FD__log_unlock, /* unlock */ - H5FD__log_delete, /* del */ - H5FD_FLMAP_DICHOTOMY /* fl_map */ + H5FD_LOG_VALUE, /* value */ + "log", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__log_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_log_fapl_t), /* fapl_size */ + H5FD__log_fapl_get, /* fapl_get */ + H5FD__log_fapl_copy, /* fapl_copy */ + H5FD__log_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__log_open, /* open */ + H5FD__log_close, /* close */ + H5FD__log_cmp, /* cmp */ + H5FD__log_query, /* query */ + NULL, /* get_type_map */ + H5FD__log_alloc, /* alloc */ + H5FD__log_free, /* free */ + H5FD__log_get_eoa, /* get_eoa */ + H5FD__log_set_eoa, /* set_eoa */ + H5FD__log_get_eof, /* get_eof */ + H5FD__log_get_handle, /* get_handle */ + H5FD__log_read, /* read */ + H5FD__log_write, /* write */ + NULL, /* flush */ + H5FD__log_truncate, /* truncate */ + H5FD__log_lock, /* lock */ + H5FD__log_unlock, /* unlock */ + H5FD__log_delete, /* del */ + NULL, /* ctl */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ }; +/* Default configuration, if none provided */ +static const H5FD_log_fapl_t H5FD_log_default_config_g = {NULL, H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC, 4096}; + /* Declare a free list to manage the H5FD_log_t struct */ H5FL_DEFINE_STATIC(H5FD_log_t); /*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initializes any interface-specific data or routines. - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__init_package(void) -{ - char * lock_env_var = NULL; /* Environment variable pointer */ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - /* Check the use disabled file locks environment variable */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) - ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ - else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) - ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ - else - ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ - - if (H5FD_log_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize log VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - -/*------------------------------------------------------------------------- * Function: H5FD_log_init * * Purpose: Initialize this driver by registering the driver with the @@ -268,9 +240,19 @@ done: hid_t H5FD_log_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + char *lock_env_var = NULL; /* Environment variable pointer */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ - FUNC_ENTER_NOAPI(H5I_INVALID_HID) + FUNC_ENTER_NOAPI_NOERR + + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv(HDF5_USE_FILE_LOCKING); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ if (H5I_VFL != H5I_get_type(H5FD_LOG_g)) H5FD_LOG_g = H5FD_register(&H5FD_log_g, sizeof(H5FD_class_t), FALSE); @@ -278,7 +260,6 @@ H5FD_log_init(void) /* Set return value */ ret_value = H5FD_LOG_g; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_log_init() */ @@ -347,7 +328,7 @@ H5Pset_fapl_log(hid_t fapl_id, const char *logfile, unsigned long long flags, si fa.flags = flags; fa.buf_size = buf_size; - ret_value = H5P_set_driver(plist, H5FD_LOG, &fa); + ret_value = H5P_set_driver(plist, H5FD_LOG, &fa, NULL); done: if (fa.logfile) @@ -482,10 +463,11 @@ static H5FD_t * H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { H5FD_log_t * file = NULL; - H5P_genplist_t * plist; /* Property list */ - const H5FD_log_fapl_t *fa; /* File access property list information */ - int fd = -1; /* File descriptor */ - int o_flags; /* Flags for open() call */ + H5P_genplist_t * plist; /* Property list */ + const H5FD_log_fapl_t *fa; /* File access property list information */ + H5FD_log_fapl_t default_fa = H5FD_log_default_config_g; + int fd = -1; /* File descriptor */ + int o_flags; /* Flags for open() call */ #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif @@ -523,8 +505,10 @@ H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) /* Get the driver specific information */ if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if (NULL == (fa = (const H5FD_log_fapl_t *)H5P_peek_driver_info(plist))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") + if (NULL == (fa = (const H5FD_log_fapl_t *)H5P_peek_driver_info(plist))) { + /* Use default driver configuration*/ + fa = &default_fa; + } /* Start timer for open() call */ if (fa->flags & H5FD_LOG_TIME_OPEN) @@ -1559,7 +1543,7 @@ H5FD__log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, ha HDfprintf(file->logfp, "%10" PRIuHADDR "-%10" PRIuHADDR " (%10zu bytes) (%s) Written", orig_addr, (orig_addr + orig_size) - 1, orig_size, flavors[type]); - /* Check if this is the first write into a "default" section, grabbed by the metadata agregation + /* Check if this is the first write into a "default" section, grabbed by the metadata aggregation * algorithm */ if (file->fa.flags & H5FD_LOG_FLAVOR) { if ((H5FD_mem_t)file->flavor[orig_addr] == H5FD_MEM_DEFAULT) { diff --git a/src/H5FDlog.h b/src/H5FDlog.h index 969c091..a916fec 100644 --- a/src/H5FDlog.h +++ b/src/H5FDlog.h @@ -20,7 +20,8 @@ #ifndef H5FDlog_H #define H5FDlog_H -#define H5FD_LOG (H5FD_log_init()) +#define H5FD_LOG (H5FDperform_init(H5FD_log_init)) +#define H5FD_LOG_VALUE H5_VFD_LOG /* Flags for H5Pset_fapl_log() */ /* Flags for tracking 'meta' operations (truncate) */ diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c index 8cbeff6..e74901a 100644 --- a/src/H5FDmirror.c +++ b/src/H5FDmirror.c @@ -160,6 +160,7 @@ static herr_t H5FD__mirror_unlock(H5FD_t *_file); static herr_t H5FD__mirror_verify_reply(H5FD_mirror_t *file); static const H5FD_class_t H5FD_mirror_g = { + H5FD_MIRROR_VALUE, /* value */ "mirror", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -192,6 +193,7 @@ static const H5FD_class_t H5FD_mirror_g = { H5FD__mirror_lock, /* lock */ H5FD__mirror_unlock, /* unlock */ NULL, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -204,30 +206,6 @@ H5FL_DEFINE_STATIC(H5FD_mirror_t); /* Declare a free list to manage the H5FD_mirror_xmit_open_t struct */ H5FL_DEFINE_STATIC(H5FD_mirror_xmit_open_t); -/*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initializes any interface-specific data or routines. - * - * Return: Non-negative on success/Negative on failure - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__init_package(void) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - LOG_OP_CALL(__func__); - - if (H5FD_mirror_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize mirror VFD"); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - /* ------------------------------------------------------------------------- * Function: H5FD_mirror_init * @@ -247,9 +225,11 @@ H5FD_mirror_init(void) LOG_OP_CALL(__func__); - if (H5I_VFL != H5I_get_type(H5FD_MIRROR_g)) + if (H5I_VFL != H5I_get_type(H5FD_MIRROR_g)) { H5FD_MIRROR_g = H5FD_register(&H5FD_mirror_g, sizeof(H5FD_class_t), FALSE); - + if (H5I_INVALID_HID == H5FD_MIRROR_g) + HGOTO_ERROR(H5E_ID, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to register mirror"); + } ret_value = H5FD_MIRROR_g; done: @@ -1167,7 +1147,7 @@ done: /* ------------------------------------------------------------------------- * Function: H5FD__mirror_fapl_get * - * Purpose: Get the file access propety list which could be used to create + * Purpose: Get the file access property list which could be used to create * an identical file. * * Return: Success: pointer to the new file access property list value. @@ -1335,7 +1315,7 @@ H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa) if (H5FD_MIRROR_CURR_FAPL_T_VERSION != fa->version) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown fapl_t version"); - ret_value = H5P_set_driver(plist, H5FD_MIRROR, (const void *)fa); + ret_value = H5P_set_driver(plist, H5FD_MIRROR, (const void *)fa, NULL); done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h index 49e24c1..b196b2b 100644 --- a/src/H5FDmirror.h +++ b/src/H5FDmirror.h @@ -19,7 +19,8 @@ #ifdef H5_HAVE_MIRROR_VFD -#define H5FD_MIRROR (H5FD_mirror_init()) +#define H5FD_MIRROR (H5FDperform_init(H5FD_mirror_init)) +#define H5FD_MIRROR_VALUE H5_VFD_MIRROR #ifdef __cplusplus extern "C" { @@ -33,7 +34,7 @@ extern "C" { /* --------------------------------------------------------------------------- * Structure: H5FD_mirror_fapl_t * - * Used to pass configuraiton information to the Mirror VFD. + * Used to pass configuration information to the Mirror VFD. * Populate components as appropriate and pass structure pointer to * `H5Pset_fapl_mirror()`. * diff --git a/src/H5FDmirror_priv.h b/src/H5FDmirror_priv.h index 21de97b..f647c21 100644 --- a/src/H5FDmirror_priv.h +++ b/src/H5FDmirror_priv.h @@ -28,10 +28,10 @@ extern "C" { * = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */ -/* The maximum allowed size for a receiving buffer when accepting bytes to +/* Define the maximum allowed size for a receiving buffer when accepting bytes to * write. Writes larger than this size are performed by multiple accept-write * steps by the Writer. */ -#define H5FD_MIRROR_DATA_BUFFER_MAX H5_GB /* 1 Gigabyte */ +#define H5FD_MIRROR_DATA_BUFFER_MAX (1024 * 1024 * 1024) /* 1 Gigabyte */ #define H5FD_MIRROR_XMIT_CURR_VERSION 1 #define H5FD_MIRROR_XMIT_MAGIC 0x87F8005B @@ -80,7 +80,7 @@ extern "C" { * * `magic` (uint32_t) * A "unique" number identifying the structure and endianness of - * transmitting maching. + * transmitting machine. * Must be set to H5FD_MIRROR_XMIT_MAGIC native to the VFD "sender". * * `version` (uint8_t) @@ -214,13 +214,13 @@ typedef struct H5FD_mirror_xmit_open_t { * * `status` (uint32_t) * Number indicating whether the command was successful or if an - * occured. + * occurred. * Allowed values are H5FD_MIRROR_STATUS_OK and * H5FD_MIRROR_STATUS_ERROR. * * `message` (char[]) * Error message. Populated if and only if there was a problem. - * It is possible that a message may reach the end of the alloted + * It is possible that a message may reach the end of the allotted * space without a NULL terminator -- the onus is on the programmer to * handle this situation. * diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c index 048b8f3..7eb1463 100644 --- a/src/H5FDmpi.c +++ b/src/H5FDmpi.c @@ -41,26 +41,42 @@ * Programmer: Quincey Koziol * Friday, January 30, 2004 * + * Changes: Reworked function to use the ctl callback so we can get + * rid of H5FD_class_mpi_t. Since there are no real limits + * on what the ctl callback can do, its file parameter can't + * be constant. Thus, I had to remove the const qualifier + * on this functions file parameter as well. Note also the + * circumlocution required to use the ctl callbacks output + * parameter to pass back the rank without introducing + * compiler warnings. + * JRM -- 8/13/21 + * *------------------------------------------------------------------------- */ int -H5FD_mpi_get_rank(const H5FD_t *file) +H5FD_mpi_get_rank(H5FD_t *file) { - const H5FD_class_mpi_t *cls; - - int ret_value; + const H5FD_class_t *cls; + uint64_t flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + int rank = -1; + void * rank_ptr = (void *)(&rank); + int ret_value; FUNC_ENTER_NOAPI(FAIL) HDassert(file); - cls = (const H5FD_class_mpi_t *)(file->cls); + cls = (const H5FD_class_t *)(file->cls); HDassert(cls); - HDassert(cls->get_rank); /* All MPI drivers must implement this */ + HDassert(cls->ctl); /* All MPI drivers must implement this */ /* Dispatch to driver */ - if ((ret_value = (cls->get_rank)(file)) < 0) + if ((cls->ctl)(file, H5FD_CTL__GET_MPI_RANK_OPCODE, flags, NULL, &rank_ptr) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_rank request failed") + HDassert(rank >= 0); + + ret_value = rank; + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_mpi_get_rank() */ @@ -77,25 +93,43 @@ done: * Programmer: Quincey Koziol * Friday, January 30, 2004 * + * Changes: Reworked function to use the ctl callback so we can get + * rid of H5FD_class_mpi_t. Since there are no real limits + * on what the ctl callback can do, its file parameter can't + * be constant. Thus, I had to remove the const qualifier + * on this functions file parameter as well. Note also the + * circumlocution required to use the ctl callbacks output + * parameter to pass back the rank without introducing + * compiler warnings. + * JRM -- 8/13/21 + * *------------------------------------------------------------------------- */ int -H5FD_mpi_get_size(const H5FD_t *file) +H5FD_mpi_get_size(H5FD_t *file) { - const H5FD_class_mpi_t *cls; - int ret_value; + const H5FD_class_t *cls; + uint64_t flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + int size = 0; + void * size_ptr = (void *)(&size); + int ret_value; FUNC_ENTER_NOAPI(FAIL) HDassert(file); - cls = (const H5FD_class_mpi_t *)(file->cls); + cls = (const H5FD_class_t *)(file->cls); HDassert(cls); - HDassert(cls->get_size); /* All MPI drivers must implement this */ + HDassert(cls->ctl); /* All MPI drivers must implement this */ /* Dispatch to driver */ - if ((ret_value = (cls->get_size)(file)) < 0) + if ((cls->ctl)(file, H5FD_CTL__GET_MPI_SIZE_OPCODE, flags, NULL, &size_ptr) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_size request failed") + if (0 >= size) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_size request returned bad value") + + ret_value = size; + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_mpi_get_size() */ @@ -112,25 +146,43 @@ done: * Programmer: Quincey Koziol * Friday, January 30, 2004 * + * Changes: Reworked function to use the ctl callback so we can get + * rid of H5FD_class_mpi_t. Since there are no real limits + * on what the ctl callback can do, its file parameter can't + * be constant. Thus, I had to remove the const qualifier + * on this functions file parameter as well. Note also the + * circumlocution required to use the ctl callbacks output + * parameter to pass back the rank without introducing + * compiler warnings. + * JRM -- 8/13/21 + * *------------------------------------------------------------------------- */ MPI_Comm -H5FD_mpi_get_comm(const H5FD_t *file) +H5FD_mpi_get_comm(H5FD_t *file) { - const H5FD_class_mpi_t *cls; - MPI_Comm ret_value; + const H5FD_class_t *cls; + uint64_t flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + MPI_Comm comm = MPI_COMM_NULL; + void * comm_ptr = (void *)(&comm); + MPI_Comm ret_value; FUNC_ENTER_NOAPI(MPI_COMM_NULL) HDassert(file); - cls = (const H5FD_class_mpi_t *)(file->cls); + cls = (const H5FD_class_t *)(file->cls); HDassert(cls); - HDassert(cls->get_comm); /* All MPI drivers must implement this */ + HDassert(cls->ctl); /* All MPI drivers must implement this */ /* Dispatch to driver */ - if ((ret_value = (cls->get_comm)(file)) == MPI_COMM_NULL) + if ((cls->ctl)(file, H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE, flags, NULL, &comm_ptr) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed") + if (comm == MPI_COMM_NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed -- bad comm") + + ret_value = comm; + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_mpi_get_comm() */ diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index dd40399..4aa8a96 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -45,6 +45,9 @@ static hid_t H5FD_MPIO_g = 0; /* (Can be changed by setting "HDF5_MPI_OPT_TYPES" environment variable to '0' or '1') */ hbool_t H5FD_mpi_opt_types_g = TRUE; +/* Whether the driver initialized MPI on its own */ +hbool_t H5FD_mpi_self_initialized = FALSE; + /* * The view is set to this value */ @@ -72,66 +75,61 @@ typedef struct H5FD_mpio_t { /* Private Prototypes */ /* Callbacks */ -static herr_t H5FD__mpio_term(void); -static H5FD_t * H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); -static herr_t H5FD__mpio_close(H5FD_t *_file); -static herr_t H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags); -static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); -static haddr_t H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle); -static herr_t H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, - void *buf); -static herr_t H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, - const void *buf); -static herr_t H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); -static herr_t H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); -static herr_t H5FD__mpio_delete(const char *filename, hid_t fapl_id); -static int H5FD__mpio_mpi_rank(const H5FD_t *_file); -static int H5FD__mpio_mpi_size(const H5FD_t *_file); -static MPI_Comm H5FD__mpio_communicator(const H5FD_t *_file); +static herr_t H5FD__mpio_term(void); +static H5FD_t *H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__mpio_close(H5FD_t *_file); +static herr_t H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle); +static herr_t H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, + void *buf); +static herr_t H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, + const void *buf); +static herr_t H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__mpio_delete(const char *filename, hid_t fapl_id); +static herr_t H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, + void **output); /* The MPIO file driver information */ -static const H5FD_class_mpi_t H5FD_mpio_g = { - { - /* Start of superclass information */ - "mpio", /* name */ - HADDR_MAX, /* maxaddr */ - H5F_CLOSE_SEMI, /* fc_degree */ - H5FD__mpio_term, /* terminate */ - NULL, /* sb_size */ - NULL, /* sb_encode */ - NULL, /* sb_decode */ - 0, /* fapl_size */ - NULL, /* fapl_get */ - NULL, /* fapl_copy */ - NULL, /* fapl_free */ - 0, /* dxpl_size */ - NULL, /* dxpl_copy */ - NULL, /* dxpl_free */ - H5FD__mpio_open, /* open */ - H5FD__mpio_close, /* close */ - NULL, /* cmp */ - H5FD__mpio_query, /* query */ - NULL, /* get_type_map */ - NULL, /* alloc */ - NULL, /* free */ - H5FD__mpio_get_eoa, /* get_eoa */ - H5FD__mpio_set_eoa, /* set_eoa */ - H5FD__mpio_get_eof, /* get_eof */ - H5FD__mpio_get_handle, /* get_handle */ - H5FD__mpio_read, /* read */ - H5FD__mpio_write, /* write */ - H5FD__mpio_flush, /* flush */ - H5FD__mpio_truncate, /* truncate */ - NULL, /* lock */ - NULL, /* unlock */ - H5FD__mpio_delete, /* del */ - H5FD_FLMAP_DICHOTOMY /* fl_map */ - }, /* End of superclass information */ - H5FD__mpio_mpi_rank, /* get_rank */ - H5FD__mpio_mpi_size, /* get_size */ - H5FD__mpio_communicator /* get_comm */ +static const H5FD_class_t H5FD_mpio_g = { + H5_VFD_MPIO, /* value */ + "mpio", /* name */ + HADDR_MAX, /* maxaddr */ + H5F_CLOSE_SEMI, /* fc_degree */ + H5FD__mpio_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + 0, /* fapl_size */ + NULL, /* fapl_get */ + NULL, /* fapl_copy */ + NULL, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__mpio_open, /* open */ + H5FD__mpio_close, /* close */ + NULL, /* cmp */ + H5FD__mpio_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD__mpio_get_eoa, /* get_eoa */ + H5FD__mpio_set_eoa, /* set_eoa */ + H5FD__mpio_get_eof, /* get_eof */ + H5FD__mpio_get_handle, /* get_handle */ + H5FD__mpio_read, /* read */ + H5FD__mpio_write, /* write */ + H5FD__mpio_flush, /* flush */ + H5FD__mpio_truncate, /* truncate */ + NULL, /* lock */ + NULL, /* unlock */ + H5FD__mpio_delete, /* del */ + H5FD__mpio_ctl, /* ctl */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ }; #ifdef H5FDmpio_DEBUG @@ -157,35 +155,6 @@ static int H5FD_mpio_debug_rank_s = -1; (H5FD_mpio_debug_rank_s < 0 || H5FD_mpio_debug_rank_s == (file)->mpi_rank) #endif -/*-------------------------------------------------------------------------- -NAME - H5FD__init_package -- Initialize interface-specific information - -USAGE - herr_t H5FD__init_package() - -RETURNS - SUCCEED/FAIL - -DESCRIPTION - Initializes any interface-specific data or routines. (Just calls - H5FD_mpio_init currently). - ---------------------------------------------------------------------------*/ -static herr_t -H5FD__init_package(void) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - if (H5FD_mpio_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize mpio VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - #ifdef H5FDmpio_DEBUG /*--------------------------------------------------------------------------- @@ -219,6 +188,41 @@ H5FD__mpio_parse_debug_str(const char *s) FUNC_LEAVE_NOAPI_VOID } /* end H5FD__mpio_parse_debug_str() */ + +/*--------------------------------------------------------------------------- + * Function: H5FD__mem_t_to_str + * + * Purpose: Returns a string representing the enum value in an H5FD_mem_t + * enum + * + * Returns: H5FD_mem_t enum value string + * + *--------------------------------------------------------------------------- + */ +static const char * +H5FD__mem_t_to_str(H5FD_mem_t mem_type) +{ + switch (mem_type) { + case H5FD_MEM_NOLIST: + return "H5FD_MEM_NOLIST"; + case H5FD_MEM_DEFAULT: + return "H5FD_MEM_DEFAULT"; + case H5FD_MEM_SUPER: + return "H5FD_MEM_SUPER"; + case H5FD_MEM_BTREE: + return "H5FD_MEM_BTREE"; + case H5FD_MEM_DRAW: + return "H5FD_MEM_DRAW"; + case H5FD_MEM_GHEAP: + return "H5FD_MEM_GHEAP"; + case H5FD_MEM_LHEAP: + return "H5FD_MEM_LHEAP"; + case H5FD_MEM_OHDR: + return "H5FD_MEM_OHDR"; + default: + return "(Unknown)"; + } +} #endif /* H5FDmpio_DEBUG */ /*------------------------------------------------------------------------- @@ -239,13 +243,30 @@ hid_t H5FD_mpio_init(void) { static int H5FD_mpio_Debug_inited = 0; + char * env = NULL; hid_t ret_value = H5I_INVALID_HID; /* Return value */ FUNC_ENTER_NOAPI(H5I_INVALID_HID) /* Register the MPI-IO VFD, if it isn't already */ - if (H5I_VFL != H5I_get_type(H5FD_MPIO_g)) - H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_mpi_t), FALSE); + if (H5I_VFL != H5I_get_type(H5FD_MPIO_g)) { + H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_t), FALSE); + + /* Check if MPI driver has been loaded dynamically */ + env = HDgetenv(HDF5_DRIVER); + if (env && !HDstrcmp(env, "mpio")) { + int mpi_initialized = 0; + + /* Initialize MPI if not already initialized */ + if (MPI_SUCCESS != MPI_Initialized(&mpi_initialized)) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, H5I_INVALID_HID, "can't check if MPI is initialized") + if (!mpi_initialized) { + if (MPI_SUCCESS != MPI_Init(NULL, NULL)) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, "can't initialize MPI") + H5FD_mpi_self_initialized = TRUE; + } + } + } if (!H5FD_mpio_Debug_inited) { const char *s; /* String for environment variables */ @@ -292,6 +313,17 @@ H5FD__mpio_term(void) { FUNC_ENTER_STATIC_NOERR + /* Terminate MPI if the driver initialized it */ + if (H5FD_mpi_self_initialized) { + int mpi_finalized = 0; + + MPI_Finalized(&mpi_finalized); + if (!mpi_finalized) + MPI_Finalize(); + + H5FD_mpi_self_initialized = FALSE; + } + /* Reset VFL ID */ H5FD_MPIO_g = 0; @@ -353,7 +385,7 @@ H5Pset_fapl_mpio(hid_t fapl_id, MPI_Comm comm, MPI_Info info) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI info object") /* duplication is done during driver setting. */ - ret_value = H5P_set_driver(plist, H5FD_MPIO, NULL); + ret_value = H5P_set_driver(plist, H5FD_MPIO, NULL, NULL); done: FUNC_LEAVE_API(ret_value) @@ -808,11 +840,16 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t H5_ATTR if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - /* Get the MPI communicator and info object from the property list */ - if (H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get MPI communicator") - if (H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get MPI info object") + if (H5FD_mpi_self_initialized) { + comm = MPI_COMM_WORLD; + } + else { + /* Get the MPI communicator and info object from the property list */ + if (H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get MPI communicator") + if (H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get MPI info object") + } /* Get the MPI rank of this process and the total number of processes */ if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank(comm, &mpi_rank))) @@ -862,14 +899,19 @@ H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t H5_ATTR file->mpi_size = mpi_size; /* Only processor p0 will get the filesize and broadcast it. */ - if (mpi_rank == 0) + if (mpi_rank == 0) { + /* If MPI_File_get_size fails, broadcast file size as -1 to signal error */ if (MPI_SUCCESS != (mpi_code = MPI_File_get_size(fh, &file_size))) - HMPI_GOTO_ERROR(NULL, "MPI_File_get_size failed", mpi_code) + file_size = (MPI_Offset)-1; + } /* Broadcast file size */ if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&file_size, (int)sizeof(MPI_Offset), MPI_BYTE, 0, comm))) HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + if (file_size < 0) + HMPI_GOTO_ERROR(NULL, "MPI_File_get_size failed", mpi_code) + /* Determine if the file should be truncated */ if (file_size && (flags & H5F_ACC_TRUNC)) { /* Truncate the file */ @@ -987,7 +1029,6 @@ H5FD__mpio_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out *flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ *flags |= H5FD_FEAT_HAS_MPI; /* This driver uses MPI */ - *flags |= H5FD_FEAT_ALLOCATE_EARLY; /* Allocate space early instead of late */ *flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default VFD */ } /* end if */ @@ -1165,6 +1206,7 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU int n; #endif hbool_t use_view_this_time = FALSE; + hbool_t derived_type = FALSE; hbool_t rank0_bcast = FALSE; /* If read-with-rank0-and-bcast flag was used */ #ifdef H5FDmpio_DEBUG hbool_t H5FD_mpio_debug_t_flag = (H5FD_mpio_debug_flags_s[(int)'t'] && H5FD_MPIO_TRACE_THIS_RANK(file)); @@ -1192,8 +1234,6 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU if (H5FD_mpi_haddr_to_MPIOff(addr, &mpi_off /*out*/) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from haddr to MPI off") size_i = (int)size; - if ((hsize_t)size_i != size) - HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "can't convert from size to size_i") /* Only look for MPI views for raw data transfers */ if (type == H5FD_MEM_DRAW) { @@ -1260,10 +1300,14 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU rank0_bcast = TRUE; /* Read on rank 0 Bcast to other ranks */ - if (file->mpi_rank == 0) + if (file->mpi_rank == 0) { + /* If MPI_File_read_at fails, push an error, but continue + * to participate in following MPI_Bcast */ if (MPI_SUCCESS != (mpi_code = MPI_File_read_at(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) + HMPI_DONE_ERROR(FAIL, "MPI_File_read_at failed", mpi_code) + } + if (MPI_SUCCESS != (mpi_code = MPI_Bcast(buf, size_i, buf_type, 0, file->comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) } /* end if */ @@ -1293,6 +1337,21 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) } /* end if */ else { + if (size != (hsize_t)size_i) { + /* If HERE, then we need to work around the integer size limit + * of 2GB. The input size_t size variable cannot fit into an integer, + * but we can get around that limitation by creating a different datatype + * and then setting the integer size (or element count) to 1 when using + * the derived_type. + */ + + if (H5_mpio_create_large_type(size, 0, MPI_BYTE, &buf_type) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_CANTGET, FAIL, "can't create MPI-I/O datatype") + + derived_type = TRUE; + size_i = 1; + } + #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_r_flag) HDfprintf(stderr, "%s: (%d) doing MPI independent IO\n", __func__, file->mpi_rank); @@ -1307,11 +1366,21 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU if (!rank0_bcast || (rank0_bcast && file->mpi_rank == 0)) { /* How many bytes were actually read? */ #if MPI_VERSION >= 3 - if (MPI_SUCCESS != (mpi_code = MPI_Get_elements_x(&mpi_stat, buf_type, &bytes_read))) + if (MPI_SUCCESS != (mpi_code = MPI_Get_elements_x(&mpi_stat, buf_type, &bytes_read))) { #else - if (MPI_SUCCESS != (mpi_code = MPI_Get_elements(&mpi_stat, MPI_BYTE, &bytes_read))) + if (MPI_SUCCESS != (mpi_code = MPI_Get_elements(&mpi_stat, MPI_BYTE, &bytes_read))) { #endif - HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements failed", mpi_code) + if (rank0_bcast && file->mpi_rank == 0) { + /* If MPI_Get_elements(_x) fails for a rank 0 bcast strategy, + * push an error, but continue to participate in the following + * MPI_Bcast. + */ + bytes_read = -1; + HMPI_DONE_ERROR(FAIL, "MPI_Get_elements failed", mpi_code) + } + else + HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements failed", mpi_code) + } } /* end if */ /* If the rank0-bcast feature was used, broadcast the # of bytes read to @@ -1345,8 +1414,8 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_r_flag) - HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_read = %lld\n", __func__, file->mpi_rank, - (long)mpi_off, bytes_read); + HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_read = %lld type = %s\n", __func__, file->mpi_rank, + (long)mpi_off, bytes_read, H5FD__mem_t_to_str(type)); #endif /* @@ -1356,6 +1425,9 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU HDmemset((char *)buf + bytes_read, 0, (size_t)n); done: + if (derived_type) + MPI_Type_free(&buf_type); + #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_t_flag) HDfprintf(stderr, "%s: (%d) Leaving\n", __func__, file->mpi_rank); @@ -1468,20 +1540,6 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h */ mpi_off = 0; } /* end if */ - else if (size != (hsize_t)size_i) { - /* If HERE, then we need to work around the integer size limit - * of 2GB. The input size_t size variable cannot fit into an integer, - * but we can get around that limitation by creating a different datatype - * and then setting the integer size (or element count) to 1 when using - * the derived_type. - */ - - if (H5_mpio_create_large_type(size, 0, MPI_BYTE, &buf_type) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTGET, FAIL, "can't create MPI-I/O datatype") - - derived_type = TRUE; - size_i = 1; - } /* Write the data. */ if (use_view_this_time) { @@ -1527,6 +1585,21 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) } /* end if */ else { + if (size != (hsize_t)size_i) { + /* If HERE, then we need to work around the integer size limit + * of 2GB. The input size_t size variable cannot fit into an integer, + * but we can get around that limitation by creating a different datatype + * and then setting the integer size (or element count) to 1 when using + * the derived_type. + */ + + if (H5_mpio_create_large_type(size, 0, MPI_BYTE, &buf_type) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_CANTGET, FAIL, "can't create MPI-I/O datatype") + + derived_type = TRUE; + size_i = 1; + } + #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_w_flag) HDfprintf(stderr, "%s: (%d) doing MPI independent IO\n", __func__, file->mpi_rank); @@ -1562,8 +1635,8 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_w_flag) - HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_written = %lld\n", __func__, file->mpi_rank, - (long)mpi_off, bytes_written); + HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_written = %lld type = %s\n", __func__, + file->mpi_rank, (long)mpi_off, bytes_written, H5FD__mem_t_to_str(type)); #endif /* Each process will keep track of its perceived EOF value locally, and @@ -1701,17 +1774,19 @@ H5FD__mpio_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code) /* Only processor p0 will get the filesize and broadcast it. */ - /* (Note that throwing an error here will cause non-rank 0 processes - * to hang in following Bcast. -QAK, 3/17/2018) - */ - if (0 == file->mpi_rank) + if (0 == file->mpi_rank) { + /* If MPI_File_get_size fails, broadcast file size as -1 to signal error */ if (MPI_SUCCESS != (mpi_code = MPI_File_get_size(file->f, &size))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_get_size failed", mpi_code) + size = (MPI_Offset)-1; + } /* Broadcast file size */ if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&size, (int)sizeof(MPI_Offset), MPI_BYTE, 0, file->comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) + if (size < 0) + HMPI_GOTO_ERROR(FAIL, "MPI_File_get_size failed", mpi_code) + if (H5FD_mpi_haddr_to_MPIOff(file->eoa, &needed_eof) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "cannot convert from haddr_t to MPI_Offset") @@ -1774,11 +1849,16 @@ H5FD__mpio_delete(const char *filename, hid_t fapl_id) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") HDassert(H5FD_MPIO == H5P_peek_driver(plist)); - /* Get the MPI communicator and info from the fapl */ - if (H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI info object") - if (H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI communicator") + if (H5FD_mpi_self_initialized) { + comm = MPI_COMM_WORLD; + } + else { + /* Get the MPI communicator and info from the fapl */ + if (H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI info object") + if (H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get MPI communicator") + } /* Get the MPI rank of this process */ if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank(comm, &mpi_rank))) @@ -1789,96 +1869,101 @@ H5FD__mpio_delete(const char *filename, hid_t fapl_id) HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code) /* Delete the file */ - if (mpi_rank == 0) + if (mpi_rank == 0) { + /* If MPI_File_delete fails, push an error but + * still participate in the following MPI_Barrier + */ if (MPI_SUCCESS != (mpi_code = MPI_File_delete(filename, info))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_delete failed", mpi_code) + HMPI_DONE_ERROR(FAIL, "MPI_File_delete failed", mpi_code) + } /* Set up a barrier (don't want processes to run ahead of the delete) */ if (MPI_SUCCESS != (mpi_code = MPI_Barrier(comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code) done: + /* Free duplicated MPI Communicator and Info objects */ + if (H5_mpi_comm_free(&comm) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI communicator") + if (H5_mpi_info_free(&info) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI info object") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__mpio_delete() */ /*------------------------------------------------------------------------- - * Function: H5FD__mpio_mpi_rank + * Function: H5FD__mpio_ctl * - * Purpose: Returns the MPI rank for a process + * Purpose: MPIO version of the ctl callback. * - * Return: Success: non-negative - * Failure: negative + * The desired operation is specified by the op_code + * parameter. * - * Programmer: Quincey Koziol - * Thursday, May 16, 2002 + * The flags parameter controls management of op_codes that + * are unknown to the callback * - *------------------------------------------------------------------------- - */ -static int -H5FD__mpio_mpi_rank(const H5FD_t *_file) -{ - const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file; - - FUNC_ENTER_STATIC_NOERR - - /* Sanity checks */ - HDassert(file); - HDassert(H5FD_MPIO == file->pub.driver_id); - - FUNC_LEAVE_NOAPI(file->mpi_rank) -} /* end H5FD__mpio_mpi_rank() */ - -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_mpi_size + * The input and output parameters allow op_code specific + * input and output * - * Purpose: Returns the number of MPI processes + * At present, the supported op codes are: * - * Return: Success: non-negative - * Failure: negative + * H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE + * H5FD_CTL__GET_MPI_RANK_OPCODE + * H5FD_CTL__GET_MPI_SIZE_OPCODE * - * Programmer: Quincey Koziol - * Thursday, May 16, 2002 + * Note that these opcodes must be supported by all VFDs that + * support MPI. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: JRM -- 8/3/21 * *------------------------------------------------------------------------- */ -static int -H5FD__mpio_mpi_size(const H5FD_t *_file) +static herr_t +H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input, + void **output) { - const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file; + H5FD_mpio_t *file = (H5FD_mpio_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC_NOERR + FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ HDassert(file); HDassert(H5FD_MPIO == file->pub.driver_id); - FUNC_LEAVE_NOAPI(file->mpi_size) -} /* end H5FD__mpio_mpi_size() */ + switch (op_code) { -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_communicator - * - * Purpose: Returns the MPI communicator for the file. - * - * Return: Success: The communicator - * Failure: Can't fail - * - * Programmer: Robb Matzke - * Monday, August 9, 1999 - * - *------------------------------------------------------------------------- - */ -static MPI_Comm -H5FD__mpio_communicator(const H5FD_t *_file) -{ - const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file; + case H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE: + HDassert(output); + HDassert(*output); + **((MPI_Comm **)output) = file->comm; + break; - FUNC_ENTER_STATIC_NOERR + case H5FD_CTL__GET_MPI_RANK_OPCODE: + HDassert(output); + HDassert(*output); + **((int **)output) = file->mpi_rank; + break; - /* Sanity checks */ - HDassert(file); - HDassert(H5FD_MPIO == file->pub.driver_id); + case H5FD_CTL__GET_MPI_SIZE_OPCODE: + HDassert(output); + HDassert(*output); + **((int **)output) = file->mpi_size; + break; + + default: /* unknown op code */ + if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) { + + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown") + } + break; + } + +done: + + FUNC_LEAVE_NOAPI(ret_value) - FUNC_LEAVE_NOAPI(file->comm) -} /* end H5FD__mpio_communicator() */ +} /* end H5FD__mpio_ctl() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5FDmpio.h b/src/H5FDmpio.h index 8caf11c..5ce98ca 100644 --- a/src/H5FDmpio.h +++ b/src/H5FDmpio.h @@ -23,7 +23,7 @@ /* Macros */ #ifdef H5_HAVE_PARALLEL -#define H5FD_MPIO (H5FD_mpio_init()) +#define H5FD_MPIO (H5FDperform_init(H5FD_mpio_init)) #else #define H5FD_MPIO (H5I_INVALID_HID) #endif /* H5_HAVE_PARALLEL */ @@ -223,7 +223,7 @@ H5_DLL herr_t H5Pset_dxpl_mpio_collective_opt(hid_t dxpl_id, H5FD_mpio_collectiv * * Use of this function is optional. * - * \todo Add missing version information + * \since 1.8.0 * */ H5_DLL herr_t H5Pset_dxpl_mpio_chunk_opt(hid_t dxpl_id, H5FD_mpio_chunk_opt_t opt_mode); @@ -247,7 +247,7 @@ H5_DLL herr_t H5Pset_dxpl_mpio_chunk_opt(hid_t dxpl_id, H5FD_mpio_chunk_opt_t op * otherwise, a separate I/O process will be invoked for each chunk * (multi-chunk I/O). * - * \todo Add missing version information + * \since 1.8.0 * */ H5_DLL herr_t H5Pset_dxpl_mpio_chunk_opt_num(hid_t dxpl_id, unsigned num_chunk_per_proc); @@ -272,7 +272,7 @@ H5_DLL herr_t H5Pset_dxpl_mpio_chunk_opt_num(hid_t dxpl_id, unsigned num_chunk_p * percent_proc_per_chunk, the library will do collective I/O for this * chunk; otherwise, independent I/O will be done for the chunk. * - * \todo Add missing version information + * \since 1.8.0 * */ H5_DLL herr_t H5Pset_dxpl_mpio_chunk_opt_ratio(hid_t dxpl_id, unsigned percent_num_proc_per_chunk); diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index cae4174..d9a6ce9 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -135,8 +135,13 @@ typedef struct H5FD_multi_dxpl_t { } H5FD_multi_dxpl_t; /* Private functions */ -static int compute_next(H5FD_multi_t *file); -static int open_members(H5FD_multi_t *file); +static herr_t H5FD_split_populate_config(const char *meta_ext, hid_t meta_plist_id, const char *raw_ext, + hid_t raw_plist_id, hbool_t relax, H5FD_multi_fapl_t *fa_out); +static herr_t H5FD_multi_populate_config(const H5FD_mem_t *memb_map, const hid_t *memb_fapl, + const char *const *memb_name, const haddr_t *memb_addr, + hbool_t relax, H5FD_multi_fapl_t *fa_out); +static int compute_next(H5FD_multi_t *file); +static int open_members(H5FD_multi_t *file); /* Callback prototypes */ static herr_t H5FD_multi_term(void); @@ -166,9 +171,12 @@ static herr_t H5FD_multi_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing static herr_t H5FD_multi_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD_multi_unlock(H5FD_t *_file); static herr_t H5FD_multi_delete(const char *filename, hid_t fapl_id); +static herr_t H5FD_multi_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, + void **output); /* The class struct */ static const H5FD_class_t H5FD_multi_g = { + H5_VFD_MULTI, /* value */ "multi", /* name */ HADDR_MAX, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -201,6 +209,7 @@ static const H5FD_class_t H5FD_multi_g = { H5FD_multi_lock, /* lock */ H5FD_multi_unlock, /* unlock */ H5FD_multi_delete, /* del */ + H5FD_multi_ctl, /* ctl */ H5FD_FLMAP_DEFAULT /* fl_map */ }; @@ -271,74 +280,19 @@ herr_t H5Pset_fapl_split(hid_t fapl, const char *meta_ext, hid_t meta_plist_id, const char *raw_ext, hid_t raw_plist_id) { - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; - hid_t memb_fapl[H5FD_MEM_NTYPES]; - const char *memb_name[H5FD_MEM_NTYPES]; - char meta_name[H5FD_MULT_MAX_FILE_NAME_LEN]; - char raw_name[H5FD_MULT_MAX_FILE_NAME_LEN]; - haddr_t memb_addr[H5FD_MEM_NTYPES]; + H5FD_multi_fapl_t fa; + static const char *func = "H5Pset_fapl_split"; /* Function Name for error reporting */ /*NO TRACE*/ /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); - /* Initialize */ - ALL_MEMBERS (mt) { - /* Treat global heap as raw data, not metadata */ - memb_map[mt] = ((mt == H5FD_MEM_DRAW || mt == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : H5FD_MEM_SUPER); - memb_fapl[mt] = -1; - memb_name[mt] = NULL; - memb_addr[mt] = HADDR_UNDEF; - } - END_MEMBERS; - - /* The file access properties */ - memb_fapl[H5FD_MEM_SUPER] = meta_plist_id; - memb_fapl[H5FD_MEM_DRAW] = raw_plist_id; - - /* The names */ - /* process meta filename */ - if (meta_ext) { - if (strstr(meta_ext, "%s")) { - /* Note: this doesn't accommodate for when the '%s' in the user's - * string is at a position >sizeof(meta_name) - QK & JK - 2013/01/17 - */ - strncpy(meta_name, meta_ext, sizeof(meta_name)); - meta_name[sizeof(meta_name) - 1] = '\0'; - } - else - sprintf(meta_name, "%%s%s", meta_ext); - } - else { - strncpy(meta_name, "%s.meta", sizeof(meta_name)); - meta_name[sizeof(meta_name) - 1] = '\0'; - } - memb_name[H5FD_MEM_SUPER] = meta_name; - - /* process raw filename */ - if (raw_ext) { - if (strstr(raw_ext, "%s")) { - /* Note: this doesn't accommodate for when the '%s' in the user's - * string is at a position >sizeof(raw_name) - QK & JK - 2013/01/17 - */ - strncpy(raw_name, raw_ext, sizeof(raw_name)); - raw_name[sizeof(raw_name) - 1] = '\0'; - } - else - sprintf(raw_name, "%%s%s", raw_ext); - } - else { - strncpy(raw_name, "%s.raw", sizeof(raw_name)); - raw_name[sizeof(raw_name) - 1] = '\0'; - } - memb_name[H5FD_MEM_DRAW] = raw_name; - - /* The sizes */ - memb_addr[H5FD_MEM_SUPER] = 0; - memb_addr[H5FD_MEM_DRAW] = HADDR_MAX / 2; + if (H5FD_split_populate_config(meta_ext, meta_plist_id, raw_ext, raw_plist_id, TRUE, &fa) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_CANTSET, "can't setup split driver configuration", + -1); - return H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, TRUE); + return H5Pset_driver(fapl, H5FD_MULTI, &fa); } /*------------------------------------------------------------------------- @@ -422,14 +376,7 @@ H5Pset_fapl_multi(hid_t fapl_id, const H5FD_mem_t *memb_map, const hid_t *memb_f const char *const *memb_name, const haddr_t *memb_addr, hbool_t relax) { H5FD_multi_fapl_t fa; - H5FD_mem_t mt, mmt; - H5FD_mem_t _memb_map[H5FD_MEM_NTYPES]; - hid_t _memb_fapl[H5FD_MEM_NTYPES]; - char _memb_name[H5FD_MEM_NTYPES][16]; - const char * _memb_name_ptrs[H5FD_MEM_NTYPES]; - haddr_t _memb_addr[H5FD_MEM_NTYPES]; - static const char *letters = "Xsbrglo"; - static const char *func = "H5FDset_fapl_multi"; /* Function Name for error reporting */ + static const char *func = "H5FDset_fapl_multi"; /* Function Name for error reporting */ /*NO TRACE*/ @@ -439,66 +386,9 @@ H5Pset_fapl_multi(hid_t fapl_id, const H5FD_mem_t *memb_map, const hid_t *memb_f /* Check arguments and supply default values */ if (H5I_GENPROP_LST != H5Iget_type(fapl_id) || TRUE != H5Pisa_class(fapl_id, H5P_FILE_ACCESS)) H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADVALUE, "not an access list", -1); - if (!memb_map) { - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) - _memb_map[mt] = H5FD_MEM_DEFAULT; - memb_map = _memb_map; - } - if (!memb_fapl) { - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) - _memb_fapl[mt] = H5Pcreate(H5P_FILE_ACCESS); - memb_fapl = _memb_fapl; - } - if (!memb_name) { - assert(strlen(letters) == H5FD_MEM_NTYPES); - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) { - sprintf(_memb_name[mt], "%%s-%c.h5", letters[mt]); - _memb_name_ptrs[mt] = _memb_name[mt]; - } - memb_name = _memb_name_ptrs; - } - if (!memb_addr) { - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) - _memb_addr[mt] = (hsize_t)(mt ? (mt - 1) : 0) * (HADDR_MAX / (H5FD_MEM_NTYPES - 1)); - memb_addr = _memb_addr; - } - - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) { - /* Map usage type */ - mmt = memb_map[mt]; - if (mmt < 0 || mmt >= H5FD_MEM_NTYPES) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADRANGE, "file resource type out of range", -1); - if (H5FD_MEM_DEFAULT == mmt) - mmt = mt; - - /* - * All members of MEMB_FAPL must be either defaults or actual file - * access property lists. - */ - if (H5P_DEFAULT != memb_fapl[mmt] && TRUE != H5Pisa_class(memb_fapl[mmt], H5P_FILE_ACCESS)) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "file resource type incorrect", -1); - - /* All names must be defined */ - if (!memb_name[mmt] || !memb_name[mmt][0]) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "file resource type not set", -1); - } - - /* - * Initialize driver specific information. No need to copy it into the FA - * struct since all members will be copied by H5Pset_driver(). - */ - memset(&fa, 0, sizeof(H5FD_multi_fapl_t)); - memcpy(fa.memb_map, memb_map, H5FD_MEM_NTYPES * sizeof(H5FD_mem_t)); - memcpy(fa.memb_fapl, memb_fapl, H5FD_MEM_NTYPES * sizeof(hid_t)); - memcpy(fa.memb_name, memb_name, H5FD_MEM_NTYPES * sizeof(char *)); - memcpy(fa.memb_addr, memb_addr, H5FD_MEM_NTYPES * sizeof(haddr_t)); - fa.relax = relax; + if (H5FD_multi_populate_config(memb_map, memb_fapl, memb_name, memb_addr, relax, &fa) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_CANTSET, "can't setup driver configuration", -1); - /* Patch up H5P_DEFAULT property lists for members */ - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) { - if (fa.memb_fapl[mt] == H5P_DEFAULT) - fa.memb_fapl[mt] = H5Pcreate(H5P_FILE_ACCESS); - } return H5Pset_driver(fapl_id, H5FD_MULTI, &fa); } @@ -523,6 +413,7 @@ H5Pget_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map /*out*/, hid_t *memb_fapl char **memb_name /*out*/, haddr_t *memb_addr /*out*/, hbool_t *relax) { const H5FD_multi_fapl_t *fa; + H5FD_multi_fapl_t default_fa; H5FD_mem_t mt; static const char * func = "H5FDget_fapl_multi"; /* Function Name for error reporting */ @@ -535,8 +426,17 @@ H5Pget_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map /*out*/, hid_t *memb_fapl H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADTYPE, "not an access list", -1); if (H5FD_MULTI != H5Pget_driver(fapl_id)) H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADVALUE, "incorrect VFL driver", -1); - if (NULL == (fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id))) - H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADVALUE, "bad VFL driver info", -1); + H5E_BEGIN_TRY + { + fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); + } + H5E_END_TRY; + if (!fa || (H5P_FILE_ACCESS_DEFAULT == fapl_id)) { + if (H5FD_multi_populate_config(NULL, NULL, NULL, NULL, TRUE, &default_fa) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTSET, "can't setup default driver configuration", + -1); + fa = &default_fa; + } if (memb_map) memcpy(memb_map, fa->memb_map, H5FD_MEM_NTYPES * sizeof(H5FD_mem_t)); @@ -565,6 +465,231 @@ H5Pget_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map /*out*/, hid_t *memb_fapl } /*------------------------------------------------------------------------- + * Function: H5FD_split_populate_config + * + * Purpose: Populates a H5FD_multi_fapl_t structure with the provided + * split driver values, supplying defaults where values are not + * provided. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_split_populate_config(const char *meta_ext, hid_t meta_plist_id, const char *raw_ext, hid_t raw_plist_id, + hbool_t relax, H5FD_multi_fapl_t *fa_out) +{ + static const char *func = "H5FD_split_populate_config"; /* Function Name for error reporting */ + static char + meta_name_g[H5FD_MULT_MAX_FILE_NAME_LEN]; /* Static scratch buffer to store metadata member name */ + static char + raw_name_g[H5FD_MULT_MAX_FILE_NAME_LEN]; /* Static scratch buffer to store raw data member name */ + const char *_memb_name[H5FD_MEM_NTYPES]; + H5FD_mem_t _memb_map[H5FD_MEM_NTYPES]; + hid_t _memb_fapl[H5FD_MEM_NTYPES]; + haddr_t _memb_addr[H5FD_MEM_NTYPES]; + herr_t ret_value = 0; + + assert(fa_out); + + /* Initialize */ + ALL_MEMBERS (mt) { + /* Treat global heap as raw data, not metadata */ + _memb_map[mt] = ((mt == H5FD_MEM_DRAW || mt == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : H5FD_MEM_SUPER); + _memb_fapl[mt] = H5P_DEFAULT; + _memb_name[mt] = NULL; + _memb_addr[mt] = HADDR_UNDEF; + } + END_MEMBERS; + + /* The file access properties */ + _memb_fapl[H5FD_MEM_SUPER] = meta_plist_id; + _memb_fapl[H5FD_MEM_DRAW] = raw_plist_id; + + /* The names */ + /* process meta filename */ + if (meta_ext) { + if (strstr(meta_ext, "%s")) { + /* Note: this doesn't accommodate for when the '%s' in the user's + * string is at a position >sizeof(meta_name) - QK & JK - 2013/01/17 + */ + strncpy(meta_name_g, meta_ext, sizeof(meta_name_g)); + meta_name_g[sizeof(meta_name_g) - 1] = '\0'; + } + else + snprintf(meta_name_g, sizeof(meta_name_g), "%%s%s", meta_ext); + } + else { + strncpy(meta_name_g, "%s.meta", sizeof(meta_name_g)); + meta_name_g[sizeof(meta_name_g) - 1] = '\0'; + } + _memb_name[H5FD_MEM_SUPER] = meta_name_g; + + /* process raw filename */ + if (raw_ext) { + if (strstr(raw_ext, "%s")) { + /* Note: this doesn't accommodate for when the '%s' in the user's + * string is at a position >sizeof(raw_name) - QK & JK - 2013/01/17 + */ + strncpy(raw_name_g, raw_ext, sizeof(raw_name_g)); + raw_name_g[sizeof(raw_name_g) - 1] = '\0'; + } + else + snprintf(raw_name_g, sizeof(raw_name_g), "%%s%s", raw_ext); + } + else { + strncpy(raw_name_g, "%s.raw", sizeof(raw_name_g)); + raw_name_g[sizeof(raw_name_g) - 1] = '\0'; + } + _memb_name[H5FD_MEM_DRAW] = raw_name_g; + + /* The sizes */ + _memb_addr[H5FD_MEM_SUPER] = 0; + _memb_addr[H5FD_MEM_DRAW] = HADDR_MAX / 2; + + ALL_MEMBERS (mt) { + /* Map usage type */ + H5FD_mem_t mmt = _memb_map[mt]; + if (mmt < 0 || mmt >= H5FD_MEM_NTYPES) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADRANGE, "file resource type out of range", -1); + + /* + * All members of MEMB_FAPL must be either defaults or actual file + * access property lists. + */ + if (H5P_DEFAULT != _memb_fapl[mmt] && TRUE != H5Pisa_class(_memb_fapl[mmt], H5P_FILE_ACCESS)) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "file resource type incorrect", -1); + + /* All names must be defined */ + if (!_memb_name[mmt] || !_memb_name[mmt][0]) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "file resource type not set", -1); + } + END_MEMBERS; + + /* + * Initialize driver specific information. No need to copy it into the FA + * struct since all members will be copied by H5Pset_driver(). + */ + memset(fa_out, 0, sizeof(H5FD_multi_fapl_t)); + memcpy(fa_out->memb_map, _memb_map, H5FD_MEM_NTYPES * sizeof(H5FD_mem_t)); + memcpy(fa_out->memb_fapl, _memb_fapl, H5FD_MEM_NTYPES * sizeof(hid_t)); + memcpy(fa_out->memb_name, _memb_name, H5FD_MEM_NTYPES * sizeof(char *)); + memcpy(fa_out->memb_addr, _memb_addr, H5FD_MEM_NTYPES * sizeof(haddr_t)); + fa_out->relax = relax; + + /* Patch up H5P_DEFAULT property lists for members */ + ALL_MEMBERS (mt) { + if (fa_out->memb_fapl[mt] == H5P_DEFAULT) { + fa_out->memb_fapl[mt] = H5Pcreate(H5P_FILE_ACCESS); + if (H5Pset_fapl_sec2(fa_out->memb_fapl[mt]) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_CANTSET, + "can't set sec2 driver on member FAPL", -1); + } + } + END_MEMBERS; + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: H5FD_multi_populate_config + * + * Purpose: Populates a H5FD_multi_fapl_t structure with the provided + * values, supplying defaults where values are not provided. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_multi_populate_config(const H5FD_mem_t *memb_map, const hid_t *memb_fapl, const char *const *memb_name, + const haddr_t *memb_addr, hbool_t relax, H5FD_multi_fapl_t *fa_out) +{ + static const char *func = "H5FD_multi_populate_config"; /* Function Name for error reporting */ + static const char *letters = "Xsbrglo"; + static char _memb_name_g[H5FD_MEM_NTYPES][16]; /* Static scratch buffer to store member names */ + H5FD_mem_t mt, mmt; + H5FD_mem_t _memb_map[H5FD_MEM_NTYPES]; + hid_t _memb_fapl[H5FD_MEM_NTYPES]; + const char * _memb_name_ptrs[H5FD_MEM_NTYPES]; + haddr_t _memb_addr[H5FD_MEM_NTYPES]; + herr_t ret_value = 0; + + assert(fa_out); + + if (!memb_map) { + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) + _memb_map[mt] = H5FD_MEM_DEFAULT; + memb_map = _memb_map; + } + if (!memb_fapl) { + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) { + _memb_fapl[mt] = H5Pcreate(H5P_FILE_ACCESS); + if (H5Pset_fapl_sec2(_memb_fapl[mt]) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_CANTSET, + "can't set sec2 driver on member FAPL", -1); + } + memb_fapl = _memb_fapl; + } + if (!memb_name) { + assert(strlen(letters) == H5FD_MEM_NTYPES); + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) { + snprintf(_memb_name_g[mt], 16, "%%s-%c.h5", letters[mt]); + _memb_name_ptrs[mt] = _memb_name_g[mt]; + } + memb_name = _memb_name_ptrs; + } + if (!memb_addr) { + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) + _memb_addr[mt] = (hsize_t)(mt ? (mt - 1) : 0) * (HADDR_MAX / (H5FD_MEM_NTYPES - 1)); + memb_addr = _memb_addr; + } + + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) { + /* Map usage type */ + mmt = memb_map[mt]; + if (mmt < 0 || mmt >= H5FD_MEM_NTYPES) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADRANGE, "file resource type out of range", -1); + if (H5FD_MEM_DEFAULT == mmt) + mmt = mt; + + /* + * All members of MEMB_FAPL must be either defaults or actual file + * access property lists. + */ + if (H5P_DEFAULT != memb_fapl[mmt] && TRUE != H5Pisa_class(memb_fapl[mmt], H5P_FILE_ACCESS)) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "file resource type incorrect", -1); + + /* All names must be defined */ + if (!memb_name[mmt] || !memb_name[mmt][0]) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "file resource type not set", -1); + } + + /* + * Initialize driver specific information. No need to copy it into the FA + * struct since all members will be copied by H5Pset_driver(). + */ + memset(fa_out, 0, sizeof(H5FD_multi_fapl_t)); + memcpy(fa_out->memb_map, memb_map, H5FD_MEM_NTYPES * sizeof(H5FD_mem_t)); + memcpy(fa_out->memb_fapl, memb_fapl, H5FD_MEM_NTYPES * sizeof(hid_t)); + memcpy(fa_out->memb_name, memb_name, H5FD_MEM_NTYPES * sizeof(char *)); + memcpy(fa_out->memb_addr, memb_addr, H5FD_MEM_NTYPES * sizeof(haddr_t)); + fa_out->relax = relax; + + /* Patch up H5P_DEFAULT property lists for members */ + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt = (H5FD_mem_t)(mt + 1)) { + if (fa_out->memb_fapl[mt] == H5P_DEFAULT) { + fa_out->memb_fapl[mt] = H5Pcreate(H5P_FILE_ACCESS); + if (H5Pset_fapl_sec2(fa_out->memb_fapl[mt]) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_CANTSET, + "can't set sec2 driver on member FAPL", -1); + } + } + + return ret_value; +} /* end H5FD_multi_populate_config() */ + +/*------------------------------------------------------------------------- * Function: H5FD_multi_sb_size * * Purpose: Returns the size of the private information to be stored in @@ -1009,12 +1134,26 @@ H5FD_multi_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr */ if (NULL == (file = (H5FD_multi_t *)calloc((size_t)1, sizeof(H5FD_multi_t)))) H5Epush_ret(func, H5E_ERR_CLS, H5E_RESOURCE, H5E_NOSPACE, "memory allocation failed", NULL); - if (H5P_FILE_ACCESS_DEFAULT == fapl_id || H5FD_MULTI != H5Pget_driver(fapl_id)) { + H5E_BEGIN_TRY + { + fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); + } + H5E_END_TRY; + if (!fa || (H5P_FILE_ACCESS_DEFAULT == fapl_id) || (H5FD_MULTI != H5Pget_driver(fapl_id))) { + char *env = getenv(HDF5_DRIVER); + close_fapl = fapl_id = H5Pcreate(H5P_FILE_ACCESS); - if (H5Pset_fapl_multi(fapl_id, NULL, NULL, NULL, NULL, TRUE) < 0) - H5Epush_goto(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTSET, "can't set property value", error) + if (env && !strcmp(env, "split")) { + if (H5Pset_fapl_split(fapl_id, NULL, H5P_DEFAULT, NULL, H5P_DEFAULT) < 0) + H5Epush_goto(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTSET, "can't set property value", error) + } + else { + if (H5Pset_fapl_multi(fapl_id, NULL, NULL, NULL, NULL, TRUE) < 0) + H5Epush_goto(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTSET, "can't set property value", error) + } + + fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); } - fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); assert(fa); ALL_MEMBERS (mt) { file->fa.memb_map[mt] = fa->memb_map[mt]; @@ -2037,6 +2176,7 @@ H5FD_multi_delete(const char *filename, hid_t fapl_id) char full_filename[H5FD_MULT_MAX_FILE_NAME_LEN]; int nchars; const H5FD_multi_fapl_t *fa; + H5FD_multi_fapl_t default_fa; static const char * func = "H5FD_multi_delete"; /* Function Name for error reporting */ /* Clear the error stack */ @@ -2044,11 +2184,26 @@ H5FD_multi_delete(const char *filename, hid_t fapl_id) assert(filename); - /* Quiet compiler */ - (void)fapl_id; - /* Get the driver info */ - fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); + H5E_BEGIN_TRY + { + fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); + } + H5E_END_TRY; + if (!fa) { + char *env = getenv(HDF5_DRIVER); + + if (env && !strcmp(env, "split")) { + if (H5FD_split_populate_config(NULL, H5P_DEFAULT, NULL, H5P_DEFAULT, TRUE, &default_fa) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTSET, "can't setup driver configuration", -1); + } + else { + if (H5FD_multi_populate_config(NULL, NULL, NULL, NULL, TRUE, &default_fa) < 0) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTSET, "can't setup driver configuration", -1); + } + + fa = &default_fa; + } assert(fa); /* Delete each member file using the underlying fapl */ @@ -2070,6 +2225,54 @@ H5FD_multi_delete(const char *filename, hid_t fapl_id) } /* end H5FD_multi_delete() */ H5_MULTI_GCC_DIAG_ON("format-nonliteral") +/*------------------------------------------------------------------------- + * Function: H5FD_multi_ctl + * + * Purpose: Multi VFD version of the ctl callback. + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * At present, this VFD supports no op codes of its own. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_multi_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, void **output) +{ + H5FD_multi_t * file = (H5FD_multi_t *)_file; + static const char *func = "H5FD_multi_ctl"; /* Function Name for error reporting */ + herr_t ret_value = 0; + + /* Silence compiler */ + (void)file; + (void)input; + (void)output; + + /* Clear the error stack */ + H5Eclear2(H5E_DEFAULT); + + switch (op_code) { + /* Unknown op code */ + default: + if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_FCNTL, + "VFD ctl request failed (unknown op code and fail if unknown flag is set)", -1); + + break; + } + + return ret_value; +} /* end H5FD_multi_ctl() */ + #ifdef H5private_H /* * This is not related to the functionality of the driver code. diff --git a/src/H5FDmulti.h b/src/H5FDmulti.h index 62cc9c8..7a01f4c 100644 --- a/src/H5FDmulti.h +++ b/src/H5FDmulti.h @@ -20,7 +20,7 @@ #ifndef H5FDmulti_H #define H5FDmulti_H -#define H5FD_MULTI (H5FD_multi_init()) +#define H5FD_MULTI (H5FDperform_init(H5FD_multi_init)) #ifdef __cplusplus extern "C" { diff --git a/src/H5FDperform.c b/src/H5FDperform.c new file mode 100644 index 0000000..096fdd6 --- /dev/null +++ b/src/H5FDperform.c @@ -0,0 +1,58 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5FDmodule.h" /* This source code file is part of the H5FD module */ + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FDpkg.h" /* File Drivers */ +#include "H5Iprivate.h" /* IDs */ + +/*------------------------------------------------------------------------- + * Function: H5FDperform_init + * + * Purpose: Ensure that the library is initialized and then call + * the provided VFD initializer. + * + * Return: Success: identifier for the VFD just initialized + * Failure: H5I_INVALID_HID + *------------------------------------------------------------------------- + */ +hid_t H5FDperform_init(hid_t (*init)(void)) +{ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_API_NOINIT + /*NO TRACE*/ + + /* It is possible that an application will evaluate an + * `H5FD_*` symbol (`H5FD_FAMILY`, `H5FD_MULTI`, `H5FD_SEC2`, et + * cetera) before the library has had an opportunity to initialize. + * Call H5_init_library() to make sure that the library has been + * initialized before `init` is run. + */ + if (H5_init_library() < 0) { + HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, H5I_INVALID_HID, "library initialization failed") + } + + ret_value = init(); +done: + FUNC_LEAVE_API_NOINIT(ret_value) +} diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 7c0dd77..4ba7060 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -88,7 +88,7 @@ * * struct H5FD_vfd_swmr_idx_entry_t * - * Indicies into the VFD SWMR metadata file are maintained in arrays of + * Indices into the VFD SWMR metadata file are maintained in arrays of * instances of H5FD_vfd_swmr_index_t. * * The fields of H5FD_vfd_swmr_idx_entry_t are discussed below. @@ -182,7 +182,7 @@ typedef struct H5FD_vfd_swmr_idx_entry_t { * tick_num: Sequence number of the current tick. * Initialized to zero on file creation/open, and incremented * by the VFD SWMR writer at the end of each tick. - * num_entries: The number of entires in the index. + * num_entries: The number of entries in the index. * entries: The array of index entries */ typedef struct H5FD_vfd_swmr_md_index { @@ -259,13 +259,6 @@ vfd_swmr_pageno_to_mdf_idx_entry(H5FD_vfd_swmr_idx_entry_t *idx, uint32_t nentri /* Definitions for file MPI type property */ #define H5FD_MPI_XFER_FILE_MPI_TYPE_NAME "H5FD_mpi_file_mpi_type" -/* Sub-class the H5FD_class_t to add more specific functions for MPI-based VFDs */ -typedef struct H5FD_class_mpi_t { - H5FD_class_t super; /* Superclass information & methods */ - int (*get_rank)(const H5FD_t *file); /* Get the MPI rank of a process */ - int (*get_size)(const H5FD_t *file); /* Get the MPI size of a communicator */ - MPI_Comm (*get_comm)(const H5FD_t *file); /* Get the communicator for a file */ -} H5FD_class_mpi_t; #endif /****************************/ @@ -302,12 +295,19 @@ typedef struct { } \ } -/* Define structure to hold driver ID & info for FAPLs */ +/* Define structure to hold driver ID, info & configuration string for FAPLs */ typedef struct { - hid_t driver_id; /* Driver's ID */ - const void *driver_info; /* Driver info, for open callbacks */ + hid_t driver_id; /* Driver's ID */ + const void *driver_info; /* Driver info, for open callbacks */ + const char *driver_config_str; /* Driver configuration string */ } H5FD_driver_prop_t; +/* Which kind of VFD field to use for searching */ +typedef enum H5FD_get_driver_kind_t { + H5FD_GET_DRIVER_BY_NAME, /* Name field is set */ + H5FD_GET_DRIVER_BY_VALUE /* Value field is set */ +} H5FD_get_driver_kind_t; + /*****************************/ /* Library Private Variables */ /*****************************/ @@ -318,6 +318,7 @@ typedef struct { /* Forward declarations for prototype arguments */ struct H5F_t; +union H5PL_key_t; H5_DLL int H5FD_term_interface(void); H5_DLL herr_t H5FD_locate_signature(H5FD_t *file, haddr_t *sig_addr); @@ -328,11 +329,18 @@ H5_DLL herr_t H5FD_sb_load(H5FD_t *file, const char *name, const uint8_t H5_DLL void * H5FD_fapl_get(H5FD_t *file); H5_DLL herr_t H5FD_free_driver_info(hid_t driver_id, const void *driver_info); H5_DLL hid_t H5FD_register(const void *cls, size_t size, hbool_t app_ref); +H5_DLL hid_t H5FD_register_driver_by_name(const char *name, hbool_t app_ref); +H5_DLL hid_t H5FD_register_driver_by_value(H5FD_class_value_t value, hbool_t app_ref); +H5_DLL htri_t H5FD_is_driver_registered_by_name(const char *driver_name, hid_t *registered_id); +H5_DLL htri_t H5FD_is_driver_registered_by_value(H5FD_class_value_t driver_value, hid_t *registered_id); +H5_DLL hid_t H5FD_get_driver_id_by_name(const char *name, hbool_t is_api); +H5_DLL hid_t H5FD_get_driver_id_by_value(H5FD_class_value_t value, hbool_t is_api); H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); H5FD_t * H5FD_deduplicate(H5FD_t *, hid_t); H5_DLL herr_t H5FD_close(H5FD_t *file); H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2); H5_DLL herr_t H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags /*out*/); +H5_DLL herr_t H5FD_check_plugin_load(const H5FD_class_t *cls, const union H5PL_key_t *key, hbool_t *success); H5_DLL haddr_t H5FD_alloc(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size); H5_DLL herr_t H5FD_free(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f, haddr_t addr, hsize_t size); @@ -352,13 +360,14 @@ H5_DLL herr_t H5FD_truncate(H5FD_t *file, hbool_t closing); H5_DLL herr_t H5FD_lock(H5FD_t *file, hbool_t rw); H5_DLL herr_t H5FD_unlock(H5FD_t *file); H5_DLL herr_t H5FD_delete(const char *name, hid_t fapl_id); +H5_DLL herr_t H5FD_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output); H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum); H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle); H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); -H5_DLL herr_t H5FD_get_driver_name(const H5FD_t *file, char **driver_name); +H5_DLL herr_t H5FD_init(void); /* Function prototypes for VFD SWMR */ H5_DLL int shadow_image_defer_free(struct H5F_shared_t *, const H5FD_vfd_swmr_idx_entry_t *); H5_DLL herr_t H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t read_index, uint64_t *tick_ptr, @@ -382,9 +391,9 @@ H5_DLL herr_t H5FD_set_mpio_atomicity(H5FD_t *file, hbool_t flag); H5_DLL herr_t H5FD_get_mpio_atomicity(H5FD_t *file, hbool_t *flag); /* Driver specific methods */ -H5_DLL int H5FD_mpi_get_rank(const H5FD_t *file); -H5_DLL int H5FD_mpi_get_size(const H5FD_t *file); -H5_DLL MPI_Comm H5FD_mpi_get_comm(const H5FD_t *_file); +H5_DLL int H5FD_mpi_get_rank(H5FD_t *file); +H5_DLL int H5FD_mpi_get_size(H5FD_t *file); +H5_DLL MPI_Comm H5FD_mpi_get_comm(H5FD_t *file); #endif /* H5_HAVE_PARALLEL */ #endif /* H5FDprivate_H */ diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 0cfb072..a8d85c6 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -28,6 +28,40 @@ #define H5FD_VFD_DEFAULT 0 /* Default VFL driver value */ +/* VFD identifier values + * These are H5FD_class_value_t values, NOT hid_t values! + */ +#define H5_VFD_INVALID ((H5FD_class_value_t)(-1)) +#define H5_VFD_SEC2 ((H5FD_class_value_t)(0)) +#define H5_VFD_CORE ((H5FD_class_value_t)(1)) +#define H5_VFD_LOG ((H5FD_class_value_t)(2)) +#define H5_VFD_FAMILY ((H5FD_class_value_t)(3)) +#define H5_VFD_MULTI ((H5FD_class_value_t)(4)) +#define H5_VFD_STDIO ((H5FD_class_value_t)(5)) +#define H5_VFD_SPLITTER ((H5FD_class_value_t)(6)) +#ifdef H5_HAVE_PARALLEL +#define H5_VFD_MPIO ((H5FD_class_value_t)(7)) +#endif +#ifdef H5_HAVE_DIRECT +#define H5_VFD_DIRECT ((H5FD_class_value_t)(8)) +#endif +#ifdef H5_HAVE_MIRROR_VFD +#define H5_VFD_MIRROR ((H5FD_class_value_t)(9)) +#endif +#ifdef H5_HAVE_LIBHDFS +#define H5_VFD_HDFS ((H5FD_class_value_t)(10)) +#endif +#ifdef H5_HAVE_ROS3_VFD +#define H5_VFD_ROS3 ((H5FD_class_value_t)(11)) +#endif +#define H5_VFD_SWMR ((H5FD_class_value_t)(12)) + +/* VFD IDs below this value are reserved for library use. */ +#define H5_VFD_RESERVED 256 + +/* Maximum VFD ID */ +#define H5_VFD_MAX 65535 + /* Define VFL driver features that can be enabled on a per-driver basis */ /* These are returned with the 'query' function pointer in H5FD_class_t */ /* @@ -137,11 +171,103 @@ * enabled may be used as the Write-Only (W/O) channel driver. */ #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000 +/* + * Defining H5FD_FEAT_MEMMANAGE for a VFL driver means that + * the driver uses special memory management routines or wishes + * to do memory management in a specific manner. Therefore, HDF5 + * should request that the driver handle any memory management + * operations when appropriate. + */ +#define H5FD_FEAT_MEMMANAGE 0x00010000 + +/* ctl function definitions: */ +#define H5FD_CTL_OPC_RESERVED 512 /* Opcodes below this value are reserved for library use */ +#define H5FD_CTL_OPC_EXPER_MIN \ + H5FD_CTL_OPC_RESERVED /* Minimum opcode value available for experimental use \ + */ +#define H5FD_CTL_OPC_EXPER_MAX \ + (H5FD_CTL_OPC_RESERVED + 511) /* Maximum opcode value available for experimental use */ + +/* ctl function op codes: */ +#define H5FD_CTL__INVALID_OPCODE 0 +#define H5FD_CTL__TEST_OPCODE 1 +#define H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE 2 +#define H5FD_CTL__GET_MPI_RANK_OPCODE 3 +#define H5FD_CTL__GET_MPI_SIZE_OPCODE 4 +#define H5FD_CTL__MEM_ALLOC 5 +#define H5FD_CTL__MEM_FREE 6 +#define H5FD_CTL__MEM_COPY 7 + +/* ctl function flags: */ + +/* Definitions: + * + * WARNING: While the following definitions of Terminal + * and Passthrough VFDs should be workable for now, they + * have to be adjusted as our use cases for VFDs expand. + * + * JRM -- 8/4/21 + * + * + * Terminal VFD: Lowest VFD in the VFD stack through + * which all VFD calls pass. Note that this definition + * is situational. For example, the sec2 VFD is typically + * terminal. However, in the context of the family file + * driver, it is not -- the family file driver is the + * bottom VFD through which all VFD calls pass, and thus + * it is terminal. + * + * Similarly, on the splitter VFD, a sec2 VFD on the + * R/W channel is terminal, but a sec2 VFD on the W/O + * channel is not. + * + * + * Pass through VFD: Any VFD that relays all VFD calls + * (with the possible exception of some non-I/O related + * calls) to underlying VFD(s). + */ + +/* Unknown op codes should be ignored silently unless the + * H5FD_CTL__FAIL_IF_UNKNOWN_FLAG is set. + * + * On terminal VFDs, unknown op codes should generate an + * error unconditionally if this flag is set. + * + * On pass through VFDs, unknown op codes should be routed + * to the underlying VFD(s) as indicated by any routing + * flags. In the absence of such flags, the VFD should + * generate an error. + */ +#define H5FD_CTL__FAIL_IF_UNKNOWN_FLAG 0x0001 + +/* The H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG is used only + * by non-ternminal VFDs, and only applies to unknown + * opcodes. (known op codes should be handled as + * appropriate.) + * + * If this flag is set for an unknown op code, that + * op code should be passed to the next VFD down + * the VFD stack en-route to the terminal VFD. + * If that VFD does not support the ctl call, the + * pass through VFD should fail or succeed as directed + * by the H5FD_CTL__FAIL_IF_UNKNOWN_FLAG. + */ +#define H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG 0x0002 /*******************/ /* Public Typedefs */ /*******************/ +/* + * File driver identifiers. + * + * Values 0 through 255 are for drivers defined by the HDF5 library. + * Values 256 through 511 are available for testing new drivers. + * Subsequent values should be obtained from the HDF5 development + * team at mailto:help@hdfgroup.org. + */ +typedef int H5FD_class_value_t; + /* Types of allocation requests: see H5Fpublic.h */ typedef enum H5F_mem_t H5FD_mem_t; @@ -253,6 +379,19 @@ typedef struct { } H5FD_file_image_callbacks_t; //! +/** + * Define structure to hold "ctl memory copy" parameters + */ +//! +typedef struct H5FD_ctl_memcpy_args_t { + void * dstbuf; /**< Destination buffer */ + hsize_t dst_off; /**< Offset within destination buffer */ + const void *srcbuf; /**< Source buffer */ + hsize_t src_off; /**< Offset within source buffer */ + size_t len; /**< Length of data to copy from source buffer */ +} H5FD_ctl_memcpy_args_t; +//! + /********************/ /* Public Variables */ /********************/ diff --git a/src/H5FDros3.c b/src/H5FDros3.c index c0361f9..922dac5 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -237,6 +237,7 @@ static herr_t H5FD__ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing static herr_t H5FD__ros3_validate_config(const H5FD_ros3_fapl_t *fa); static const H5FD_class_t H5FD_ros3_g = { + H5FD_ROS3_VALUE, /* value */ "ros3", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -269,6 +270,7 @@ static const H5FD_class_t H5FD_ros3_g = { NULL, /* lock */ NULL, /* unlock */ NULL, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -276,31 +278,6 @@ static const H5FD_class_t H5FD_ros3_g = { H5FL_DEFINE_STATIC(H5FD_ros3_t); /*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initializes any interface-specific data or routines. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Jacob Smith 2017 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__init_package(void) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - if (H5FD_ros3_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize ros3 VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD__init_package() */ - -/*------------------------------------------------------------------------- * Function: H5FD_ros3_init * * Purpose: Initialize this driver by registering the driver with the @@ -327,8 +304,12 @@ H5FD_ros3_init(void) HDfprintf(stdout, "H5FD_ros3_init() called.\n"); #endif - if (H5I_VFL != H5I_get_type(H5FD_ROS3_g)) + if (H5I_VFL != H5I_get_type(H5FD_ROS3_g)) { H5FD_ROS3_g = H5FD_register(&H5FD_ros3_g, sizeof(H5FD_class_t), FALSE); + if (H5I_INVALID_HID == H5FD_ROS3_g) { + HGOTO_ERROR(H5E_ID, H5E_CANTREGISTER, H5I_INVALID_HID, "unable to register ros3"); + } + } #if ROS3_STATS /* pre-compute statsbin boundaries @@ -377,7 +358,7 @@ H5FD__ros3_term(void) * Function: H5Pset_fapl_ros3 * * Purpose: Modify the file access property list to use the H5FD_ROS3 - * driver defined in this source file. All driver specfic + * driver defined in this source file. All driver specific * properties are passed in as a pointer to a suitably * initialized instance of H5FD_ros3_fapl_t * @@ -410,7 +391,7 @@ H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa) if (FAIL == H5FD__ros3_validate_config(fa)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config") - ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa); + ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa, NULL); done: FUNC_LEAVE_API(ret_value) @@ -420,16 +401,16 @@ done: * Function: H5FD__ros3_validate_config() * * Purpose: Test to see if the supplied instance of H5FD_ros3_fapl_t - * contains internally consistant data. Return SUCCEED if so, + * contains internally consistent data. Return SUCCEED if so, * and FAIL otherwise. * - * Note the difference between internally consistant and + * Note the difference between internally consistent and * correct. As we will have to try to access the target * object to determine whether the supplied data is correct, - * we will settle for internal consistancy at this point + * we will settle for internal consistency at this point * * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally - * consistant data, FAIL otherwise. + * consistent data, FAIL otherwise. * * Programmer: Jacob Smith * 9/10/17 diff --git a/src/H5FDros3.h b/src/H5FDros3.h index 8e42ca2..4a623c4 100644 --- a/src/H5FDros3.h +++ b/src/H5FDros3.h @@ -22,9 +22,11 @@ #define H5FDros3_H #ifdef H5_HAVE_ROS3_VFD -#define H5FD_ROS3 (H5FD_ros3_init()) +#define H5FD_ROS3 (H5FDperform_init(H5FD_ros3_init)) +#define H5FD_ROS3_VALUE H5_VFD_ROS3 #else -#define H5FD_ROS3 (H5I_INVALID_HID) +#define H5FD_ROS3 (H5I_INVALID_HID) +#define H5FD_ROS3_VALUE H5_VFD_INVALID #endif /* H5_HAVE_ROS3_VFD */ #ifdef H5_HAVE_ROS3_VFD diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h index da6a62d..b81bfae 100644 --- a/src/H5FDs3comms.h +++ b/src/H5FDs3comms.h @@ -179,7 +179,7 @@ * HTTP header fields, of particular use when composing an * "S3 Canonical Request" for authentication. * - * - The creation of a Canoncial Request involves: + * - The creation of a Canonical Request involves: * - convert field names to lower case * - sort by this lower-case name * - convert ": " name-value separator in HTTP string to ":" @@ -459,7 +459,7 @@ typedef struct { * * Pointer to NULL-terminated string for "secret" access id to S3 resource. * - * Requred to authenticate. + * Required to authenticate. * * `signing_key` (unsigned char *) * @@ -470,7 +470,7 @@ typedef struct { * which may be re-used for several (up to seven (7)) days from creation? * Computed once upon file open. * - * Requred to authenticate. + * Required to authenticate. * *---------------------------------------------------------------------------- */ diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index d823e3c..46f5fd4 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -139,8 +139,11 @@ static herr_t H5FD__sec2_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing static herr_t H5FD__sec2_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD__sec2_unlock(H5FD_t *_file); static herr_t H5FD__sec2_delete(const char *filename, hid_t fapl_id); +static herr_t H5FD__sec2_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, + void **output); static const H5FD_class_t H5FD_sec2_g = { + H5FD_SEC2_VALUE, /* value */ "sec2", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -173,6 +176,7 @@ static const H5FD_class_t H5FD_sec2_g = { H5FD__sec2_lock, /* lock */ H5FD__sec2_unlock, /* unlock */ H5FD__sec2_delete, /* del */ + H5FD__sec2_ctl, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -180,39 +184,6 @@ static const H5FD_class_t H5FD_sec2_g = { H5FL_DEFINE_STATIC(H5FD_sec2_t); /*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initializes any interface-specific data or routines. - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__init_package(void) -{ - char * lock_env_var = NULL; /* Environment variable pointer */ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - /* Check the use disabled file locks environment variable */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) - ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ - else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) - ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ - else - ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ - - if (H5FD_sec2_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize sec2 VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - -/*------------------------------------------------------------------------- * Function: H5FD_sec2_init * * Purpose: Initialize this driver by registering the driver with the @@ -229,9 +200,19 @@ done: hid_t H5FD_sec2_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + char *lock_env_var = NULL; /* Environment variable pointer */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ - FUNC_ENTER_NOAPI(H5I_INVALID_HID) + FUNC_ENTER_NOAPI_NOERR + + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv(HDF5_USE_FILE_LOCKING); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ if (H5I_VFL != H5I_get_type(H5FD_SEC2_g)) H5FD_SEC2_g = H5FD_register(&H5FD_sec2_g, sizeof(H5FD_class_t), FALSE); @@ -239,7 +220,6 @@ H5FD_sec2_init(void) /* Set return value */ ret_value = H5FD_SEC2_g; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_sec2_init() */ @@ -292,7 +272,7 @@ H5Pset_fapl_sec2(hid_t fapl_id) if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - ret_value = H5P_set_driver(plist, H5FD_SEC2, NULL); + ret_value = H5P_set_driver(plist, H5FD_SEC2, NULL, NULL); done: FUNC_LEAVE_API(ret_value) @@ -1068,3 +1048,47 @@ H5FD__sec2_delete(const char *filename, hid_t H5_ATTR_UNUSED fapl_id) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__sec2_delete() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__sec2_ctl + * + * Purpose: Sec2 VFD version of the ctl callback. + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * At present, no op codes are supported by this VFD. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__sec2_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input, + void H5_ATTR_UNUSED **output) +{ + H5FD_sec2_t *file = (H5FD_sec2_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(file); + + switch (op_code) { + /* Unknown op code */ + default: + if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set") + break; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__sec2_ctl() */ diff --git a/src/H5FDsec2.h b/src/H5FDsec2.h index 541ac71..df4d4d4 100644 --- a/src/H5FDsec2.h +++ b/src/H5FDsec2.h @@ -20,7 +20,8 @@ #ifndef H5FDsec2_H #define H5FDsec2_H -#define H5FD_SEC2 (H5FD_sec2_init()) +#define H5FD_SEC2 (H5FDperform_init(H5FD_sec2_init)) +#define H5FD_SEC2_VALUE H5_VFD_SEC2 #ifdef __cplusplus extern "C" { diff --git a/src/H5FDspace.c b/src/H5FDspace.c index de52dc3..48b06ba 100644 --- a/src/H5FDspace.c +++ b/src/H5FDspace.c @@ -148,7 +148,7 @@ H5FD__alloc_real(H5FD_t *file, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr FUNC_ENTER_PACKAGE #ifdef H5FD_ALLOC_DEBUG - HDfprintf(stderr, "%s: type = %u, size = %Hu\n", __func__, (unsigned)type, size); + HDfprintf(stderr, "%s: type = %u, size = %" PRIuHSIZE "\n", __func__, (unsigned)type, size); #endif /* H5FD_ALLOC_DEBUG */ /* check args */ @@ -211,7 +211,7 @@ H5FD__alloc_real(H5FD_t *file, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr done: #ifdef H5FD_ALLOC_DEBUG - HDfprintf(stderr, "%s: ret_value = %a\n", __func__, ret_value); + HDfprintf(stderr, "%s: ret_value = %" PRIuHADDR "\n", __func__, ret_value); #endif /* H5FD_ALLOC_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__alloc_real() */ @@ -287,7 +287,8 @@ H5FD__free_real(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size) HDassert(size > 0); #ifdef H5FD_ALLOC_DEBUG - HDfprintf(stderr, "%s: type = %u, addr = %a, size = %Hu\n", __func__, (unsigned)type, addr, size); + HDfprintf(stderr, "%s: type = %u, addr = %" PRIuHADDR ", size = %" PRIuHSIZE "\n", __func__, + (unsigned)type, addr, size); #endif /* H5FD_ALLOC_DEBUG */ /* Sanity checking */ @@ -317,11 +318,11 @@ H5FD__free_real(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size) eoa = file->cls->get_eoa(file, type); #ifdef H5FD_ALLOC_DEBUG - HDfprintf(stderr, "%s: eoa = %a\n", __func__, eoa); + HDfprintf(stderr, "%s: eoa = %" PRIuHADDR "\n", __func__, eoa); #endif /* H5FD_ALLOC_DEBUG */ if (eoa == (addr + size)) { #ifdef H5FD_ALLOC_DEBUG - HDfprintf(stderr, "%s: Reducing file size to = %a\n", __func__, addr); + HDfprintf(stderr, "%s: Reducing file size to = %" PRIuHADDR "\n", __func__, addr); #endif /* H5FD_ALLOC_DEBUG */ if (file->cls->set_eoa(file, type, addr) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "set end of space allocation request failed") @@ -330,8 +331,8 @@ H5FD__free_real(H5FD_t *file, H5FD_mem_t type, haddr_t addr, hsize_t size) else { /* leak memory */ #ifdef H5FD_ALLOC_DEBUG - HDfprintf(stderr, "%s: LEAKED MEMORY!!! type = %u, addr = %a, size = %Hu\n", __func__, (unsigned)type, - addr, size); + HDfprintf(stderr, "%s: LEAKED MEMORY!!! type = %u, addr = %" PRIuHADDR ", size = %" PRIuHSIZE "\n", + __func__, (unsigned)type, addr, size); #endif /* H5FD_ALLOC_DEBUG */ } /* end else */ diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c index 73c898a..31438cd 100644 --- a/src/H5FDsplitter.c +++ b/src/H5FDsplitter.c @@ -104,6 +104,10 @@ static int H5FD__copy_plist(hid_t fapl_id, hid_t *id_out_ptr); /* Prototypes */ static herr_t H5FD__splitter_term(void); +static herr_t H5FD__splitter_populate_config(H5FD_splitter_vfd_config_t *vfd_config, + H5FD_splitter_fapl_t * fapl_out); +static herr_t H5FD__splitter_get_default_wo_path(char *new_path, size_t new_path_len, + const char *base_filename); static hsize_t H5FD__splitter_sb_size(H5FD_t *_file); static herr_t H5FD__splitter_sb_encode(H5FD_t *_file, char *name /*out*/, unsigned char *buf /*out*/); static herr_t H5FD__splitter_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf); @@ -129,8 +133,12 @@ static herr_t H5FD__splitter_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closin static herr_t H5FD__splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD__splitter_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD__splitter_unlock(H5FD_t *_file); +static herr_t H5FD__splitter_delete(const char *filename, hid_t fapl_id); +static herr_t H5FD__splitter_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, + void **output); static const H5FD_class_t H5FD_splitter_g = { + H5FD_SPLITTER_VALUE, /* value */ "splitter", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -138,7 +146,7 @@ static const H5FD_class_t H5FD_splitter_g = { H5FD__splitter_sb_size, /* sb_size */ H5FD__splitter_sb_encode, /* sb_encode */ H5FD__splitter_sb_decode, /* sb_decode */ - sizeof(H5FD_splitter_fapl_t), /* fapl_size */ + sizeof(H5FD_splitter_fapl_t), /* fapl_size */ H5FD__splitter_fapl_get, /* fapl_get */ H5FD__splitter_fapl_copy, /* fapl_copy */ H5FD__splitter_fapl_free, /* fapl_free */ @@ -149,7 +157,7 @@ static const H5FD_class_t H5FD_splitter_g = { H5FD__splitter_close, /* close */ H5FD__splitter_cmp, /* cmp */ H5FD__splitter_query, /* query */ - H5FD__splitter_get_type_map, /* get_type_map */ + H5FD__splitter_get_type_map, /* get_type_map */ H5FD__splitter_alloc, /* alloc */ H5FD__splitter_free, /* free */ H5FD__splitter_get_eoa, /* get_eoa */ @@ -162,7 +170,8 @@ static const H5FD_class_t H5FD_splitter_g = { H5FD__splitter_truncate, /* truncate */ H5FD__splitter_lock, /* lock */ H5FD__splitter_unlock, /* unlock */ - NULL, /* del */ + H5FD__splitter_delete, /* del */ + H5FD__splitter_ctl, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -173,30 +182,6 @@ H5FL_DEFINE_STATIC(H5FD_splitter_t); H5FL_DEFINE_STATIC(H5FD_splitter_fapl_t); /*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initializes any interface-specific data or routines. - * - * Return: SUCCEED/FAIL - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__init_package(void) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - H5FD_SPLITTER_LOG_CALL(__func__); - - if (H5FD_splitter_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize splitter VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - -/*------------------------------------------------------------------------- * Function: H5FD_splitter_init * * Purpose: Initialize the splitter driver by registering it with the @@ -211,7 +196,7 @@ H5FD_splitter_init(void) { hid_t ret_value = H5I_INVALID_HID; - FUNC_ENTER_NOAPI(H5I_INVALID_HID) + FUNC_ENTER_NOAPI_NOERR H5FD_SPLITTER_LOG_CALL(__func__); @@ -220,7 +205,6 @@ H5FD_splitter_init(void) ret_value = H5FD_SPLITTER_g; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_splitter_init() */ @@ -309,57 +293,14 @@ H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *vfd_config) if (NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid property list") - /* Make sure that the W/O channel supports write-only capability. - * Some drivers (e.g. family or multi) do revision of the superblock - * in-memory, causing problems in that channel. - * Uses the feature flag H5FD_FEAT_DEFAULT_VFD_COMPATIBLE as the - * determining attribute. - */ - if (H5P_DEFAULT != vfd_config->wo_fapl_id) { - H5FD_class_t * wo_driver = NULL; - H5FD_driver_prop_t wo_driver_prop; - H5P_genplist_t * wo_plist_ptr = NULL; - unsigned long wo_driver_flags = 0; - - wo_plist_ptr = (H5P_genplist_t *)H5I_object(vfd_config->wo_fapl_id); - if (NULL == wo_plist_ptr) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - if (H5P_peek(wo_plist_ptr, H5F_ACS_FILE_DRV_NAME, &wo_driver_prop) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get driver ID & info") - wo_driver = (H5FD_class_t *)H5I_object(wo_driver_prop.driver_id); - if (NULL == wo_driver) - HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid driver ID in file access property list") - if (H5FD_driver_query(wo_driver, &wo_driver_flags) < 0) - HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't query VFD flags") - if (0 == (H5FD_FEAT_DEFAULT_VFD_COMPATIBLE & wo_driver_flags)) - HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unsuitable W/O driver") - } /* end if W/O VFD is non-default */ - info = H5FL_CALLOC(H5FD_splitter_fapl_t); if (NULL == info) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate file access property list struct") - info->ignore_wo_errs = vfd_config->ignore_wo_errs; - HDstrncpy(info->wo_path, vfd_config->wo_path, H5FD_SPLITTER_PATH_MAX + 1); - info->wo_path[H5FD_SPLITTER_PATH_MAX] = '\0'; - HDstrncpy(info->log_file_path, vfd_config->log_file_path, H5FD_SPLITTER_PATH_MAX + 1); - info->log_file_path[H5FD_SPLITTER_PATH_MAX] = '\0'; - info->rw_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ - info->wo_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ - - /* Set non-default channel FAPL IDs in splitter configuration info */ - if (H5P_DEFAULT != vfd_config->rw_fapl_id) { - if (FALSE == H5P_isa_class(vfd_config->rw_fapl_id, H5P_FILE_ACCESS)) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") - info->rw_fapl_id = vfd_config->rw_fapl_id; - } - if (H5P_DEFAULT != vfd_config->wo_fapl_id) { - if (FALSE == H5P_isa_class(vfd_config->wo_fapl_id, H5P_FILE_ACCESS)) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") - info->wo_fapl_id = vfd_config->wo_fapl_id; - } + if (H5FD__splitter_populate_config(vfd_config, info) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't setup driver configuration") - ret_value = H5P_set_driver(plist_ptr, H5FD_SPLITTER, info); + ret_value = H5P_set_driver(plist_ptr, H5FD_SPLITTER, info, NULL); done: if (info) @@ -383,9 +324,10 @@ done: herr_t H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config /*out*/) { - const H5FD_splitter_fapl_t *fapl_ptr = NULL; - H5P_genplist_t * plist_ptr = NULL; - herr_t ret_value = SUCCEED; + const H5FD_splitter_fapl_t *fapl_ptr = NULL; + H5FD_splitter_fapl_t * default_fapl = NULL; + H5P_genplist_t * plist_ptr = NULL; + herr_t ret_value = SUCCEED; FUNC_ENTER_API(FAIL) H5TRACE2("e", "ix", fapl_id, config); @@ -411,8 +353,14 @@ H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config /*out*/) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") if (H5FD_SPLITTER != H5P_peek_driver(plist_ptr)) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") - if (NULL == (fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr))) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unable to get specific-driver info") + fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr); + if (NULL == fapl_ptr) { + if (NULL == (default_fapl = H5FL_CALLOC(H5FD_splitter_fapl_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate file access property list struct") + if (H5FD__splitter_populate_config(NULL, default_fapl) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't initialize driver configuration info") + fapl_ptr = default_fapl; + } HDstrncpy(config->wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX + 1); HDstrncpy(config->log_file_path, fapl_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX + 1); @@ -425,10 +373,188 @@ H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config /*out*/) HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy W/O FAPL"); done: + if (default_fapl) + H5FL_FREE(H5FD_splitter_fapl_t, default_fapl); + FUNC_LEAVE_API(ret_value) } /* end H5Pget_fapl_splitter() */ /*------------------------------------------------------------------------- + * Function: H5FD__splitter_populate_config + * + * Purpose: Populates a H5FD_splitter_fapl_t structure with the provided + * values, supplying defaults where values are not provided. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_populate_config(H5FD_splitter_vfd_config_t *vfd_config, H5FD_splitter_fapl_t *fapl_out) +{ + H5P_genplist_t *def_plist; + H5P_genplist_t *plist; + hbool_t free_config = FALSE; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(fapl_out); + + HDmemset(fapl_out, 0, sizeof(H5FD_splitter_fapl_t)); + + if (!vfd_config) { + vfd_config = H5MM_calloc(sizeof(H5FD_splitter_vfd_config_t)); + if (NULL == vfd_config) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate file access property list struct") + + vfd_config->magic = H5FD_SPLITTER_MAGIC; + vfd_config->version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + vfd_config->rw_fapl_id = H5P_DEFAULT; + vfd_config->wo_fapl_id = H5P_DEFAULT; + + free_config = TRUE; + } + + /* Make sure that the W/O channel supports write-only capability. + * Some drivers (e.g. family or multi) do revision of the superblock + * in-memory, causing problems in that channel. + * Uses the feature flag H5FD_FEAT_DEFAULT_VFD_COMPATIBLE as the + * determining attribute. + */ + if (H5P_DEFAULT != vfd_config->wo_fapl_id) { + H5FD_class_t * wo_driver = NULL; + H5FD_driver_prop_t wo_driver_prop; + H5P_genplist_t * wo_plist_ptr = NULL; + unsigned long wo_driver_flags = 0; + + wo_plist_ptr = (H5P_genplist_t *)H5I_object(vfd_config->wo_fapl_id); + if (NULL == wo_plist_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5P_peek(wo_plist_ptr, H5F_ACS_FILE_DRV_NAME, &wo_driver_prop) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get driver ID & info") + wo_driver = (H5FD_class_t *)H5I_object(wo_driver_prop.driver_id); + if (NULL == wo_driver) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid driver ID in file access property list") + if (H5FD_driver_query(wo_driver, &wo_driver_flags) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't query VFD flags") + if (0 == (H5FD_FEAT_DEFAULT_VFD_COMPATIBLE & wo_driver_flags)) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unsuitable W/O driver") + } /* end if W/O VFD is non-default */ + + fapl_out->ignore_wo_errs = vfd_config->ignore_wo_errs; + HDstrncpy(fapl_out->wo_path, vfd_config->wo_path, H5FD_SPLITTER_PATH_MAX + 1); + fapl_out->wo_path[H5FD_SPLITTER_PATH_MAX] = '\0'; + HDstrncpy(fapl_out->log_file_path, vfd_config->log_file_path, H5FD_SPLITTER_PATH_MAX + 1); + fapl_out->log_file_path[H5FD_SPLITTER_PATH_MAX] = '\0'; + fapl_out->rw_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + fapl_out->wo_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + + if (NULL == (def_plist = (H5P_genplist_t *)H5I_object(H5P_FILE_ACCESS_DEFAULT))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + + /* Set non-default channel FAPL IDs in splitter configuration info */ + if (H5P_DEFAULT != vfd_config->rw_fapl_id) { + if (FALSE == H5P_isa_class(vfd_config->rw_fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + fapl_out->rw_fapl_id = vfd_config->rw_fapl_id; + } + else { + /* Use copy of default file access property list for R/W channel FAPL ID. + * The Sec2 driver is explicitly set on the FAPL ID, as the default + * driver might have been replaced with the Splitter VFD, which + * would cause recursion badness. + */ + if ((fapl_out->rw_fapl_id = H5P_copy_plist(def_plist, FALSE)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "can't copy property list") + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_out->rw_fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5P_set_driver_by_value(plist, H5_VFD_SEC2, NULL, TRUE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't set default driver on R/W channel FAPL") + } + if (H5P_DEFAULT != vfd_config->wo_fapl_id) { + if (FALSE == H5P_isa_class(vfd_config->wo_fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + fapl_out->wo_fapl_id = vfd_config->wo_fapl_id; + } + else { + /* Use copy of default file access property list for W/O channel FAPL ID. + * The Sec2 driver is explicitly set on the FAPL ID, as the default + * driver might have been replaced with the Splitter VFD, which + * would cause recursion badness. + */ + if ((fapl_out->wo_fapl_id = H5P_copy_plist(def_plist, FALSE)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "can't copy property list") + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_out->wo_fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if (H5P_set_driver_by_value(plist, H5_VFD_SEC2, NULL, TRUE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't set default driver on R/W channel FAPL") + } + +done: + if (free_config && vfd_config) + H5MM_free(vfd_config); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_populate_config() */ + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_get_default_wo_path + * + * Purpose: Given a base filename, returns a default filename for the + * W/O channel file by appending '_wo' to the base file name. + * + * Return: Non-negative on Success/Negative on Failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_get_default_wo_path(char *new_path, size_t new_path_len, const char *base_filename) +{ + const char *suffix = "_wo"; + size_t old_filename_len = 0; + char * file_extension = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(new_path); + HDassert(base_filename); + + /* Check that output buffer can hold base filename + `_wo` suffix */ + old_filename_len = HDstrlen(base_filename); + if (old_filename_len > H5FD_SPLITTER_PATH_MAX - HDstrlen(suffix) - 1) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "filename exceeds max length") + + /* Determine if filename contains a ".h5" extension. */ + if ((file_extension = strstr(base_filename, ".h5"))) { + /* Insert the suffix between the filename and ".h5" extension. */ + HDstrcpy(new_path, base_filename); + file_extension = strstr(new_path, ".h5"); + HDsprintf(file_extension, "%s%s", suffix, ".h5"); + } + else if ((file_extension = strrchr(base_filename, '.'))) { + char *new_extension_loc = NULL; + + /* If the filename doesn't contain a ".h5" extension, but contains + * AN extension, just insert the suffix before that extension. + */ + HDstrcpy(new_path, base_filename); + new_extension_loc = strrchr(new_path, '.'); + HDsprintf(new_extension_loc, "%s%s", suffix, file_extension); + } + else { + /* If the filename doesn't contain an extension at all, just insert + * the suffix at the end of the filename. + */ + HDsnprintf(new_path, new_path_len, "%s%s", base_filename, suffix); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_default_wo_path() */ + +/*------------------------------------------------------------------------- * Function: H5FD__splitter_flush * * Purpose: Flushes all data to disk for both channels. @@ -655,10 +781,11 @@ done: static H5FD_t * H5FD__splitter_open(const char *name, unsigned flags, hid_t splitter_fapl_id, haddr_t maxaddr) { - H5FD_splitter_t * file_ptr = NULL; /* Splitter VFD info */ - const H5FD_splitter_fapl_t *fapl_ptr = NULL; /* Driver-specific property list */ - H5P_genplist_t * plist_ptr = NULL; - H5FD_t * ret_value = NULL; + H5FD_splitter_t * file_ptr = NULL; /* Splitter VFD info */ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; /* Driver-specific property list */ + H5FD_splitter_fapl_t * default_fapl = NULL; + H5P_genplist_t * plist_ptr = NULL; + H5FD_t * ret_value = NULL; FUNC_ENTER_STATIC @@ -671,8 +798,7 @@ H5FD__splitter_open(const char *name, unsigned flags, hid_t splitter_fapl_id, ha HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") if (ADDR_OVERFLOW(maxaddr)) HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") - if ((H5P_FILE_ACCESS_DEFAULT == splitter_fapl_id) || (H5FD_SPLITTER != H5Pget_driver(splitter_fapl_id))) - /* presupposes that H5P_FILE_ACCESS_DEFAULT is not a splitter */ + if (H5FD_SPLITTER != H5Pget_driver(splitter_fapl_id)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "driver is not splitter") file_ptr = (H5FD_splitter_t *)H5FL_CALLOC(H5FD_splitter_t); @@ -686,8 +812,20 @@ H5FD__splitter_open(const char *name, unsigned flags, hid_t splitter_fapl_id, ha if (NULL == plist_ptr) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr); - if (NULL == fapl_ptr) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get VFL driver info") + if (NULL == fapl_ptr) { + if (NULL == (default_fapl = H5FL_CALLOC(H5FD_splitter_fapl_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file access property list struct") + if (H5FD__splitter_populate_config(NULL, default_fapl) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, NULL, "can't initialize driver configuration info") + + /* If W/O path is not set, use base filename with '_wo' suffix */ + if (*default_fapl->wo_path == '\0') + if (H5FD__splitter_get_default_wo_path(default_fapl->wo_path, H5FD_SPLITTER_PATH_MAX + 1, name) < + 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, NULL, "can't generate default filename for W/O channel") + + fapl_ptr = default_fapl; + } /* Copy simpler info */ HDstrncpy(file_ptr->fa.wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX + 1); @@ -723,6 +861,9 @@ H5FD__splitter_open(const char *name, unsigned flags, hid_t splitter_fapl_id, ha ret_value = (H5FD_t *)file_ptr; done: + if (default_fapl) + H5FL_FREE(H5FD_splitter_fapl_t, default_fapl); + if (NULL == ret_value) { if (file_ptr) { if (H5I_INVALID_HID != file_ptr->fa.rw_fapl_id) @@ -1145,6 +1286,62 @@ done: } /* end H5FD__splitter_unlock */ /*------------------------------------------------------------------------- + * Function: H5FD__splitter_ctl + * + * Purpose: Splitter VFD version of the ctl callback. + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * At present, this VFD supports no op codes of its own and + * simply passes ctl calls on to the R/W channel VFD. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, void **output) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(file); + + switch (op_code) { + /* Unknown op code */ + default: + if (flags & H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG) { + /* Pass ctl call down to R/W channel VFD */ + if (H5FDctl(file->rw_file, op_code, flags, input, output) < 0) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed") + } + else { + /* If no valid VFD routing flag is specified, fail for unknown op code + * if H5FD_CTL__FAIL_IF_UNKNOWN_FLAG flag is set. + */ + if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, + "VFD ctl request failed (unknown op code and fail if unknown flag is set)") + } + + break; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_ctl() */ + +/*------------------------------------------------------------------------- * Function: H5FD__splitter_query * * Purpose: Set the flags that this VFL driver is capable of supporting. @@ -1279,6 +1476,74 @@ done: } /* end H5FD__splitter_free() */ /*------------------------------------------------------------------------- + * Function: H5FD__splitter_delete + * + * Purpose: Delete a file + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_delete(const char *filename, hid_t fapl_id) +{ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; + H5FD_splitter_fapl_t * default_fapl = NULL; + H5P_genplist_t * plist; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(filename); + + /* Get the driver info */ + if (H5P_FILE_ACCESS_DEFAULT == fapl_id) { + if (NULL == (default_fapl = H5FL_CALLOC(H5FD_splitter_fapl_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate file access property list struct") + if (H5FD__splitter_populate_config(NULL, default_fapl) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't initialize driver configuration info") + + /* If W/O path is not set, use base filename with '_wo' suffix */ + if (*default_fapl->wo_path == '\0') + if (H5FD__splitter_get_default_wo_path(default_fapl->wo_path, H5FD_SPLITTER_PATH_MAX + 1, + filename) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't generate default filename for W/O channel") + + fapl_ptr = default_fapl; + } + else { + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if (NULL == (fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist))) { + if (NULL == (default_fapl = H5FL_CALLOC(H5FD_splitter_fapl_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, + "unable to allocate file access property list struct") + if (H5FD__splitter_populate_config(NULL, default_fapl) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't initialize driver configuration info") + + /* If W/O path is not set, use base filename with '_wo' suffix */ + if (*default_fapl->wo_path == '\0') + if (H5FD__splitter_get_default_wo_path(default_fapl->wo_path, H5FD_SPLITTER_PATH_MAX + 1, + filename) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't generate default filename for W/O channel") + + fapl_ptr = default_fapl; + } + } + + if (H5FDdelete(filename, fapl_ptr->rw_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDELETEFILE, FAIL, "unable to delete file") + if (H5FDdelete(fapl_ptr->wo_path, fapl_ptr->wo_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDELETEFILE, FAIL, "unable to delete W/O channel file") + +done: + if (default_fapl) + H5FL_FREE(H5FD_splitter_fapl_t, default_fapl); + + FUNC_LEAVE_NOAPI(ret_value) +} + +/*------------------------------------------------------------------------- * Function: H5FD__splitter_log_error * * Purpose: Log an error from the W/O channel appropriately. diff --git a/src/H5FDsplitter.h b/src/H5FDsplitter.h index ee6e7c5..c8751c8 100644 --- a/src/H5FDsplitter.h +++ b/src/H5FDsplitter.h @@ -17,7 +17,8 @@ #ifndef H5FDsplitter_H #define H5FDsplitter_H -#define H5FD_SPLITTER (H5FD_splitter_init()) +#define H5FD_SPLITTER (H5FDperform_init(H5FD_splitter_init)) +#define H5FD_SPLITTER_VALUE H5_VFD_SPLITTER /* The version of the H5FD_splitter_vfd_config_t structure used */ #define H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION 1 @@ -34,7 +35,7 @@ * Structure: H5FD_spliiter_vfd_config_t * * One-stop shopping for configuring a Splitter VFD (rather than many - * paramaters passed into H5Pset/get functions). + * parameters passed into H5Pset/get functions). * * magic (int32_t) * Semi-unique number, used to sanity-check that a given pointer is diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 6631325..122379a 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -183,6 +183,7 @@ static herr_t H5FD_stdio_unlock(H5FD_t *_file); static herr_t H5FD_stdio_delete(const char *filename, hid_t fapl_id); static const H5FD_class_t H5FD_stdio_g = { + H5_VFD_STDIO, /* value */ "stdio", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -215,6 +216,7 @@ static const H5FD_class_t H5FD_stdio_g = { H5FD_stdio_lock, /* lock */ H5FD_stdio_unlock, /* unlock */ H5FD_stdio_delete, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -242,7 +244,7 @@ H5FD_stdio_init(void) H5Eclear2(H5E_DEFAULT); /* Check the use disabled file locks environment variable */ - lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); + lock_env_var = getenv(HDF5_USE_FILE_LOCKING); if (lock_env_var && !strcmp(lock_env_var, "BEST_EFFORT")) ignore_disabled_file_locks_s = 1; /* Override: Ignore disabled locks */ else if (lock_env_var && (!strcmp(lock_env_var, "TRUE") || !strcmp(lock_env_var, "1"))) diff --git a/src/H5FDstdio.h b/src/H5FDstdio.h index 9db92ed..484c6ea 100644 --- a/src/H5FDstdio.h +++ b/src/H5FDstdio.h @@ -22,7 +22,7 @@ #include "H5Ipublic.h" -#define H5FD_STDIO (H5FD_stdio_init()) +#define H5FD_STDIO (H5FDperform_init(H5FD_stdio_init)) #ifdef __cplusplus extern "C" { diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c index 60812bd..11c177b 100644 --- a/src/H5FDvfd_swmr.c +++ b/src/H5FDvfd_swmr.c @@ -90,6 +90,7 @@ static htri_t H5FD__vfd_swmr_index_deserialize(const H5FD_vfd_swmr_t *file, H5FD static herr_t H5FD__vfd_swmr_load_hdr_and_idx(H5FD_vfd_swmr_t *, hbool_t); static const H5FD_class_t H5FD_vfd_swmr_g = { + H5FD_VFD_SWMR_VALUE, /* value */ "vfd_swmr", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -122,6 +123,7 @@ static const H5FD_class_t H5FD_vfd_swmr_g = { H5FD__vfd_swmr_lock, /* lock */ H5FD__vfd_swmr_unlock, /* unlock */ NULL, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -132,30 +134,6 @@ H5FL_DEFINE_STATIC(H5FD_vfd_swmr_t); H5FL_SEQ_DEFINE(H5FD_vfd_swmr_idx_entry_t); /*------------------------------------------------------------------------- - * Function: H5FD__init_package - * - * Purpose: Initializes any interface-specific data or routines. - * -b - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__init_package(void) -{ - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - if (H5FD_vfd_swmr_init() < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize swmr VFD") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__init_package() */ - -/*------------------------------------------------------------------------- * Function: H5FD_vfd_swmr_init * * Purpose: Initialize this driver by registering the driver with the @@ -174,7 +152,7 @@ H5FD_vfd_swmr_init(void) { hid_t ret_value = H5I_INVALID_HID; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR if (H5I_VFL != H5I_get_type(H5FD_VFD_SWMR_g)) H5FD_VFD_SWMR_g = H5FD_register(&H5FD_vfd_swmr_g, sizeof(H5FD_class_t), FALSE); @@ -182,7 +160,6 @@ H5FD_vfd_swmr_init(void) /* Set return value */ ret_value = H5FD_VFD_SWMR_g; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_vfd_swmr_init() */ @@ -231,7 +208,7 @@ H5Pset_fapl_vfd_swmr(hid_t fapl_id) if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - ret_value = H5P_set_driver(plist, H5FD_VFD_SWMR, NULL); + ret_value = H5P_set_driver(plist, H5FD_VFD_SWMR, NULL, NULL); done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5FDvfd_swmr.h b/src/H5FDvfd_swmr.h index 20a6b96..dfd8d19 100644 --- a/src/H5FDvfd_swmr.h +++ b/src/H5FDvfd_swmr.h @@ -16,7 +16,8 @@ #ifndef H5FDvfd_swmr_H #define H5FDvfd_swmr_H -#define H5FD_VFD_SWMR (H5FD_vfd_swmr_init()) +#define H5FD_VFD_SWMR (H5FDperform_init(H5FD_vfd_swmr_init)) +#define H5FD_VFD_SWMR_VALUE H5_VFD_SWMR #ifdef __cplusplus extern "C" { diff --git a/src/H5FDwindows.c b/src/H5FDwindows.c index 6ce33a7..fa3c05f 100644 --- a/src/H5FDwindows.c +++ b/src/H5FDwindows.c @@ -56,7 +56,7 @@ H5Pset_fapl_windows(hid_t fapl_id) if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") - ret_value = H5P_set_driver(plist, H5FD_WINDOWS, NULL); + ret_value = H5P_set_driver(plist, H5FD_WINDOWS, NULL, NULL); done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5FL.c b/src/H5FL.c index cb58868..a897a24 100644 --- a/src/H5FL.c +++ b/src/H5FL.c @@ -114,9 +114,6 @@ struct H5FL_fac_node_t { struct H5FL_fac_node_t *next; /* Pointer to next block in free list */ }; -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* The head of the list of factory things to garbage collect */ static H5FL_fac_gc_list_t H5FL_fac_gc_head = {0, NULL}; @@ -184,20 +181,15 @@ H5FL_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { /* Garbage collect any nodes on the free lists */ - (void)H5FL_garbage_coll(); - - /* Shut down the various kinds of free lists */ - n += H5FL__reg_term(); - n += H5FL__fac_term_all(); - n += H5FL__arr_term(); - n += H5FL__blk_term(); + (void) + H5FL_garbage_coll(); - /* Mark interface closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Shut down the various kinds of free lists */ + n += H5FL__reg_term(); + n += H5FL__fac_term_all(); + n += H5FL__arr_term(); + n += H5FL__blk_term(); #ifdef H5FL_TRACK /* If we haven't freed all the allocated memory, dump out the list now */ @@ -1009,27 +1001,32 @@ H5FL_blk_free(H5FL_blk_head_t *head, void *block) #ifdef H5FL_TRACK { - H5FL_track_t *trk = block = ((unsigned char *)block) - sizeof(H5FL_track_t); + unsigned char *block_ptr = ((unsigned char *)block) - sizeof(H5FL_track_t); + H5FL_track_t trk; + + HDmemcpy(&trk, block_ptr, sizeof(H5FL_track_t)); /* Free tracking information about the allocation location */ - H5CS_close_stack(trk->stack); + H5CS_close_stack(trk.stack); /* The 'func' & 'file' strings are statically allocated (by the compiler) * and are not allocated, so there's no need to free them. */ - trk->file = NULL; - trk->func = NULL; + trk.file = NULL; + trk.func = NULL; /* Remove from "outstanding allocations" list */ - if (trk == H5FL_out_head_g) { + if ((void *)block_ptr == (void *)H5FL_out_head_g) { H5FL_out_head_g = H5FL_out_head_g->next; if (H5FL_out_head_g) H5FL_out_head_g->prev = NULL; } /* end if */ else { - trk->prev->next = trk->next; - if (trk->next) - trk->next->prev = trk->prev; + trk.prev->next = trk.next; + if (trk.next) + trk.next->prev = trk.prev; } /* end else */ + + HDmemcpy(block_ptr, &trk, sizeof(H5FL_track_t)); } #endif /* H5FL_TRACK */ @@ -1128,25 +1125,30 @@ H5FL_blk_realloc(H5FL_blk_head_t *head, void *block, size_t new_size H5FL_TRACK_ else { #ifdef H5FL_TRACK { - H5FL_track_t *trk = (H5FL_track_t *)(((unsigned char *)block) - sizeof(H5FL_track_t)); + unsigned char *block_ptr = ((unsigned char *)block) - sizeof(H5FL_track_t); + H5FL_track_t trk; + + HDmemcpy(&trk, block_ptr, sizeof(H5FL_track_t)); /* Release previous tracking information */ - H5CS_close_stack(trk->stack); + H5CS_close_stack(trk.stack); /* The 'func' & 'file' strings are statically allocated (by the compiler) * and are not allocated, so there's no need to free them. */ - trk->file = NULL; - trk->func = NULL; + trk.file = NULL; + trk.func = NULL; /* Store new tracking information */ - trk->stack = H5CS_copy_stack(); - HDassert(trk->stack); + trk.stack = H5CS_copy_stack(); + HDassert(trk.stack); /* The 'call_func' & 'call_file' strings are statically allocated (by the compiler) * there's no need to duplicate them. */ - trk->file = call_file; - trk->func = call_func; - trk->line = call_line; + trk.file = call_file; + trk.func = call_func; + trk.line = call_line; + + HDmemcpy(block_ptr, &trk, sizeof(H5FL_track_t)); } #endif /* H5FL_TRACK */ ret_value = block; @@ -1434,10 +1436,42 @@ H5FL_arr_free(H5FL_arr_head_t *head, void *obj) /* Make certain that the free list is initialized */ HDassert(head->init); +#ifdef H5FL_TRACK + { + unsigned char *block_ptr = ((unsigned char *)obj) - sizeof(H5FL_track_t); + H5FL_track_t trk; + + HDmemcpy(&trk, block_ptr, sizeof(H5FL_track_t)); + + /* Free tracking information about the allocation location */ + H5CS_close_stack(trk.stack); + /* The 'func' & 'file' strings are statically allocated (by the compiler) + * and are not allocated, so there's no need to free them. + */ + trk.file = NULL; + trk.func = NULL; + + /* Remove from "outstanding allocations" list */ + if ((void *)block_ptr == H5FL_out_head_g) { + H5FL_out_head_g = H5FL_out_head_g->next; + if (H5FL_out_head_g) + H5FL_out_head_g->prev = NULL; + } /* end if */ + else { + trk.prev->next = trk.next; + if (trk.next) + trk.next->prev = trk.prev; + } /* end else */ + + HDmemcpy(block_ptr, &trk, sizeof(H5FL_track_t)); + } +#endif + /* Get the pointer to the info header in front of the block to free */ temp = (H5FL_arr_list_t *)(( void *)((unsigned char *)obj - - sizeof(H5FL_arr_list_t))); /*lint !e826 Pointer-to-pointer cast is appropriate here */ + (sizeof(H5FL_arr_list_t) + + H5FL_TRACK_SIZE))); /*lint !e826 Pointer-to-pointer cast is appropriate here */ /* Get the number of elements */ free_nelem = temp->nelem; @@ -1490,7 +1524,7 @@ done: *------------------------------------------------------------------------- */ void * -H5FL_arr_malloc(H5FL_arr_head_t *head, size_t elem) +H5FL_arr_malloc(H5FL_arr_head_t *head, size_t elem H5FL_TRACK_PARAMS) { H5FL_arr_list_t *new_obj; /* Pointer to the new free list node allocated */ size_t mem_size; /* Size of memory block being recycled */ @@ -1531,7 +1565,8 @@ H5FL_arr_malloc(H5FL_arr_head_t *head, size_t elem) } /* end if */ /* Otherwise allocate a node */ else { - if (NULL == (new_obj = (H5FL_arr_list_t *)H5FL__malloc(sizeof(H5FL_arr_list_t) + mem_size))) + if (NULL == + (new_obj = (H5FL_arr_list_t *)H5FL__malloc(sizeof(H5FL_arr_list_t) + H5FL_TRACK_SIZE + mem_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Increment the number of blocks of this size */ @@ -1547,6 +1582,28 @@ H5FL_arr_malloc(H5FL_arr_head_t *head, size_t elem) /* Get a pointer to the new block */ ret_value = ((char *)new_obj) + sizeof(H5FL_arr_list_t); +#ifdef H5FL_TRACK + /* Copy allocation location information */ + ((H5FL_track_t *)ret_value)->stack = H5CS_copy_stack(); + HDassert(((H5FL_track_t *)ret_value)->stack); + /* The 'call_func' & 'call_file' strings are statically allocated (by the compiler) + * there's no need to duplicate them. + */ + ((H5FL_track_t *)ret_value)->file = call_file; + ((H5FL_track_t *)ret_value)->func = call_func; + ((H5FL_track_t *)ret_value)->line = call_line; + + /* Add to "outstanding allocations" list */ + ((H5FL_track_t *)ret_value)->prev = NULL; + ((H5FL_track_t *)ret_value)->next = H5FL_out_head_g; + if (H5FL_out_head_g) + H5FL_out_head_g->prev = (H5FL_track_t *)ret_value; + H5FL_out_head_g = (H5FL_track_t *)ret_value; + + /* Adjust for allocation tracking information */ + ret_value = ((unsigned char *)ret_value) + sizeof(H5FL_track_t); +#endif + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FL_arr_malloc() */ @@ -1565,7 +1622,7 @@ done: *------------------------------------------------------------------------- */ void * -H5FL_arr_calloc(H5FL_arr_head_t *head, size_t elem) +H5FL_arr_calloc(H5FL_arr_head_t *head, size_t elem H5FL_TRACK_PARAMS) { void *ret_value = NULL; /* Pointer to the block to return */ @@ -1576,7 +1633,7 @@ H5FL_arr_calloc(H5FL_arr_head_t *head, size_t elem) HDassert(elem); /* Allocate the array */ - if (NULL == (ret_value = H5FL_arr_malloc(head, elem))) + if (NULL == (ret_value = H5FL_arr_malloc(head, elem H5FL_TRACK_INFO_INT))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Clear to zeros */ @@ -1600,7 +1657,7 @@ done: *------------------------------------------------------------------------- */ void * -H5FL_arr_realloc(H5FL_arr_head_t *head, void *obj, size_t new_elem) +H5FL_arr_realloc(H5FL_arr_head_t *head, void *obj, size_t new_elem H5FL_TRACK_PARAMS) { void *ret_value = NULL; /* Pointer to the block to return */ @@ -1612,7 +1669,7 @@ H5FL_arr_realloc(H5FL_arr_head_t *head, void *obj, size_t new_elem) /* Check if we are really allocating the object */ if (obj == NULL) - ret_value = H5FL_arr_malloc(head, new_elem); + ret_value = H5FL_arr_malloc(head, new_elem H5FL_TRACK_INFO_INT); else { H5FL_arr_list_t *temp; /* Temp. ptr to the new free list node allocated */ @@ -1622,14 +1679,15 @@ H5FL_arr_realloc(H5FL_arr_head_t *head, void *obj, size_t new_elem) /* Get the pointer to the info header in front of the block to free */ temp = (H5FL_arr_list_t *)(( void *)((unsigned char *)obj - - sizeof(H5FL_arr_list_t))); /*lint !e826 Pointer-to-pointer cast is appropriate here */ + (sizeof(H5FL_arr_list_t) + + H5FL_TRACK_SIZE))); /*lint !e826 Pointer-to-pointer cast is appropriate here */ /* Check if the size is really changing */ if (temp->nelem != new_elem) { size_t blk_size; /* Size of block */ /* Get the new array of objects */ - ret_value = H5FL_arr_malloc(head, new_elem); + ret_value = H5FL_arr_malloc(head, new_elem H5FL_TRACK_INFO_INT); /* Copy the appropriate amount of elements */ blk_size = head->list_arr[MIN(temp->nelem, new_elem)].size; @@ -1638,8 +1696,35 @@ H5FL_arr_realloc(H5FL_arr_head_t *head, void *obj, size_t new_elem) /* Free the old block */ H5FL_arr_free(head, obj); } /* end if */ - else + else { +#ifdef H5FL_TRACK + unsigned char *block_ptr = ((unsigned char *)obj) - sizeof(H5FL_track_t); + H5FL_track_t trk; + + HDmemcpy(&trk, block_ptr, sizeof(H5FL_track_t)); + + /* Release previous tracking information */ + H5CS_close_stack(trk.stack); + /* The 'func' & 'file' strings are statically allocated (by the compiler) + * and are not allocated, so there's no need to free them. + */ + trk.file = NULL; + trk.func = NULL; + + /* Store new tracking information */ + trk.stack = H5CS_copy_stack(); + HDassert(trk.stack); + /* The 'call_func' & 'call_file' strings are statically allocated (by the compiler) + * there's no need to duplicate them. + */ + trk.file = call_file; + trk.func = call_func; + trk.line = call_line; + + HDmemcpy(block_ptr, &trk, sizeof(H5FL_track_t)); +#endif ret_value = obj; + } } /* end else */ FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FLprivate.h b/src/H5FLprivate.h index 381f977..6519551 100644 --- a/src/H5FLprivate.h +++ b/src/H5FLprivate.h @@ -148,8 +148,8 @@ typedef struct H5FL_reg_head_t { typedef union H5FL_blk_list_t { size_t size; /* Size of the page */ union H5FL_blk_list_t *next; /* Pointer to next block in free list */ - double unused1; /* Unused normally, just here for aligment */ - haddr_t unused2; /* Unused normally, just here for aligment */ + double unused1; /* Unused normally, just here for alignment */ + haddr_t unused2; /* Unused normally, just here for alignment */ } H5FL_blk_list_t; /* Data structure for priority queue node of block free lists */ @@ -223,8 +223,8 @@ typedef struct H5FL_blk_head_t { typedef union H5FL_arr_list_t { union H5FL_arr_list_t *next; /* Pointer to next block in free list */ size_t nelem; /* Number of elements in this array */ - double unused1; /* Unused normally, just here for aligment */ - haddr_t unused2; /* Unused normally, just here for aligment */ + double unused1; /* Unused normally, just here for alignment */ + haddr_t unused2; /* Unused normally, just here for alignment */ } H5FL_arr_list_t; /* Data structure for each size of array element */ diff --git a/src/H5FS.c b/src/H5FS.c index cb18ab3..3c259f4 100644 --- a/src/H5FS.c +++ b/src/H5FS.c @@ -61,9 +61,6 @@ static herr_t H5FS__sinfo_free_node_cb(void *item, void *key, void *op_data); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declare a free list to manage the H5FS_section_class_t sequence information */ H5FL_SEQ_DEFINE(H5FS_section_class_t); @@ -146,7 +143,8 @@ H5FS_create(H5F_t *f, haddr_t *fs_addr, const H5FS_create_t *fs_create, uint16_t /* Set the return value */ ret_value = fspace; #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: fspace = %p, fspace->addr = %a\n", __func__, fspace, fspace->addr); + HDfprintf(stderr, "%s: fspace = %p, fspace->addr = %" PRIuHADDR "\n", __func__, (void *)fspace, + fspace->addr); #endif /* H5FS_DEBUG */ done: @@ -155,7 +153,7 @@ done: HDONE_ERROR(H5E_FSPACE, H5E_CANTFREE, NULL, "unable to destroy free space header") #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", __func__, ret_value); + HDfprintf(stderr, "%s: Leaving, ret_value = %p\n", __func__, (void *)ret_value); #endif /* H5FS_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_create() */ @@ -183,8 +181,8 @@ H5FS_open(H5F_t *f, haddr_t fs_addr, uint16_t nclasses, const H5FS_section_class FUNC_ENTER_NOAPI(NULL) #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: Opening free space manager, fs_addr = %a, nclasses = %Zu\n", __func__, fs_addr, - nclasses); + HDfprintf(stderr, "%s: Opening free space manager, fs_addr = %" PRIuHADDR ", nclasses = %Zu\n", __func__, + fs_addr, nclasses); #endif /* H5FS_DEBUG */ /* Check arguments. */ @@ -204,10 +202,10 @@ H5FS_open(H5F_t *f, haddr_t fs_addr, uint16_t nclasses, const H5FS_section_class (fspace = (H5FS_t *)H5AC_protect(f, H5AC_FSPACE_HDR, fs_addr, &cache_udata, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_FSPACE, H5E_CANTPROTECT, NULL, "unable to load free space header") #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: fspace->sect_addr = %a\n", __func__, fspace->sect_addr); - HDfprintf(stderr, "%s: fspace->sect_size = %Hu\n", __func__, fspace->sect_size); - HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu\n", __func__, fspace->alloc_sect_size); - HDfprintf(stderr, "%s: fspace->sinfo = %p\n", __func__, fspace->sinfo); + HDfprintf(stderr, "%s: fspace->sect_addr = %" PRIuHADDR "\n", __func__, fspace->sect_addr); + HDfprintf(stderr, "%s: fspace->sect_size = %" PRIuHSIZE "\n", __func__, fspace->sect_size); + HDfprintf(stderr, "%s: fspace->alloc_sect_size = %" PRIuHSIZE "\n", __func__, fspace->alloc_sect_size); + HDfprintf(stderr, "%s: fspace->sinfo = %p\n", __func__, (void *)fspace->sinfo); HDfprintf(stderr, "%s: fspace->rc = %u\n", __func__, fspace->rc); #endif /* H5FS_DEBUG */ @@ -251,7 +249,7 @@ H5FS_delete(H5F_t *f, haddr_t fs_addr) FUNC_ENTER_NOAPI(FAIL) #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: Deleting free space manager, fs_addr = %a\n", __func__, fs_addr); + HDfprintf(stderr, "%s: Deleting free space manager, fs_addr = %" PRIuHADDR "\n", __func__, fs_addr); #endif /* H5FS_DEBUG */ /* Check arguments. */ @@ -321,7 +319,7 @@ H5FS_delete(H5F_t *f, haddr_t fs_addr) /* Delete serialized section storage, if there are any */ #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: fspace->sect_addr = %a\n", __func__, fspace->sect_addr); + HDfprintf(stderr, "%s: fspace->sect_addr = %" PRIuHADDR "\n", __func__, fspace->sect_addr); #endif /* H5FS_DEBUG */ if (fspace->serial_sect_count > 0) { unsigned sinfo_status = 0; /* Free space section info's status in the metadata cache */ @@ -407,8 +405,8 @@ H5FS_close(H5F_t *f, H5FS_t *fspace) HDassert(f); HDassert(fspace); #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: Entering, fspace = %p, fspace->addr = %a, fspace->sinfo = %p\n", __func__, fspace, - fspace->addr, fspace->sinfo); + HDfprintf(stderr, "%s: Entering, fspace = %p, fspace->addr = %" PRIuHADDR ", fspace->sinfo = %p\n", + __func__, (void *)fspace, fspace->addr, (void *)fspace->sinfo); #endif /* H5FS_DEBUG */ /* Check if section info is valid */ @@ -416,11 +414,12 @@ H5FS_close(H5F_t *f, H5FS_t *fspace) if (fspace->sinfo) { #ifdef H5FS_DEBUG HDfprintf(stderr, - "%s: fspace->tot_sect_count = %Hu, fspace->serial_sect_count = %Hu, fspace->sect_addr = " - "%a, fspace->rc = %u\n", + "%s: fspace->tot_sect_count = %" PRIuHSIZE ", fspace->serial_sect_count = %" PRIuHSIZE + ", fspace->sect_addr = %" PRIuHADDR ", fspace->rc = %u\n", __func__, fspace->tot_sect_count, fspace->serial_sect_count, fspace->sect_addr, fspace->rc); - HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n", __func__, - fspace->alloc_sect_size, fspace->sect_size); + HDfprintf(stderr, + "%s: fspace->alloc_sect_size = %" PRIuHSIZE ", fspace->sect_size = %" PRIuHSIZE "\n", + __func__, fspace->alloc_sect_size, fspace->sect_size); #endif /* H5FS_DEBUG */ /* If there are sections to serialize, update them */ /* (if the free space manager is persistent) */ @@ -711,7 +710,7 @@ H5FS__incr(H5FS_t *fspace) FUNC_ENTER_PACKAGE #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: Entering, fpace->addr = %a, fspace->rc = %u\n", __func__, fspace->addr, + HDfprintf(stderr, "%s: Entering, fpace->addr = %" PRIuHADDR ", fspace->rc = %u\n", __func__, fspace->addr, fspace->rc); #endif /* H5FS_DEBUG */ @@ -751,7 +750,7 @@ H5FS__decr(H5FS_t *fspace) FUNC_ENTER_PACKAGE #ifdef H5FS_DEBUG - HDfprintf(stderr, "%s: Entering, fpace->addr = %a, fspace->rc = %u\n", __func__, fspace->addr, + HDfprintf(stderr, "%s: Entering, fpace->addr = %" PRIuHADDR ", fspace->rc = %u\n", __func__, fspace->addr, fspace->rc); #endif /* H5FS_DEBUG */ diff --git a/src/H5FSint.c b/src/H5FSint.c index 0ab975b..d7950ab 100644 --- a/src/H5FSint.c +++ b/src/H5FSint.c @@ -68,31 +68,6 @@ /*******************/ /*------------------------------------------------------------------------- - * Function: H5FS_init - * - * Purpose: Initialize the interface in case it is unable to initialize - * itself soon enough. - * - * Return: Success: non-negative - * Failure: negative - * - * Programmer: Quincey Koziol - * Saturday, March 4, 2000 - * - *------------------------------------------------------------------------- - */ -herr_t -H5FS_init(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOERR - /* FUNC_ENTER() does all the work */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FS_init() */ - -/*------------------------------------------------------------------------- * Function: H5FS__create_flush_depend * * Purpose: Create a flush dependency between two data structure components diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h index 67983d3..679cb26 100644 --- a/src/H5FSprivate.h +++ b/src/H5FSprivate.h @@ -116,7 +116,7 @@ typedef struct H5FS_section_class_t { const struct H5FS_section_class_t *, const uint8_t *, haddr_t, hsize_t, unsigned *); /* Routine to deserialize a buffer into a "live" section */ htri_t (*can_merge)(const H5FS_section_info_t *, const H5FS_section_info_t *, - void *); /* Routine to determine if two nodes are mergable */ + void *); /* Routine to determine if two nodes are mergeable */ herr_t (*merge)(H5FS_section_info_t **, H5FS_section_info_t *, void *); /* Routine to merge two nodes */ htri_t (*can_shrink)(const H5FS_section_info_t *, void *); /* Routine to determine if node can shrink container */ @@ -187,9 +187,6 @@ H5FL_SEQ_EXTERN(H5FS_section_class_t); /* Library-private Function Prototypes */ /***************************************/ -/* Package initialization routine */ -H5_DLL herr_t H5FS_init(void); - /* Free space manager routines */ H5_DLL H5FS_t *H5FS_create(H5F_t *f, haddr_t *fs_addr, const H5FS_create_t *fs_create, uint16_t nclasses, const H5FS_section_class_t *classes[], void *cls_init_udata, hsize_t alignment, diff --git a/src/H5FSsection.c b/src/H5FSsection.c index d28358c..55f8a94 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -123,7 +123,7 @@ H5FS__sinfo_new(H5F_t *f, H5FS_t *fspace) HDassert(f); HDassert(fspace); #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: fspace->addr = %a\n", __func__, fspace->addr); + HDfprintf(stderr, "%s: fspace->addr = %" PRIuHADDR "\n", __func__, fspace->addr); #endif /* H5FS_SINFO_DEBUG */ /* Allocate the free space header */ @@ -136,7 +136,7 @@ H5FS__sinfo_new(H5F_t *f, H5FS_t *fspace) sinfo->sect_off_size = (fspace->max_sect_addr + 7) / 8; sinfo->sect_len_size = H5VM_limit_enc_size((uint64_t)fspace->max_sect_size); #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: fspace->max_sect_size = %Hu\n", __func__, fspace->max_sect_size); + HDfprintf(stderr, "%s: fspace->max_sect_size = %" PRIuHSIZE "\n", __func__, fspace->max_sect_size); HDfprintf(stderr, "%s: fspace->max_sect_addr = %u\n", __func__, fspace->max_sect_addr); HDfprintf(stderr, "%s: sinfo->nbins = %u\n", __func__, sinfo->nbins); HDfprintf(stderr, "%s: sinfo->sect_off_size = %u, sinfo->sect_len_size = %u\n", __func__, @@ -200,10 +200,12 @@ H5FS__sinfo_lock(H5F_t *f, H5FS_t *fspace, unsigned accmode) FUNC_ENTER_STATIC #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: Called, fspace->addr = %a, fspace->sinfo = %p, fspace->sect_addr = %a\n", __func__, - fspace->addr, fspace->sinfo, fspace->sect_addr); - HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n", __func__, - fspace->alloc_sect_size, fspace->sect_size); + HDfprintf(stderr, + "%s: Called, fspace->addr = %" PRIuHADDR ", fspace->sinfo = %p, fspace->sect_addr = %" PRIuHADDR + "\n", + __func__, fspace->addr, (void *)fspace->sinfo, fspace->sect_addr); + HDfprintf(stderr, "%s: fspace->alloc_sect_size = %" PRIuHSIZE ", fspace->sect_size = %" PRIuHSIZE "\n", + __func__, fspace->alloc_sect_size, fspace->sect_size); #endif /* H5FS_SINFO_DEBUG */ /* Check arguments. */ @@ -251,8 +253,8 @@ H5FS__sinfo_lock(H5F_t *f, H5FS_t *fspace, unsigned accmode) HDassert(H5F_addr_defined(fspace->addr)); #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: Reading in existing sections, fspace->sect_addr = %a\n", __func__, - fspace->sect_addr); + HDfprintf(stderr, "%s: Reading in existing sections, fspace->sect_addr = %" PRIuHADDR "\n", + __func__, fspace->sect_addr); #endif /* H5FS_SINFO_DEBUG */ /* Protect the free space sections */ cache_udata.f = f; @@ -289,10 +291,12 @@ H5FS__sinfo_lock(H5F_t *f, H5FS_t *fspace, unsigned accmode) done: #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: Leaving, fspace->addr = %a, fspace->sinfo = %p, fspace->sect_addr = %a\n", - __func__, fspace->addr, fspace->sinfo, fspace->sect_addr); - HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n", __func__, - fspace->alloc_sect_size, fspace->sect_size); + HDfprintf(stderr, + "%s: Leaving, fspace->addr = %" PRIuHADDR + ", fspace->sinfo = %p, fspace->sect_addr = %" PRIuHADDR "\n", + __func__, fspace->addr, (void *)fspace->sinfo, fspace->sect_addr); + HDfprintf(stderr, "%s: fspace->alloc_sect_size = %" PRIuHSIZE ", fspace->sect_size = %" PRIuHSIZE "\n", + __func__, fspace->alloc_sect_size, fspace->sect_size); #endif /* H5FS_SINFO_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) } /* H5FS__sinfo_lock() */ @@ -331,14 +335,16 @@ H5FS__sinfo_unlock(H5F_t *f, H5FS_t *fspace, hbool_t modified) FUNC_ENTER_STATIC #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: Called, modified = %t, fspace->addr = %a, fspace->sect_addr = %a\n", __func__, - modified, fspace->addr, fspace->sect_addr); + HDfprintf(stderr, + "%s: Called, modified = %d, fspace->addr = %" PRIuHADDR ", fspace->sect_addr = %" PRIuHADDR + "\n", + __func__, modified, fspace->addr, fspace->sect_addr); HDfprintf( stderr, - "%s: fspace->sinfo_lock_count = %u, fspace->sinfo_modified = %t, fspace->sinfo_protected = %t\n", + "%s: fspace->sinfo_lock_count = %u, fspace->sinfo_modified = %d, fspace->sinfo_protected = %d\n", __func__, fspace->sinfo_lock_count, fspace->sinfo_modified, fspace->sinfo_protected); - HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n", __func__, - fspace->alloc_sect_size, fspace->sect_size); + HDfprintf(stderr, "%s: fspace->alloc_sect_size = %" PRIuHSIZE ", fspace->sect_size = %" PRIuHSIZE "\n", + __func__, fspace->alloc_sect_size, fspace->sect_size); #endif /* H5FS_SINFO_DEBUG */ /* Check arguments. */ @@ -490,7 +496,8 @@ H5FS__sinfo_unlock(H5F_t *f, H5FS_t *fspace, hbool_t modified) #ifdef H5FS_SINFO_DEBUG HDfprintf(stderr, - "%s: Freeing section info on disk, old_sect_addr = %a, old_alloc_sect_size = %Hu\n", + "%s: Freeing section info on disk, old_sect_addr = %" PRIuHADDR + ", old_alloc_sect_size = %" PRIuHSIZE "\n", __func__, old_sect_addr, old_alloc_sect_size); #endif /* H5FS_SINFO_DEBUG */ /* Release space for section info in file */ @@ -1343,7 +1350,8 @@ H5FS_sect_add(H5F_t *f, H5FS_t *fspace, H5FS_section_info_t *sect, unsigned flag FUNC_ENTER_NOAPI(FAIL) #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: *sect = {%a, %Hu, %u, %s}\n", __func__, sect->addr, sect->size, sect->type, + HDfprintf(stderr, "%s: *sect = {%" PRIuHADDR ", %" PRIuHSIZE ", %u, %s}\n", __func__, sect->addr, + sect->size, sect->type, (sect->state == H5FS_SECT_LIVE ? "H5FS_SECT_LIVE" : "H5FS_SECT_SERIALIZED")); #endif /* H5FS_SINFO_DEBUG */ @@ -1384,7 +1392,7 @@ H5FS_sect_add(H5F_t *f, H5FS_t *fspace, H5FS_section_info_t *sect, unsigned flag HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list") #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: fspace->tot_space = %Hu\n", __func__, fspace->tot_space); + HDfprintf(stderr, "%s: fspace->tot_space = %" PRIuHSIZE "\n", __func__, fspace->tot_space); #endif /* H5FS_SINFO_DEBUG */ /* Mark free space sections as changed */ /* (if adding sections while deserializing sections, don't set the flag) */ @@ -1429,8 +1437,8 @@ H5FS_sect_try_extend(H5F_t *f, H5FS_t *fspace, haddr_t addr, hsize_t size, hsize FUNC_ENTER_NOAPI(FAIL) #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: addr = %a, size = %Hu, extra_requested = %hu\n", __func__, addr, size, - extra_requested); + HDfprintf(stderr, "%s: addr = %" PRIuHADDR ", size = %" PRIuHSIZE ", extra_requested = %" PRIuHSIZE "\n", + __func__, addr, size, extra_requested); #endif /* H5FS_SINFO_DEBUG */ /* Check arguments. */ @@ -1442,9 +1450,10 @@ H5FS_sect_try_extend(H5F_t *f, H5FS_t *fspace, haddr_t addr, hsize_t size, hsize /* Check for any sections on free space list */ #ifdef H5FS_SINFO_DEBUG - HDfprintf(stderr, "%s: fspace->tot_sect_count = %Hu\n", __func__, fspace->tot_sect_count); - HDfprintf(stderr, "%s: fspace->serial_sect_count = %Hu\n", __func__, fspace->serial_sect_count); - HDfprintf(stderr, "%s: fspace->ghost_sect_count = %Hu\n", __func__, fspace->ghost_sect_count); + HDfprintf(stderr, "%s: fspace->tot_sect_count = %" PRIuHSIZE "\n", __func__, fspace->tot_sect_count); + HDfprintf(stderr, "%s: fspace->serial_sect_count = %" PRIuHSIZE "\n", __func__, + fspace->serial_sect_count); + HDfprintf(stderr, "%s: fspace->ghost_sect_count = %" PRIuHSIZE "\n", __func__, fspace->ghost_sect_count); #endif /* H5FS_SINFO_DEBUG */ if (fspace->tot_sect_count > 0) { H5FS_section_info_t *sect; /* Temporary free space section */ @@ -2082,11 +2091,11 @@ H5FS_sect_change_class(H5F_t *f, H5FS_t *fspace, H5FS_section_info_t *sect, uint } /* end else */ } /* end if */ - /* Check if the section's class change will affect the mergable list */ + /* Check if the section's class change will affect the mergeable list */ if ((old_cls->flags & H5FS_CLS_SEPAR_OBJ) != (new_cls->flags & H5FS_CLS_SEPAR_OBJ)) { - hbool_t to_mergable; /* Flag if the section is changing to a mergable section */ + hbool_t to_mergable; /* Flag if the section is changing to a mergeable section */ - /* Determine if this section is becoming mergable or is becoming separate */ + /* Determine if this section is becoming mergeable or is becoming separate */ if (old_cls->flags & H5FS_CLS_SEPAR_OBJ) to_mergable = TRUE; else diff --git a/src/H5Fcwfs.c b/src/H5Fcwfs.c index d689737..43c8cf7 100644 --- a/src/H5Fcwfs.c +++ b/src/H5Fcwfs.c @@ -252,7 +252,7 @@ H5F_cwfs_advance_heap(H5F_t *f, H5HG_heap_t *heap, hbool_t add_heap) unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Check args */ HDassert(f); @@ -272,7 +272,6 @@ H5F_cwfs_advance_heap(H5F_t *f, H5HG_heap_t *heap, hbool_t add_heap) f->shared->cwfs[f->shared->ncwfs - 1] = heap; } /* end if */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* H5F_cwfs_advance_heap() */ @@ -295,7 +294,7 @@ H5F_cwfs_remove_heap(H5F_shared_t *shared, H5HG_heap_t *heap) unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Check args */ HDassert(shared); @@ -310,6 +309,5 @@ H5F_cwfs_remove_heap(H5F_shared_t *shared, H5HG_heap_t *heap) } /* end if */ } /* end for */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* H5F_cwfs_remove_heap() */ diff --git a/src/H5Fefc.c b/src/H5Fefc.c index 2d4eff2..2c7dd92 100644 --- a/src/H5Fefc.c +++ b/src/H5Fefc.c @@ -450,7 +450,7 @@ H5F__efc_release_real(H5F_efc_t *efc) /* Sanity checks */ HDassert(efc); - /* Lock the EFC to prevent manipulation of the EFC wile we are releasing it. + /* Lock the EFC to prevent manipulation of the EFC while we are releasing it. * The EFC should never be locked when we enter this function because that * would require a cycle, a cycle would necessarily invoke * H5F__efc_try_close(), and that function checks the status of the lock diff --git a/src/H5Fint.c b/src/H5Fint.c index b80fd15..729aeea 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -92,9 +92,6 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Based on the value of the HDF5_USE_FILE_LOCKING environment variable. * TRUE/FALSE have obvious meanings. FAIL means the environment variable was * not set, so the code should ignore it and use the fapl value instead. @@ -139,29 +136,6 @@ H5F_init(void) herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_init() */ - -/*-------------------------------------------------------------------------- -NAME - H5F__init_package -- Initialize interface-specific information -USAGE - herr_t H5F__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5F__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE /* Initialize the ID group for the file IDs */ if (H5I_register_type(H5I_FILE_CLS) < 0) @@ -173,7 +147,7 @@ H5F__init_package(void) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5F__init_package() */ +} /* end H5F_init() */ /*------------------------------------------------------------------------- * Function: H5F_term_package @@ -197,23 +171,17 @@ H5F_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - if (H5I_nmembers(H5I_FILE) > 0) { - (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - else { - /* Make certain we've cleaned up all the shared file objects */ - H5F_sfile_assert_num(0); - - /* Destroy the file object id group */ - n += (H5I_dec_type_ref(H5I_FILE) > 0); + if (H5I_nmembers(H5I_FILE) > 0) { + (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + else { + /* Make certain we've cleaned up all the shared file objects */ + H5F_sfile_assert_num(0); - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ + /* Destroy the file object id group */ + n += (H5I_dec_type_ref(H5I_FILE) > 0); + } /* end else */ FUNC_LEAVE_NOAPI(n) } /* end H5F_term_package() */ @@ -278,7 +246,7 @@ H5F__parse_file_lock_env_var(htri_t *use_locks) FUNC_ENTER_PACKAGE_NOERR /* Check the file locking environment variable */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + lock_env_var = HDgetenv(HDF5_USE_FILE_LOCKING); if (lock_env_var && (!HDstrcmp(lock_env_var, "FALSE") || !HDstrcmp(lock_env_var, "0"))) *use_locks = FALSE; /* Override: Never use locks */ else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "BEST_EFFORT") || @@ -430,7 +398,7 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) } /* end if */ if (H5P_set(new_plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, &(f->shared->vfd_swmr_config)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set initial metadata cache resize config.") + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set VFD SWMR config") #ifdef H5_HAVE_PARALLEL if (H5P_set(new_plist, H5_COLL_MD_READ_FLAG_NAME, &(f->shared->coll_md_read)) < 0) @@ -459,9 +427,10 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, H5I_INVALID_HID, "can't set initial metadata cache resize config.") /* Prepare the driver property */ - driver_prop.driver_id = f->shared->lf->driver_id; - driver_prop.driver_info = H5FD_fapl_get(f->shared->lf); - driver_prop_copied = TRUE; + driver_prop.driver_id = f->shared->lf->driver_id; + driver_prop.driver_info = H5FD_fapl_get(f->shared->lf); + driver_prop.driver_config_str = H5P_peek_driver_config_str(old_plist); + driver_prop_copied = TRUE; /* Set the driver property */ if (H5P_set(new_plist, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0) @@ -1064,7 +1033,7 @@ H5F__is_hdf5(const char *name, hid_t fapl_id) { H5FD_t * file = NULL; /* Low-level file struct */ H5F_shared_t *shared = NULL; /* Shared part of file */ - haddr_t sig_addr = HADDR_UNDEF; /* Addess of hdf5 file signature */ + haddr_t sig_addr = HADDR_UNDEF; /* Address of hdf5 file signature */ htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_PACKAGE @@ -1161,7 +1130,7 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F /* Initialization for handling file space (for paged aggregation) */ f->shared->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; - /* intialize point of no return */ + /* initialize point of no return */ f->shared->point_of_no_return = FALSE; /* Copy the file creation and file access property lists into the @@ -1298,7 +1267,7 @@ H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5F f->shared->read_attempts = H5F_METADATA_READ_ATTEMPTS; } - /* Determine the # of bins for metdata read retries */ + /* Determine the # of bins for metadata read retries */ if (H5F_set_retries(f) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set retries and retries_nbins") @@ -1874,7 +1843,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* When configured with VFD SWMR */ if (vfd_swmr_config_ptr->version) { - /* Verify that file access flags are consistent with VFD SWMR configuartion */ + /* Verify that file access flags are consistent with VFD SWMR configuration */ if ((flags & H5F_ACC_RDWR) && !vfd_swmr_config_ptr->writer) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "file access is writer but VFD SWMR config is reader") if ((flags & H5F_ACC_RDWR) == 0 && vfd_swmr_config_ptr->writer) @@ -1910,7 +1879,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) * or creating it) so we can compare it with files that are already * open. If that fails then we try again with the full set of flags * (only if they're different than the original failed attempt). - * However, if the file driver can't distinquish between files then + * However, if the file driver can't distinguish between files then * there's no reason to open the file tentatively because it's the * application's responsibility to prevent this situation (there's no * way for us to detect it here anyway). @@ -2020,8 +1989,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Short cuts */ shared = file->shared; - - lf = shared->lf; + lf = shared->lf; /* Set the file locking flag. If the file is already open, the file * requested file locking flag must match that of the open file. @@ -3304,7 +3272,7 @@ H5F__get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, size_t *image_le * file driver. However, this test will not work if there is some * other file driver sitting on top of the multi file driver. * - * I'm not sure if this is possible at present, but in all likelyhood, + * I'm not sure if this is possible at present, but in all likelihood, * it will become possible in the future. On the other hand, we may * remove the split/multi file drivers before then. * @@ -3324,7 +3292,7 @@ H5F__get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, size_t *image_le * * While this problem is quite solvable, the required time and * resources are lacking at present. Hence, for now, we don't - * allow the get file image operation to be perfomed on files + * allow the get file image operation to be performed on files * opened with the family file driver. * * Observe that the following test only looks at the top level @@ -3755,7 +3723,7 @@ done: * 1) The file being opened has v3 superblock * 2) The file is opened with H5F_ACC_RDWR * 3) The file is not already marked for SWMR writing - * 4) Current implementaion for opened objects: + * 4) Current implementation for opened objects: * --only allow datasets and groups without attributes * --disallow named datatype with/without attributes * --disallow opened attributes attached to objects @@ -3774,7 +3742,7 @@ herr_t H5F__start_swmr_write(H5F_t *f) { hbool_t ci_load = FALSE; /* whether MDC ci load requested */ - hbool_t ci_write = FALSE; /* whether MDC ci write requested */ + hbool_t ci_write = FALSE; /* whether MDC CI write requested */ size_t grp_dset_count = 0; /* # of open objects: groups & datasets */ size_t nt_attr_count = 0; /* # of opened named datatypes + opened attributes */ hid_t * obj_ids = NULL; /* List of ids */ diff --git a/src/H5Fmount.c b/src/H5Fmount.c index 4b90ea3..7d27234 100644 --- a/src/H5Fmount.c +++ b/src/H5Fmount.c @@ -104,7 +104,7 @@ H5F_mount(const H5G_loc_t *loc, const char *name, H5F_t *child, hid_t H5_ATTR_UN H5F_t * parent = NULL; /*file containing mount point */ unsigned lt, rt, md; /*binary search indices */ int cmp; /*binary search comparison value*/ - H5G_loc_t mp_loc; /* entry of moint point to be opened */ + H5G_loc_t mp_loc; /* entry of mount point to be opened */ H5G_name_t mp_path; /* Mount point group hier. path */ H5O_loc_t mp_oloc; /* Mount point object location */ H5G_loc_t root_loc; /* Group location of root of file to mount */ diff --git a/src/H5Fmpi.c b/src/H5Fmpi.c index 53d2d78..02d8d52 100644 --- a/src/H5Fmpi.c +++ b/src/H5Fmpi.c @@ -31,11 +31,12 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fpkg.h" /* File access */ -#include "H5FDprivate.h" /* File drivers */ -#include "H5Iprivate.h" /* IDs */ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Iprivate.h" /* IDs */ #include "H5VLnative_private.h" /* Native VOL connector */ @@ -402,4 +403,189 @@ H5F_mpi_retrieve_comm(hid_t loc_id, hid_t acspl_id, MPI_Comm *mpi_comm) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_mpi_retrieve_comm */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_coll_metadata_reads + * + * Purpose: Determines whether collective metadata reads should be + * performed. This routine is meant to be the single source of + * truth for the collective metadata reads status, as it + * coordinates between the file-global flag and the flag set + * for the current operation in the current API context. + * + * Return: TRUE/FALSE (can't fail) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_coll_metadata_reads(const H5F_t *file) +{ + H5P_coll_md_read_flag_t file_flag = H5P_USER_FALSE; + hbool_t ret_value = FALSE; + + FUNC_ENTER_NOAPI_NOERR + + HDassert(file && file->shared); + + /* Retrieve the file-global flag */ + file_flag = H5F_COLL_MD_READ(file); + + /* If file flag is set to H5P_FORCE_FALSE, exit early + * with FALSE, since collective metadata reads have + * been explicitly disabled somewhere in the library. + */ + if (H5P_FORCE_FALSE == file_flag) + ret_value = FALSE; + else { + /* If file flag is set to H5P_USER_TRUE, ignore + * any settings in the API context. A file-global + * setting of H5P_USER_TRUE for collective metadata + * reads should ignore any settings on an Access + * Property List for an individual operation. + */ + if (H5P_USER_TRUE == file_flag) + ret_value = TRUE; + else { + /* Get the collective metadata reads flag from + * the current API context. + */ + ret_value = H5CX_get_coll_metadata_read(); + } + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_get_coll_metadata_reads() */ + +/*------------------------------------------------------------------------- + * Function: H5F_set_coll_metadata_reads + * + * Purpose: Used to temporarily modify the collective metadata reads + * status. This is useful for cases where either: + * + * * Collective metadata reads are enabled, but need to be + * disabled for an operation about to occur that may trigger + * an independent metadata read (such as only rank 0 doing + * something) + * + * * Metadata reads are currently independent, but it is + * guaranteed that the application has maintained + * collectivity at the interface level (e.g., an operation + * that modifies metadata is being performed). In this case, + * it should be safe to enable collective metadata reads, + * barring any internal library issues that may occur + * + * After completion, the `file_flag` parameter will be set to + * the previous value of the file-global collective metadata + * reads flag. The `context_flag` parameter will be set to the + * previous value of the API context's collective metadata + * reads flag. Another call to this routine should be made to + * restore these values (see below warning). + * + * !! WARNING !! + * It is dangerous to modify the collective metadata reads + * status, as this can cause crashes, hangs and corruption in + * the HDF5 file when improperly done. Therefore, the + * `file_flag` and `context_flag` parameters are both + * mandatory, and it is assumed that the caller will guarantee + * these settings are restored with another call to this + * routine once the bracketed operation is complete. + * !! WARNING !! + * + * Return: Nothing + * + *------------------------------------------------------------------------- + */ +void +H5F_set_coll_metadata_reads(H5F_t *file, H5P_coll_md_read_flag_t *file_flag, hbool_t *context_flag) +{ + H5P_coll_md_read_flag_t prev_file_flag = H5P_USER_FALSE; + hbool_t prev_context_flag = FALSE; + + FUNC_ENTER_NOAPI_NOERR + + HDassert(file && file->shared); + HDassert(file_flag); + HDassert(context_flag); + + /* Save old state */ + prev_file_flag = H5F_COLL_MD_READ(file); + prev_context_flag = H5CX_get_coll_metadata_read(); + + /* Set new desired state */ + if (prev_file_flag != *file_flag) { + file->shared->coll_md_read = *file_flag; + *file_flag = prev_file_flag; + } + if (prev_context_flag != *context_flag) { + H5CX_set_coll_metadata_read(*context_flag); + *context_flag = prev_context_flag; + } + + FUNC_LEAVE_NOAPI_VOID +} /* end H5F_set_coll_metadata_reads() */ + +/*------------------------------------------------------------------------- + * Function: H5F_mpi_get_file_block_type + * + * Purpose: Creates an MPI derived datatype for communicating an + * H5F_block_t structure. If `commit` is specified as TRUE, + * the resulting datatype will be committed and ready for + * use in communication. Otherwise, the type is only suitable + * for building other derived types. + * + * If TRUE is returned through `new_type_derived`, this lets + * the caller know that the datatype has been derived and + * should be freed with MPI_Type_free once it is no longer + * needed. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_mpi_get_file_block_type(hbool_t commit, MPI_Datatype *new_type, hbool_t *new_type_derived) +{ + MPI_Datatype types[2]; + MPI_Aint displacements[2]; + int block_lengths[2]; + int field_count; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(new_type); + HDassert(new_type_derived); + + *new_type_derived = FALSE; + + field_count = 2; + HDassert(field_count == sizeof(types) / sizeof(MPI_Datatype)); + + block_lengths[0] = 1; + block_lengths[1] = 1; + displacements[0] = offsetof(H5F_block_t, offset); + displacements[1] = offsetof(H5F_block_t, length); + types[0] = HADDR_AS_MPI_TYPE; + types[1] = HSIZE_AS_MPI_TYPE; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + *new_type_derived = TRUE; + + if (commit && MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + +done: + if (ret_value < 0) { + if (*new_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(new_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *new_type_derived = FALSE; + } + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_mpi_get_file_block_type() */ + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index ca26941..2edcc60 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -79,7 +79,7 @@ #define H5F_SUPERBLOCK_FIXED_SIZE (H5F_SIGNATURE_LEN + 1) /* superblock version */ /* The H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE is the minimal amount of super block - * variable length data guarnateed to load the sizeof offsets and the sizeof + * variable length data guaranteed to load the sizeof offsets and the sizeof * lengths fields in all versions of the superblock. * * This is necessary in the V3 cache, as on the initial load, we need to @@ -218,10 +218,10 @@ typedef struct H5F_mtab_t { * the shadow file to release after max_lag ticks. */ typedef struct shadow_defree { - uint64_t offset; // offset of the region in *bytes* - uint32_t length; // length of the region in *bytes* - uint64_t tick_num; // tick number when the free was deferred - TAILQ_ENTRY(shadow_defree) link; // deferred-free queue linkage + uint64_t offset; /* offset of the region in *bytes* */ + uint32_t length; /* length of the region in *bytes* */ + uint64_t tick_num; /* tick number when the free was deferred */ + TAILQ_ENTRY(shadow_defree) link; /* deferred-free queue linkage */ } shadow_defree_t; /* Structure specifically to store superblock. This was originally @@ -247,10 +247,10 @@ typedef struct H5F_super_t { * the file below the SWMR VFD to release after a delay. */ typedef struct lower_defree { - SIMPLEQ_ENTRY(lower_defree) link; // deferred-free queue linkage - H5FD_mem_t alloc_type; // type with which the region was allocated - haddr_t addr; // start of the region *in bytes* - hsize_t size; // length of the region *in bytes* + SIMPLEQ_ENTRY(lower_defree) link; /* deferred-free queue linkage */ + H5FD_mem_t alloc_type; /* type with which the region was allocated */ + haddr_t addr; /* start of the region *in bytes* */ + hsize_t size; /* length of the region *in bytes* */ uint64_t free_after_tick; /* the region may be reused on tick * free_after_tick + 1 at the earliest */ @@ -385,7 +385,7 @@ struct H5F_shared_t { /* Metadata retry info */ unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */ unsigned retries_nbins; /* # of bins for each retries[] */ - uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */ + uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metadata items with checksum */ /* Object flush info */ H5F_object_flush_t object_flush; /* Information for object flush callback */ @@ -441,7 +441,7 @@ struct H5F_shared_t { */ uint32_t mdf_idx_entries_used; /* Number of entries in *mdf_idx * that are in use -- these will - * be contiguous at indicies 0 + * be contiguous at indices 0 * through mdf_idx_entries_used - 1. */ @@ -482,7 +482,7 @@ struct H5F_shared_t { * file */ uint64_t updater_seq_num; /* Sequence number of the next updater file to be - * genereated. This field must be initialized to zero, + * generated. This field must be initialized to zero, * and incremented after each updater file is generated. */ @@ -659,14 +659,14 @@ H5_DLL herr_t H5F__reparse_file_lock_variable_test(void); * in the testhdf5 program, which involves the test of a non-existing HDF5 file. */ -H5_DLL void H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info); +H5_DLL void H5F__post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, const char *log_info); #define H5F_POST_VFD_SWMR_LOG_ENTRY_INTERNAL(fp, entry_type_code, log_info) \ do { \ if (fp != NULL) { \ if (fp->shared != NULL) { \ if (fp->shared->vfd_swmr_log_on == TRUE) { \ - H5F_post_vfd_swmr_log_entry(fp, entry_type_code, log_info); \ + H5F__post_vfd_swmr_log_entry(fp, entry_type_code, log_info); \ } \ } \ } \ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 9a4f363..e94ee64 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -24,11 +24,9 @@ typedef struct H5F_t H5F_t; /* Include package's public header */ #include "H5Fpublic.h" -/* Public headers needed by this file */ -#include "H5FDpublic.h" /* File drivers */ - /* Private headers needed by this file */ #include "H5MMprivate.h" /* Memory management */ +#include "H5FDprivate.h" /* File drivers */ #ifdef H5_HAVE_PARALLEL #include "H5Pprivate.h" /* Property lists */ #endif /* H5_HAVE_PARALLEL */ @@ -938,7 +936,7 @@ typedef enum H5F_prefix_open_t { * the past tick is used to assemble the associated data in preparation for * writing an updater file. * - * Each entry in this array pertains to a given modified metdata page or + * Each entry in this array pertains to a given modified metadata page or * multi-page metadata entry, and contains the following fields: * * entry_image_ptr: void pointer to a buffer containing the image of the @@ -1187,6 +1185,7 @@ H5_DLL hbool_t H5F_shared_has_feature(const H5F_shared_t *f, unsigned feature); H5_DLL hbool_t H5F_has_feature(const H5F_t *f, unsigned feature); H5_DLL haddr_t H5F_shared_get_eoa(const H5F_shared_t *f_sh, H5FD_mem_t type); H5_DLL haddr_t H5F_get_eoa(const H5F_t *f, H5FD_mem_t type); +H5_DLL herr_t H5F_shared_get_file_driver(const H5F_shared_t *f_sh, H5FD_t **file_handle); H5_DLL herr_t H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_handle); /* VFD SWMR functions */ @@ -1247,6 +1246,9 @@ H5_DLL MPI_Comm H5F_mpi_get_comm(const H5F_t *f); H5_DLL int H5F_shared_mpi_get_size(const H5F_shared_t *f_sh); H5_DLL int H5F_mpi_get_size(const H5F_t *f); H5_DLL herr_t H5F_mpi_retrieve_comm(hid_t loc_id, hid_t acspl_id, MPI_Comm *mpi_comm); +H5_DLL herr_t H5F_mpi_get_file_block_type(hbool_t commit, MPI_Datatype *new_type, hbool_t *new_type_derived); +H5_DLL hbool_t H5F_get_coll_metadata_reads(const H5F_t *f); +H5_DLL void H5F_set_coll_metadata_reads(H5F_t *f, H5P_coll_md_read_flag_t *file_flag, hbool_t *context_flag); #endif /* H5_HAVE_PARALLEL */ /* External file cache routines */ diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 7120085..6d292bf 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -322,7 +322,7 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); * An integer field indicating the threshold for the page buffer size. * During a tick, the page buffer must expand as necessary to retain copies * of all modified metadata pages and multi-page metadata entries. - * If the page buffer size exceeds this thresold, an early end of tick + * If the page buffer size exceeds this threshold, an early end of tick * will be triggered. * Note that this is not a limit on the maximum page buffer size, as the * metadata cache is flushed as part of end of tick processing. @@ -1727,7 +1727,7 @@ H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], u * \brief Obtains information about a cache image if it exists * * \file_id - * \param[out] image_addr Offset of the cache image if it exists, or \c HADDR_UNDEF if it does not + * \param[out] image_addr Offset of the cache image if it exists, or #HADDR_UNDEF if it does not * \param[out] image_size Length of the cache image if it exists, or 0 if it does not * \returns \herr_t * @@ -1806,7 +1806,6 @@ H5_DLL herr_t H5Fget_dset_no_attrs_hint(hid_t file_id, hbool_t *minimize); * */ H5_DLL herr_t H5Fset_dset_no_attrs_hint(hid_t file_id, hbool_t minimize); -H5_DLL herr_t H5Fwait(hid_t file_id); /* VFD SWMR */ /** @@ -2022,6 +2021,7 @@ H5_DLL herr_t H5Fget_info1(hid_t obj_id, H5F_info1_t *file_info); * * \deprecated When? * + * \todo In which version was this function introduced? * \todo In which version was this function deprecated? * */ @@ -2040,6 +2040,7 @@ H5_DLL herr_t H5Fset_latest_format(hid_t file_id, hbool_t latest_format); * \details H5Fis_hdf5() determines whether a file is in the HDF5 format. * * \todo In which version was this function deprecated? + * \todo In which version was this function introduced? * */ H5_DLL htri_t H5Fis_hdf5(const char *file_name); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 24b25ff..2665505 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -950,6 +950,29 @@ done: } /* end H5F_get_eoa() */ /*------------------------------------------------------------------------- + * Function: H5F_shared_get_file_driver + * + * Purpose: Returns a pointer to the file driver structure of the + * file's 'shared' structure. + * + * Return: file handle on success/abort on failure (shouldn't fail) + *------------------------------------------------------------------------- + */ +herr_t +H5F_shared_get_file_driver(const H5F_shared_t *f_sh, H5FD_t **file_handle) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f_sh); + HDassert(file_handle); + + *file_handle = f_sh->lf; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F_shared_get_file_driver() */ + +/*------------------------------------------------------------------------- * Function: H5F_get_vfd_handle * * Purpose: Returns a pointer to the file handle of the low-level file diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index e7a3b53..bf02b06 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -375,7 +375,7 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read) if (0 == mpi_rank) { herr_t status; - /* Try detecting file's siganture */ + /* Try detecting file's signature */ /* (Don't leave before Bcast, to avoid hang on error) */ H5E_BEGIN_TRY { @@ -467,7 +467,7 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read) * * After upgrading low_bound, the library will check to ensure that the * superblock version does not exceed the version allowed by high_bound. - * Otherise fail file open. + * Otherwise fail file open. * * For details, please see RFC:Setting Bounds for Object Creation in HDF5 1.10.0. */ @@ -1043,7 +1043,7 @@ done: HDONE_ERROR(H5E_FILE, H5E_CANTUNPIN, FAIL, "unable to unpin driver info") /* Evict the driver info block from the cache */ - if (H5AC_expunge_entry(f, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0) + if (sblock && H5AC_expunge_entry(f, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTEXPUNGE, FAIL, "unable to expunge driver info block") } /* end if */ diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 7d1ff86..258570e 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -701,7 +701,7 @@ H5F__cache_superblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNU HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") H5F_addr_encode(f, &image, (rel_eof + sblock->base_addr)); - /* Encode the driver informaton block address */ + /* Encode the driver information block address */ H5F_addr_encode(f, &image, sblock->driver_addr); /* Encode the root group object entry, including the cached stab info */ @@ -800,7 +800,7 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_drvrinfo_get_initial_load_size * - * Purpose: Compute the intiial size of the data structure on disk. + * Purpose: Compute the initial size of the data structure on disk. * * Return: Non-negative on success/Negative on failure * diff --git a/src/H5Fvfd_swmr.c b/src/H5Fvfd_swmr.c index e5189b7..06d0c0d 100644 --- a/src/H5Fvfd_swmr.c +++ b/src/H5Fvfd_swmr.c @@ -828,7 +828,7 @@ clean_shadow_index(H5F_t *f, uint32_t nentries, H5FD_vfd_swmr_idx_entry_t *idx, * dirty the first page in the HDF5 file. However, the * metadata cache will no longer exist at this point. * - * Thus, we must check for the existance of the metadata + * Thus, we must check for the existence of the metadata * cache, and only attempt to flush it if it exists. * * 3) If this is the first tick (i.e. tick == 1), create the @@ -1021,7 +1021,7 @@ update_eot: HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to insert entry into the EOT queue") done: - /* Kent: Calcuate the processing time and write the time info to the log file */ + /* Kent: Calculate the processing time and write the time info to the log file */ if (shared->vfd_swmr_log_on == true) { if (H5_timer_get_times(shared->vfd_swmr_log_start_time, ¤t_time) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time from H5_timer_get_times") @@ -1060,7 +1060,7 @@ H5F_vfd_swmr_writer__dump_index(H5F_shared_t *shared) H5FD_vfd_swmr_idx_entry_t *index = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(shared); HDassert(shared->vfd_swmr); @@ -1080,8 +1080,6 @@ H5F_vfd_swmr_writer__dump_index(H5F_shared_t *shared) index[i].md_file_page_offset, index[i].length); } -done: - FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_vfd_swmr_writer__dump_index() */ @@ -1105,11 +1103,11 @@ done: * * If any such pages or multi-page metadata entries are found: * - * a) direct the page buffer to evict any such superceeded + * a) direct the page buffer to evict any such superseded * pages, and * * b) direct the metadata cache to either evict or refresh - * any entries residing in the superceeded pages. + * any entries residing in the superseded pages. * * Note that this operation MUST be performed in this order, * as the metadata cache will refer to the page buffer @@ -2091,7 +2089,7 @@ done: */ void -H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info) +H5F__post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, const char *log_info) { double temp_time; H5_timevals_t current_time; @@ -2276,7 +2274,7 @@ H5F__vfd_swmr_construct_ud_cl(H5F_vfd_swmr_updater_t *updater) * --determine num_change_list entries * --allocate buffers * --construct on disk image (serialize) of the updater header and change list - * --create updater file using a temporay file name: + * --create updater file using a temporary file name: * --vfd_swmr_config.updater_file_path>.ud_tmp * --allocate space and write the following to the updater file * --updater file header diff --git a/src/H5Gcache.c b/src/H5Gcache.c index df9943f..0ad3de0 100644 --- a/src/H5Gcache.c +++ b/src/H5Gcache.c @@ -135,7 +135,7 @@ H5G__cache_node_get_initial_load_size(void *_udata, size_t *image_len) * Function: H5G__cache_node_deserialize * * Purpose: Given a buffer containing the on disk image of a symbol table - * node, allocate an instance of H5G_node_t, load the contence of the + * node, allocate an instance of H5G_node_t, load the contents of the * image into it, and return a pointer to the instance. * * Note that deserializing the image requires access to the file @@ -241,7 +241,7 @@ H5G__cache_node_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5G__cache_node_serialize * - * Purpose: Given a correctly sized buffer and an instace of H5G_node_t, + * Purpose: Given a correctly sized buffer and an instance of H5G_node_t, * serialize the contents of the instance of H5G_node_t, and write * this data into the supplied buffer. This buffer will be written * to disk. diff --git a/src/H5Gcompact.c b/src/H5Gcompact.c index 0c0b5db..4988738 100644 --- a/src/H5Gcompact.c +++ b/src/H5Gcompact.c @@ -419,7 +419,7 @@ done: * Function: H5G__compact_lookup_cb * * Purpose: Callback routine for searching 'link' messages for a particular - * name & gettting object location for it + * name & getting object location for it * * Return: SUCCEED/FAIL * diff --git a/src/H5Gint.c b/src/H5Gint.c index 90a2c6b..ba0cb6d 100644 --- a/src/H5Gint.c +++ b/src/H5Gint.c @@ -91,9 +91,6 @@ static herr_t H5G__close_cb(H5VL_object_t *grp_vol_obj, void **request); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declare a free list to manage the H5G_t struct */ H5FL_DEFINE(H5G_t); H5FL_DEFINE(H5G_shared_t); @@ -117,9 +114,6 @@ static const H5I_class_t H5I_GROUP_CLS[1] = {{ (H5I_free_t)H5G__close_cb /* Callback routine for closing objects of this class */ }}; -/* Flag indicating "top" of interface has been initialized */ -static hbool_t H5G_top_package_initialize_s = FALSE; - /*------------------------------------------------------------------------- * Function: H5G_init * @@ -136,48 +130,13 @@ H5G_init(void) herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5G_init() */ - -/*------------------------------------------------------------------------- - * Function: H5G__init_package - * - * Purpose: Initializes the H5G interface. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Monday, January 5, 1998 - * - * Notes: The group creation properties are registered in the property - * list interface initialization routine (H5P_init_package) - * so that the file creation property class can inherit from it - * correctly. (Which allows the file creation property list to - * control the group creation properties of the root group of - * a file) QAK - 24/10/2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5G__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - /* Initialize the ID group for the group IDs */ if (H5I_register_type(H5I_GROUP_CLS) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to initialize interface") - /* Mark "top" of interface as initialized, too */ - H5G_top_package_initialize_s = TRUE; - done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5G__init_package() */ +} /* end H5G_init() */ /*------------------------------------------------------------------------- * Function: H5G_top_term_package @@ -200,16 +159,10 @@ H5G_top_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5G_top_package_initialize_s) { - if (H5I_nmembers(H5I_GROUP) > 0) { - (void)H5I_clear_type(H5I_GROUP, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - - /* Mark closed */ - if (0 == n) - H5G_top_package_initialize_s = FALSE; - } /* end if */ + if (H5I_nmembers(H5I_GROUP) > 0) { + (void)H5I_clear_type(H5I_GROUP, FALSE, FALSE); + n++; + } FUNC_LEAVE_NOAPI(n) } /* end H5G_top_term_package() */ @@ -238,18 +191,11 @@ H5G_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Sanity checks */ - HDassert(0 == H5I_nmembers(H5I_GROUP)); - HDassert(FALSE == H5G_top_package_initialize_s); - - /* Destroy the group object id group */ - n += (H5I_dec_type_ref(H5I_GROUP) > 0); + /* Sanity checks */ + HDassert(0 == H5I_nmembers(H5I_GROUP)); - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Destroy the group object id group */ + n += (H5I_dec_type_ref(H5I_GROUP) > 0); FUNC_LEAVE_NOAPI(n) } /* end H5G_term_package() */ diff --git a/src/H5Gnode.c b/src/H5Gnode.c index ba29b78..d73888e 100644 --- a/src/H5Gnode.c +++ b/src/H5Gnode.c @@ -1210,7 +1210,7 @@ H5G__node_copy(H5F_t *f, const void H5_ATTR_UNUSED *_lt_key, haddr_t addr, const &(sn->entry[i]); /* Convenience variable to refer to current source group entry */ H5O_link_t lnk; /* Link to insert */ const char * name; /* Name of source object */ - H5G_entry_t tmp_src_ent; /* Temperary copy. Change will not affect the cache */ + H5G_entry_t tmp_src_ent; /* Temporary copy. Change will not affect the cache */ H5O_type_t obj_type = H5O_TYPE_UNKNOWN; /* Target object type */ H5G_copy_file_ud_t *cpy_udata; /* Copy file udata */ H5G_obj_create_t gcrt_info; /* Group creation info */ diff --git a/src/H5Gpublic.h b/src/H5Gpublic.h index d9c29f6..ce36b84 100644 --- a/src/H5Gpublic.h +++ b/src/H5Gpublic.h @@ -1113,7 +1113,7 @@ H5_DLL herr_t H5Gget_objinfo(hid_t loc_id, const char *name, hbool_t follow_link * actual object name length, the object name is truncated to * \Code{max_size - 1} characters. * - * Note that if the size of the object's name is unkown, a preliminary + * Note that if the size of the object's name is unknown, a preliminary * call to H5Gget_objname_by_idx() with \p name set to \c NULL will * return the length of the object's name. A second call to * H5Gget_objname_by_idx() can then be used to retrieve the actual diff --git a/src/H5HF.c b/src/H5HF.c index 6c6b101..7904af9 100644 --- a/src/H5HF.c +++ b/src/H5HF.c @@ -63,9 +63,6 @@ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -358,7 +355,7 @@ H5HF_insert(H5HF_t *fh, size_t size, const void *obj, void *id /*out*/) if (size > hdr->max_man_size) { /* Store 'huge' object in heap * - * Although not ideal, we can quiet the const warning here becuase no + * Although not ideal, we can quiet the const warning here because no * obj pointer that was originally const should ever arrive here. */ H5_GCC_CLANG_DIAG_OFF("cast-qual") diff --git a/src/H5HFcache.c b/src/H5HFcache.c index 654fa36..3a48018 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -799,7 +799,7 @@ H5HF__cache_hdr_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_NDEBUG_UN /* Encode I/O filter information */ if (H5O_msg_encode(hdr->f, H5O_PLINE_ID, FALSE, image, &(hdr->pline)) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, FAIL, "can't encode I/O pipeline fiters") + HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, FAIL, "can't encode I/O pipeline filters") image += hdr->filter_len; } /* end if */ @@ -898,7 +898,7 @@ H5HF__cache_iblock_get_initial_load_size(void *_udata, size_t *image_len) * Function: H5HF__cache_iblock_verify_chksum * * Purpose: Verify the computed checksum of the data structure is the - * same as the stored chksum. + * same as the stored checksum. * * Return: Success: TRUE/FALSE * Failure: Negative diff --git a/src/H5HFdbg.c b/src/H5HFdbg.c index 8dbb3f4..d38ae20 100644 --- a/src/H5HFdbg.c +++ b/src/H5HFdbg.c @@ -50,7 +50,7 @@ /* User data for direct block debugging iterator callback */ typedef struct { FILE * stream; /* Stream for output */ - int indent; /* Indention amount */ + int indent; /* Indentation amount */ int fwidth; /* Field width mount */ haddr_t dblock_addr; /* Direct block's address */ hsize_t dblock_size; /* Direct block's size */ @@ -63,7 +63,7 @@ typedef struct { typedef struct { H5FS_t *fspace; /* Free space manager */ FILE * stream; /* Stream for output */ - int indent; /* Indention amount */ + int indent; /* Indentation amount */ int fwidth; /* Field width mount */ } H5HF_debug_iter_ud2_t; diff --git a/src/H5HFdblock.c b/src/H5HFdblock.c index 4adb47a..749b973 100644 --- a/src/H5HFdblock.c +++ b/src/H5HFdblock.c @@ -432,7 +432,7 @@ H5HF__man_dblock_protect(H5HF_hdr_t *hdr, haddr_t dblock_addr, size_t dblock_siz H5HF_indirect_t *par_iblock, unsigned par_entry, unsigned flags) { H5HF_direct_t * dblock; /* Direct block from cache */ - H5HF_dblock_cache_ud_t udata; /* parent and other infor for deserializing direct block */ + H5HF_dblock_cache_ud_t udata; /* parent and other info for deserializing direct block */ H5HF_direct_t * ret_value = NULL; /* Return value */ FUNC_ENTER_PACKAGE diff --git a/src/H5HFspace.c b/src/H5HFspace.c index d62267c..67f9b6f 100644 --- a/src/H5HFspace.c +++ b/src/H5HFspace.c @@ -383,7 +383,7 @@ H5HF__space_create_root(const H5HF_hdr_t *hdr, H5HF_indirect_t *root_iblock) /* Only need to scan the sections if the free space has been initialized */ if (hdr->fspace) - /* Iterate over all sections, seting the parent pointers in 'single' sections to the new indirect + /* Iterate over all sections, setting the parent pointers in 'single' sections to the new indirect * block */ if (H5FS_sect_iterate(hdr->f, hdr->fspace, H5HF__space_create_root_cb, root_iblock) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_BADITER, FAIL, "can't iterate over sections to set parent pointers") diff --git a/src/H5HG.c b/src/H5HG.c index f474748..7db3f98 100644 --- a/src/H5HG.c +++ b/src/H5HG.c @@ -84,9 +84,6 @@ static size_t H5HG__alloc(H5F_t *f, H5HG_heap_t *heap, size_t size, unsigned *h /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declare a free list to manage the H5HG_heap_t struct */ H5FL_DEFINE(H5HG_heap_t); diff --git a/src/H5HL.c b/src/H5HL.c index 1f2369a..eb25875 100644 --- a/src/H5HL.c +++ b/src/H5HL.c @@ -64,9 +64,6 @@ static herr_t H5HL__dirty(H5HL_t *heap); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declare a free list to manage the H5HL_free_t struct */ H5FL_DEFINE(H5HL_free_t); diff --git a/src/H5I.c b/src/H5I.c index 954a86b..0a4fa10 100644 --- a/src/H5I.c +++ b/src/H5I.c @@ -814,7 +814,7 @@ H5I__iterate_pub_cb(void H5_ATTR_UNUSED *obj, hid_t id, void *_udata) /*------------------------------------------------------------------------- * Function: H5Iiterate * - * Purpose: Call the callback funciton op for each member of the id + * Purpose: Call the callback function op for each member of the id * type type. op takes as parameters the id and a * passthrough of op_data, and returns an herr_t. A positive * return from op will cause the iteration to stop and diff --git a/src/H5Iint.c b/src/H5Iint.c index 86a2810..1ea2ad7 100644 --- a/src/H5Iint.c +++ b/src/H5Iint.c @@ -85,9 +85,6 @@ static int H5I__find_id_cb(void *_item, void *_key, void *_udata); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declared extern in H5Ipkg.h and documented there */ H5I_type_info_t *H5I_type_info_array_g[H5I_MAX_NUM_TYPES]; int H5I_next_type_g = (int)H5I_NTYPES; @@ -127,30 +124,24 @@ H5I_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - H5I_type_info_t *type_info = NULL; /* Pointer to ID type */ - int i; - - /* Count the number of types still in use */ - for (i = 0; i < H5I_next_type_g; i++) - if ((type_info = H5I_type_info_array_g[i]) && type_info->hash_table) + H5I_type_info_t *type_info = NULL; /* Pointer to ID type */ + int i; + + /* Count the number of types still in use */ + for (i = 0; i < H5I_next_type_g; i++) + if ((type_info = H5I_type_info_array_g[i]) && type_info->hash_table) + in_use++; + + /* If no types are still being used then clean up */ + if (0 == in_use) { + for (i = 0; i < H5I_next_type_g; i++) { + type_info = H5I_type_info_array_g[i]; + if (type_info) { + HDassert(NULL == type_info->hash_table); + type_info = H5MM_xfree(type_info); + H5I_type_info_array_g[i] = NULL; in_use++; - - /* If no types are still being used then clean up */ - if (0 == in_use) { - for (i = 0; i < H5I_next_type_g; i++) { - type_info = H5I_type_info_array_g[i]; - if (type_info) { - HDassert(NULL == type_info->hash_table); - type_info = H5MM_xfree(type_info); - H5I_type_info_array_g[i] = NULL; - in_use++; - } } - - /* Mark interface closed */ - if (0 == in_use) - H5_PKG_INIT_VAR = FALSE; } } diff --git a/src/H5Ipublic.h b/src/H5Ipublic.h index 8d4dbf8..d6892bb 100644 --- a/src/H5Ipublic.h +++ b/src/H5Ipublic.h @@ -508,7 +508,7 @@ H5_DLL int H5Idec_type_ref(H5I_type_t type); * * \brief Retrieves the reference count on an ID type * - * \param[in] type The identifier of the type whose reference count is to be retieved + * \param[in] type The identifier of the type whose reference count is to be retrieved * * \return Returns the current reference count on success, negative on failure. * diff --git a/src/H5Lint.c b/src/H5Lint.c index 346c37d..9e45124 100644 --- a/src/H5Lint.c +++ b/src/H5Lint.c @@ -175,9 +175,6 @@ static herr_t H5L__get_name_by_idx_cb(H5G_loc_t *grp_loc /*in*/, const char *nam /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -211,30 +208,6 @@ H5L_init(void) herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5L_init() */ - -/*------------------------------------------------------------------------- - * Function: H5L__init_package - * - * Purpose: Initialize information specific to H5L interface. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: James Laird - * Tuesday, January 24, 2006 - * - *------------------------------------------------------------------------- - */ -herr_t -H5L__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE /* Initialize user-defined link classes */ if (H5L_register_external() < 0) @@ -242,12 +215,12 @@ H5L__init_package(void) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5L_init_package() */ +} /* end H5L_init() */ /*------------------------------------------------------------------------- * Function: H5L_term_package * - * Purpose: Terminate any resources allocated in H5L__init_package. + * Purpose: Terminate any resources allocated in H5L_init. * * Return: Non-negative on success/Negative on failure * @@ -263,17 +236,11 @@ H5L_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Free the table of link types */ - if (H5L_table_g) { - H5L_table_g = (H5L_class_t *)H5MM_xfree(H5L_table_g); - H5L_table_used_g = H5L_table_alloc_g = 0; - n++; - } /* end if */ - - /* Mark the interface as uninitialized */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; + /* Free the table of link types */ + if (H5L_table_g) { + H5L_table_g = (H5L_class_t *)H5MM_xfree(H5L_table_g); + H5L_table_used_g = H5L_table_alloc_g = 0; + n++; } /* end if */ FUNC_LEAVE_NOAPI(n) diff --git a/src/H5Lpublic.h b/src/H5Lpublic.h index 72b0182..ca5f6e6 100644 --- a/src/H5Lpublic.h +++ b/src/H5Lpublic.h @@ -462,7 +462,7 @@ H5_DLL herr_t H5Ldelete_by_idx_async(const char *app_file, const char *app_func, * * \return \herr_t * - * \details H5Lget_val() returns tha value of link \p name. For smbolic links, + * \details H5Lget_val() returns the value of link \p name. For smbolic links, * this is the path to which the link points, including the null * terminator. For external and user-defined links, it is the link * buffer. @@ -492,7 +492,7 @@ H5_DLL herr_t H5Ldelete_by_idx_async(const char *app_file, const char *app_func, * * This function should be used only after H5Lget_info() has been * called to verify that \p name is a symbolic link. This can be - * deteremined from the \c link_type field of the \ref H5L_info_t + * determined from the \c link_type field of the \ref H5L_info_t * \c struct. * * \note This function will fail if called on a hard link. @@ -580,7 +580,7 @@ H5_DLL herr_t H5Lget_val_by_idx(hid_t loc_id, const char *group_name, H5_index_t * name includes either a relative path or an absolute path to the * target link, intermediate steps along the path must be verified * before the existence of the target link can be safely checked. If - * the path is not verified and an intermediate element of the path + * the path is not verified, and an intermediate element of the path * does not exist, H5Lexists() will fail. The example in the next * paragraph illustrates one step-by-step method for verifying the * existence of a link with a relative or absolute path. @@ -620,13 +620,13 @@ H5_DLL herr_t H5Lget_val_by_idx(hid_t loc_id, const char *group_name, H5_index_t * H5Lexists() with arguments \c file, \c "/", and \c lapl * returns a positive value; in other words, * \Code{H5Lexists(file, "/", lapl)} returns a positive value. - * In HDF5 version 1.8.16, this function returns 0. + * In the HDF5 1.8 release, this function returns 0. *

  • Let \c root denote a valid HDF5 group identifier that refers to the * root group of an HDF5 file, and let \c lapl denote a valid link * access property list identifier. A call to H5Lexists() with * arguments c root, \c "/", and \c lapl returns a positive value; - * in other words, \Code{H5Lexists(root, "/", lapl)} returns a postive - * value. In HDF5 version 1.8.16, this function returns 0.
  • + * in other words, \Code{H5Lexists(root, "/", lapl)} returns a positive + * value. In the HDF5 1.8 release, this function returns 0. * * Note that the function accepts link names and path names. This is * potentially misleading to callers, and we plan to separate the @@ -1094,7 +1094,7 @@ H5_DLL herr_t H5Lvisit2(hid_t grp_id, H5_index_t idx_type, H5_iter_order_t order * \p idx_type specifies the index to be used. If the links have not * been indexed by the index type, they will first be sorted by that * index then the iteration will begin; if the links have been so - * indexed, the sorting step will be unnecesary, so the iteration may + * indexed, the sorting step will be unnecessary, so the iteration may * begin more quickly. Valid values include the following: * \indexes * @@ -1853,7 +1853,7 @@ H5_DLL herr_t H5Lvisit1(hid_t grp_id, H5_index_t idx_type, H5_iter_order_t order * \p idx_type specifies the index to be used. If the links have not * been indexed by the index type, they will first be sorted by that * index then the iteration will begin; if the links have been so - * indexed, the sorting step will be unnecesary, so the iteration may + * indexed, the sorting step will be unnecessary, so the iteration may * begin more quickly. Valid values include the following: * \indexes * diff --git a/src/H5M.c b/src/H5M.c index 845c054..59ce615 100644 --- a/src/H5M.c +++ b/src/H5M.c @@ -58,9 +58,6 @@ static herr_t H5M__get_api_common(hid_t map_id, hid_t key_mem_type_id, const voi /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -77,9 +74,6 @@ static const H5I_class_t H5I_MAP_CLS[1] = {{ (H5I_free_t)H5M__close_cb /* Callback routine for closing objects of this class */ }}; -/* Flag indicating "top" of interface has been initialized */ -static hbool_t H5M_top_package_initialize_s = FALSE; - /*------------------------------------------------------------------------- * Function: H5M_init * @@ -96,41 +90,14 @@ H5M_init(void) herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5M_init() */ - -/*------------------------------------------------------------------------- -NAME - H5M__init_package -- Initialize interface-specific information -USAGE - herr_t H5M__init_package() - -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. ---------------------------------------------------------------------------- -*/ -herr_t -H5M__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE /* Initialize the ID group for the map IDs */ if (H5I_register_type(H5I_MAP_CLS) < 0) HGOTO_ERROR(H5E_MAP, H5E_CANTINIT, FAIL, "unable to initialize interface") - /* Mark "top" of interface as initialized, too */ - H5M_top_package_initialize_s = TRUE; - done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5M__init_package() */ +} /* end H5M_init() */ /*------------------------------------------------------------------------- * Function: H5M_top_term_package @@ -149,16 +116,10 @@ H5M_top_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5M_top_package_initialize_s) { - if (H5I_nmembers(H5I_MAP) > 0) { - (void)H5I_clear_type(H5I_MAP, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - - /* Mark closed */ - if (0 == n) - H5M_top_package_initialize_s = FALSE; - } /* end if */ + if (H5I_nmembers(H5I_MAP) > 0) { + (void)H5I_clear_type(H5I_MAP, FALSE, FALSE); + n++; + } FUNC_LEAVE_NOAPI(n) } /* end H5M_top_term_package() */ @@ -183,18 +144,11 @@ H5M_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Sanity checks */ - HDassert(0 == H5I_nmembers(H5I_MAP)); - HDassert(FALSE == H5M_top_package_initialize_s); + /* Sanity checks */ + HDassert(0 == H5I_nmembers(H5I_MAP)); - /* Destroy the dataset object id group */ - n += (H5I_dec_type_ref(H5I_MAP) > 0); - - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Destroy the dataset object id group */ + n += (H5I_dec_type_ref(H5I_MAP) > 0); FUNC_LEAVE_NOAPI(n) } /* end H5M_term_package() */ diff --git a/src/H5MF.c b/src/H5MF.c index ca780c3..be69b46 100644 --- a/src/H5MF.c +++ b/src/H5MF.c @@ -119,9 +119,6 @@ static herr_t H5MF__defer_free(H5F_shared_t *shared, H5FD_mem_t alloc_type, hadd /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -713,8 +710,10 @@ H5MF__add_sect(H5F_t *f, H5FD_mem_t alloc_type, H5FS_t *fspace, H5MF_free_sectio H5AC_set_ring(fsm_ring, &orig_ring); #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: adding node, node->sect_info.addr = %a, node->sect_info.size = %Hu\n", __func__, - node->sect_info.addr, node->sect_info.size); + HDfprintf(stderr, + "%s: adding node, node->sect_info.addr = %" PRIuHADDR ", node->sect_info.size = %" PRIuHSIZE + "\n", + __func__, node->sect_info.addr, node->sect_info.size); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Add the section */ if (H5FS_sect_add(f, fspace, (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) @@ -766,7 +765,7 @@ H5MF__find_sect(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5FS_t *fspace, h HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "error locating free space in file") #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: section found = %t\n", __func__, ret_value); + HDfprintf(stderr, "%s: section found = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Check for actually finding section */ @@ -794,7 +793,7 @@ H5MF__find_sect(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5FS_t *fspace, h node->sect_info.size -= size; #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: re-adding node, node->sect_info.size = %Hu\n", __func__, + HDfprintf(stderr, "%s: re-adding node, node->sect_info.size = %" PRIuHSIZE "\n", __func__, node->sect_info.size); #endif /* H5MF_ALLOC_DEBUG_MORE */ @@ -838,7 +837,7 @@ H5MF_alloc(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size) FUNC_ENTER_NOAPI_TAG(H5AC__FREESPACE_TAG, HADDR_UNDEF) #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", __func__, (unsigned)alloc_type, size); + HDfprintf(stderr, "%s: alloc_type = %u, size = %" PRIuHSIZE "\n", __func__, (unsigned)alloc_type, size); #endif /* H5MF_ALLOC_DEBUG */ /* check arguments */ @@ -916,7 +915,8 @@ done: H5AC_set_ring(orig_ring, NULL); #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", __func__, ret_value, size); + HDfprintf(stderr, "%s: Leaving: ret_value = %" PRIuHADDR ", size = %" PRIuHSIZE "\n", __func__, ret_value, + size); #endif /* H5MF_ALLOC_DEBUG */ #ifdef H5MF_ALLOC_DEBUG_DUMP H5MF__sects_dump(f, stderr); @@ -956,7 +956,7 @@ H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size) FUNC_ENTER_STATIC_TAG(H5AC__FREESPACE_TAG) #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", __func__, (unsigned)alloc_type, size); + HDfprintf(stderr, "%s: alloc_type = %u, size = %" PRIuHSIZE "\n", __func__, (unsigned)alloc_type, size); #endif /* H5MF_ALLOC_DEBUG */ H5MF__alloc_to_fs_type(f->shared, alloc_type, size, &ptype); @@ -1053,7 +1053,8 @@ H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size) done: #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", __func__, ret_value, size); + HDfprintf(stderr, "%s: Leaving: ret_value = %" PRIuHADDR ", size = %" PRIuHSIZE "\n", __func__, ret_value, + size); #endif /* H5MF_ALLOC_DEBUG */ #ifdef H5MF_ALLOC_DEBUG_DUMP H5MF__sects_dump(f, stderr); @@ -1099,7 +1100,7 @@ H5MF_alloc_tmp(H5F_t *f, hsize_t size) FUNC_ENTER_NOAPI(HADDR_UNDEF) #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: size = %Hu\n", __func__, size); + HDfprintf(stderr, "%s: size = %" PRIuHSIZE "\n", __func__, size); #endif /* H5MF_ALLOC_DEBUG */ /* check args */ @@ -1146,8 +1147,8 @@ H5MF_xfree(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size) FUNC_ENTER_NOAPI_TAG(H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", __func__, - (unsigned)alloc_type, addr, size); + HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %" PRIuHADDR ", size = %" PRIuHSIZE "\n", + __func__, (unsigned)alloc_type, addr, size); #endif /* H5MF_ALLOC_DEBUG */ /* check arguments */ @@ -1224,7 +1225,7 @@ H5MF__xfree_inner_impl(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t si * space is at the end of the file */ #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: fs_addr = %a\n", __func__, f->shared->fs_addr[fs_type]); + HDfprintf(stderr, "%s: fs_addr = %" PRIuHADDR "\n", __func__, f->shared->fs_addr[fs_type]); #endif /* H5MF_ALLOC_DEBUG_MORE */ if (!H5F_addr_defined(f->shared->fs_addr[fs_type])) { htri_t status; /* "can absorb" status for section into */ @@ -1240,8 +1241,9 @@ H5MF__xfree_inner_impl(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t si HGOTO_DONE(SUCCEED) else if (size < f->shared->fs_threshold) { #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", __func__, addr, - size); + HDfprintf(stderr, + "%s: dropping addr = %" PRIuHADDR ", size = %" PRIuHSIZE ", on the floor!\n", + __func__, addr, size); #endif /* H5MF_ALLOC_DEBUG_MORE */ HGOTO_DONE(SUCCEED) } /* end else-if */ @@ -1258,7 +1260,8 @@ H5MF__xfree_inner_impl(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t si */ if (f->shared->fs_state[fs_type] == H5F_FS_STATE_DELETING || !H5F_HAVE_FREE_SPACE_MANAGER(f)) { #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", __func__, addr, size); + HDfprintf(stderr, "%s: dropping addr = %" PRIuHADDR ", size = %" PRIuHSIZE ", on the floor!\n", + __func__, addr, size); #endif /* H5MF_ALLOC_DEBUG_MORE */ HGOTO_DONE(SUCCEED) } /* end if */ @@ -1352,7 +1355,7 @@ H5MF__xfree_impl(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size) * Further, in the VFD SWMR case, the large metadata entry will typically * be buffered in the page buffer. If an intersecting entry is left * in the page buffer, in addition to causing potential corruption in - * the HDF5 file, it may also result in overlaping entries in the page + * the HDF5 file, it may also result in overlapping entries in the page * buffer and metadata file index. * * It's ok to remove the page from the PB without flushing to @@ -1416,7 +1419,9 @@ H5MF_try_extend(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsi FUNC_ENTER_NOAPI_TAG(H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_requested = %Hu\n", + HDfprintf(stderr, + "%s: Entering: alloc_type = %u, addr = %" PRIuHADDR ", size = %" PRIuHSIZE + ", extra_requested = %" PRIuHSIZE "\n", __func__, (unsigned)alloc_type, addr, size, extra_requested); #endif /* H5MF_ALLOC_DEBUG */ @@ -1469,7 +1474,7 @@ H5MF_try_extend(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsi if ((ret_value = H5F__try_extend(f, map_type, end, extra_requested + frag_size)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: extended = %t\n", __func__, ret_value); + HDfprintf(stderr, "%s: extended = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* If extending at EOA succeeds: */ @@ -1507,7 +1512,7 @@ H5MF_try_extend(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsi HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: H5MF__aggr_try_extend = %t\n", __func__, ret_value); + HDfprintf(stderr, "%s: H5MF__aggr_try_extend = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ @@ -1533,7 +1538,7 @@ H5MF_try_extend(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsi HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending block in free space manager") #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: Try to H5FS_sect_try_extend = %t\n", __func__, ret_value); + HDfprintf(stderr, "%s: Try to H5FS_sect_try_extend = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ @@ -1544,7 +1549,7 @@ H5MF_try_extend(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsi if (frag_size <= H5F_PGEND_META_THRES(f) && extra_requested <= frag_size) ret_value = TRUE; #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: Try to extend into the page end threshold = %t\n", __func__, + HDfprintf(stderr, "%s: Try to extend into the page end threshold = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ @@ -1557,7 +1562,7 @@ done: H5AC_set_ring(orig_ring, NULL); #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", __func__, ret_value); + HDfprintf(stderr, "%s: Leaving: ret_value = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG */ #ifdef H5MF_ALLOC_DEBUG_DUMP H5MF__sects_dump(f, stderr); @@ -1592,8 +1597,8 @@ H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size) FUNC_ENTER_NOAPI_TAG(H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG - HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", __func__, - (unsigned)alloc_type, addr, size); + HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %" PRIuHADDR ", size = %" PRIuHSIZE "\n", + __func__, (unsigned)alloc_type, addr, size); #endif /* H5MF_ALLOC_DEBUG */ /* check arguments */ @@ -1729,8 +1734,9 @@ H5MF__close_delete_fstype(H5F_t *f, H5F_mem_page_t type) HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", __func__, - (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); + HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %" PRIuHADDR "\n", + __func__, (unsigned)type, (void *)f->shared->fs_man[type], (unsigned)type, + f->shared->fs_addr[type]); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* If the free space manager for this type is open, close it */ @@ -1739,8 +1745,9 @@ H5MF__close_delete_fstype(H5F_t *f, H5F_mem_page_t type) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", __func__, - (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); + HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %" PRIuHADDR "\n", + __func__, (unsigned)type, (void *)f->shared->fs_man[type], (unsigned)type, + f->shared->fs_addr[type]); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* If there is free space manager info for this type, delete it */ diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c index 4684ee2..e7b966c 100644 --- a/src/H5MFaggr.c +++ b/src/H5MFaggr.c @@ -92,7 +92,7 @@ H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size) FUNC_ENTER_NOAPI(HADDR_UNDEF) #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", __func__, (unsigned)alloc_type, size); + HDfprintf(stderr, "%s: alloc_type = %u, size = %" PRIuHSIZE "\n", __func__, (unsigned)alloc_type, size); #endif /* H5MF_AGGR_DEBUG */ /* check arguments */ @@ -120,7 +120,8 @@ H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size) done: #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", __func__, ret_value, size); + HDfprintf(stderr, "%s: Leaving: ret_value = %" PRIuHADDR ", size = %" PRIuHSIZE "\n", __func__, ret_value, + size); #endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) @@ -150,7 +151,7 @@ H5MF__aggr_alloc(H5F_t *f, H5F_blk_aggr_t *aggr, H5F_blk_aggr_t *other_aggr, H5F FUNC_ENTER_STATIC #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: type = %u, size = %Hu\n", __func__, (unsigned)type, size); + HDfprintf(stderr, "%s: type = %u, size = %" PRIuHSIZE "\n", __func__, (unsigned)type, size); #endif /* H5MF_AGGR_DEBUG */ /* check args */ @@ -195,12 +196,12 @@ H5MF__aggr_alloc(H5F_t *f, H5F_blk_aggr_t *aggr, H5F_blk_aggr_t *other_aggr, H5F haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */ hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */ hsize_t alignment; /* Alignment of this section */ - hsize_t aggr_mis_align = 0; /* Mis-alignment of aggregator */ + hsize_t aggr_mis_align = 0; /* Misalignment of aggregator */ H5FD_mem_t alloc_type, other_alloc_type; /* Current aggregator & 'other' aggregator types */ #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", __func__, aggr->addr, aggr->tot_size, - aggr->size); + HDfprintf(stderr, "%s: aggr = {%" PRIuHADDR ", %" PRIuHSIZE ", %" PRIuHSIZE "}\n", __func__, + aggr->addr, aggr->tot_size, aggr->size); #endif /* H5MF_AGGR_DEBUG */ /* Turn off alignment if allocation < threshold */ @@ -388,7 +389,7 @@ H5MF__aggr_alloc(H5F_t *f, H5F_blk_aggr_t *aggr, H5F_blk_aggr_t *other_aggr, H5F done: #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: ret_value = %a\n", __func__, ret_value); + HDfprintf(stderr, "%s: ret_value = %" PRIuHADDR "\n", __func__, ret_value); #endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5MF__aggr_alloc() */ @@ -537,8 +538,11 @@ done: if (H5F_addr_eq((sect->sect_info.addr + sect->sect_info.size), aggr->addr) || H5F_addr_eq((aggr->addr + aggr->size), sect->sect_info.addr)) { #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: section {%a, %Hu} adjoins aggr = {%a, %Hu}\n", "H5MF__aggr_can_absorb", - sect->sect_info.addr, sect->sect_info.size, aggr->addr, aggr->size); + HDfprintf(stderr, + "%s: section {%" PRIuHADDR ", %" PRIuHSIZE "} adjoins aggr = {%" PRIuHADDR + ", %" PRIuHSIZE "}\n", + "H5MF__aggr_can_absorb", sect->sect_info.addr, sect->sect_info.size, aggr->addr, + aggr->size); #endif /* H5MF_AGGR_DEBUG */ /* Check if aggregator would get too large and should be absorbed into section */ if ((aggr->size + sect->sect_info.size) >= aggr->alloc_size) @@ -587,7 +591,9 @@ done: /* Check if the section adjoins the beginning or end of the aggregator */ if (H5F_addr_eq((sect->sect_info.addr + sect->sect_info.size), aggr->addr)) { #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: aggr {%a, %Hu} adjoins front of section = {%a, %Hu}\n", + HDfprintf(stderr, + "%s: aggr {%" PRIuHADDR ", %" PRIuHSIZE "} adjoins front of section = {%" PRIuHADDR + ", %" PRIuHSIZE "}\n", "H5MF__aggr_absorb", aggr->addr, aggr->size, sect->sect_info.addr, sect->sect_info.size); #endif /* H5MF_AGGR_DEBUG */ @@ -599,7 +605,9 @@ done: HDassert(H5F_addr_eq((aggr->addr + aggr->size), sect->sect_info.addr)); #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: aggr {%a, %Hu} adjoins end of section = {%a, %Hu}\n", + HDfprintf(stderr, + "%s: aggr {%" PRIuHADDR ", %" PRIuHSIZE "} adjoins end of section = {%" PRIuHADDR + ", %" PRIuHSIZE "}\n", "H5MF__aggr_absorb", aggr->addr, aggr->size, sect->sect_info.addr, sect->sect_info.size); #endif /* H5MF_AGGR_DEBUG */ @@ -617,7 +625,9 @@ done: /* Check if the section adjoins the beginning or end of the aggregator */ if (H5F_addr_eq((sect->sect_info.addr + sect->sect_info.size), aggr->addr)) { #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: section {%a, %Hu} adjoins front of aggr = {%a, %Hu}\n", + HDfprintf(stderr, + "%s: section {%" PRIuHADDR ", %" PRIuHSIZE "} adjoins front of aggr = {%" PRIuHADDR + ", %" PRIuHSIZE "}\n", "H5MF__aggr_absorb", sect->sect_info.addr, sect->sect_info.size, aggr->addr, aggr->size); #endif /* H5MF_AGGR_DEBUG */ @@ -635,7 +645,9 @@ done: HDassert(H5F_addr_eq((aggr->addr + aggr->size), sect->sect_info.addr)); #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: section {%a, %Hu} adjoins end of aggr = {%a, %Hu}\n", + HDfprintf(stderr, + "%s: section {%" PRIuHADDR ", %" PRIuHSIZE "} adjoins end of aggr = {%" PRIuHADDR + ", %" PRIuHSIZE "}\n", "H5MF__aggr_absorb", sect->sect_info.addr, sect->sect_info.size, aggr->addr, aggr->size); #endif /* H5MF_AGGR_DEBUG */ @@ -723,7 +735,8 @@ done: tmp_addr = aggr->addr; tmp_size = aggr->size; #ifdef H5MF_AGGR_DEBUG - HDfprintf(stderr, "%s: tmp_addr = %a, tmp_size = %Hu\n", __func__, tmp_addr, tmp_size); + HDfprintf(stderr, "%s: tmp_addr = %" PRIuHADDR ", tmp_size = %" PRIuHSIZE "\n", __func__, + tmp_addr, tmp_size); #endif /* H5MF_AGGR_DEBUG */ /* Reset aggregator block information */ diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c index 6ccbe78..60d7779 100644 --- a/src/H5MFdbg.c +++ b/src/H5MFdbg.c @@ -50,7 +50,7 @@ typedef struct { H5FS_t *fspace; /* Free space manager */ FILE * stream; /* Stream for output */ - int indent; /* Indention amount */ + int indent; /* Indentation amount */ int fwidth; /* Field width amount */ } H5MF_debug_iter_ud_t; diff --git a/src/H5MFsection.c b/src/H5MFsection.c index e686842d..2f5103f 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -472,8 +472,9 @@ H5MF__sect_simple_can_shrink(const H5FS_section_info_t *_sect, void *_udata) /* Set the shrinking type */ udata->shrink = H5MF_SHRINK_EOA; #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", __func__, sect->sect_info.addr, - sect->sect_info.size, eoa); + HDfprintf(stderr, + "%s: section {%" PRIuHADDR ", %" PRIuHSIZE "}, shrinks file, eoa = %" PRIuHADDR "\n", + __func__, sect->sect_info.addr, sect->sect_info.size, eoa); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Indicate shrinking can occur */ @@ -496,8 +497,9 @@ H5MF__sect_simple_can_shrink(const H5FS_section_info_t *_sect, void *_udata) /* Set the aggregator to operate on */ udata->aggr = &(udata->f->shared->meta_aggr); #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: section {%a, %Hu}, adjoins metadata aggregator\n", __func__, - sect->sect_info.addr, sect->sect_info.size); + HDfprintf(stderr, + "%s: section {%" PRIuHADDR ", %" PRIuHSIZE "}, adjoins metadata aggregator\n", + __func__, sect->sect_info.addr, sect->sect_info.size); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Indicate shrinking can occur */ @@ -517,8 +519,9 @@ H5MF__sect_simple_can_shrink(const H5FS_section_info_t *_sect, void *_udata) /* Set the aggregator to operate on */ udata->aggr = &(udata->f->shared->sdata_aggr); #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: section {%a, %Hu}, adjoins small data aggregator\n", __func__, - sect->sect_info.addr, sect->sect_info.size); + HDfprintf(stderr, + "%s: section {%" PRIuHADDR ", %" PRIuHSIZE "}, adjoins small data aggregator\n", + __func__, sect->sect_info.addr, sect->sect_info.size); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Indicate shrinking can occur */ @@ -625,8 +628,8 @@ H5MF__sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) FUNC_ENTER_STATIC #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", __func__, (*sect)->sect_info.addr, - (*sect)->sect_info.size); + HDfprintf(stderr, "%s: Entering, section {%" PRIuHADDR ", %" PRIuHSIZE "}\n", __func__, + (*sect)->sect_info.addr, (*sect)->sect_info.size); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Do not adjust the section raw data or global heap data */ @@ -653,8 +656,8 @@ H5MF__sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) else if (prem <= H5F_PGEND_META_THRES(udata->f)) { (*sect)->sect_info.size += prem; #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: section is adjusted {%a, %Hu}\n", __func__, (*sect)->sect_info.addr, - (*sect)->sect_info.size); + HDfprintf(stderr, "%s: section is adjusted {%" PRIuHADDR ", %" PRIuHSIZE "}\n", __func__, + (*sect)->sect_info.addr, (*sect)->sect_info.size); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ @@ -702,7 +705,7 @@ H5MF__sect_small_can_merge(const H5FS_section_info_t *_sect1, const H5FS_section ret_value = FALSE; #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", __func__, ret_value); + HDfprintf(stderr, "%s: Leaving: ret_value = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG_MORE */ FUNC_LEAVE_NOAPI(ret_value) @@ -798,30 +801,20 @@ H5MF__sect_large_can_merge(const H5FS_section_info_t *_sect1, const H5FS_section const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */ htri_t ret_value = FALSE; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Check arguments. */ HDassert(sect1); HDassert(sect2); HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */ - if (!H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)) { - HDfprintf(stderr, - "%s.%d: sect1->sect_info.addr %" PRIuHADDR ", sect2->sect_info.addr %" PRIuHADDR "\n", - __func__, __LINE__, sect1->sect_info.addr, sect2->sect_info.addr); - HDfprintf(stderr, - "%s.%d: sect1->sect_info.size %" PRIuHSIZE ", sect2->sect_info.size %" PRIuHSIZE "\n", - __func__, __LINE__, sect1->sect_info.size, sect2->sect_info.size); - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't merge") - } HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)); ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr); #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", __func__, ret_value); + HDfprintf(stderr, "%s: Leaving: ret_value = %d\n", __func__, ret_value); #endif /* H5MF_ALLOC_DEBUG_MORE */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* H5MF__sect_large_can_merge() */ @@ -907,8 +900,9 @@ H5MF__sect_large_can_shrink(const H5FS_section_info_t *_sect, void *_udata) /* Set the shrinking type */ udata->shrink = H5MF_SHRINK_EOA; #ifdef H5MF_ALLOC_DEBUG_MORE - HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", __func__, sect->sect_info.addr, - sect->sect_info.size, eoa); + HDfprintf(stderr, + "%s: section {%" PRIuHADDR ", %" PRIuHSIZE "}, shrinks file, eoa = %" PRIuHADDR "\n", + __func__, sect->sect_info.addr, sect->sect_info.size, eoa); #endif /* H5MF_ALLOC_DEBUG_MORE */ /* Indicate shrinking can occur */ diff --git a/src/H5MP.c b/src/H5MP.c index 474a995..397d26b 100644 --- a/src/H5MP.c +++ b/src/H5MP.c @@ -58,9 +58,6 @@ /* Package Variable Definitions */ /********************************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /********************/ /* Static Variables */ /********************/ diff --git a/src/H5MV.c b/src/H5MV.c index 3d2bd80..28ae37d 100644 --- a/src/H5MV.c +++ b/src/H5MV.c @@ -77,9 +77,6 @@ H5_DLL htri_t H5MV__try_extend_md(H5F_shared_t *, haddr_t, hsize_t); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ diff --git a/src/H5MVprivate.h b/src/H5MVprivate.h index c02c6ae..b2a5458 100644 --- a/src/H5MVprivate.h +++ b/src/H5MVprivate.h @@ -14,7 +14,7 @@ * * Created: H5MVprivate.h * - * Purpose: Private header file for VFD SWMR free space mangement + * Purpose: Private header file for VFD SWMR free space management * *------------------------------------------------------------------------- */ diff --git a/src/H5Oainfo.c b/src/H5Oainfo.c index 10c9ede..03ea3df 100644 --- a/src/H5Oainfo.c +++ b/src/H5Oainfo.c @@ -52,7 +52,7 @@ const H5O_msg_class_t H5O_MSG_AINFO[1] = {{ H5O_AINFO_ID, /*message id number */ "ainfo", /*message name for debugging */ sizeof(H5O_ainfo_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__ainfo_decode, /*decode message */ H5O__ainfo_encode, /*encode message */ H5O__ainfo_copy, /*copy the native value */ diff --git a/src/H5Oattr.c b/src/H5Oattr.c index db53eaf..83c0370 100644 --- a/src/H5Oattr.c +++ b/src/H5Oattr.c @@ -65,7 +65,7 @@ const H5O_msg_class_t H5O_MSG_ATTR[1] = {{ H5O_ATTR_ID, /* message id number */ "attribute", /* message name for debugging */ sizeof(H5A_t), /* native message size */ - H5O_SHARE_IS_SHARABLE, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE, /* messages are shareable? */ H5O__attr_shared_decode, /* decode message */ H5O__attr_shared_encode, /* encode message */ H5O__attr_copy, /* copy the native value */ diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c index 56f7145..c4f8b27 100644 --- a/src/H5Oattribute.c +++ b/src/H5Oattribute.c @@ -248,19 +248,19 @@ H5O__attr_create(const H5O_loc_t *loc, H5A_t *attr) /* Check if switching to "dense" attribute storage is possible */ if (!H5F_addr_defined(ainfo.fheap_addr)) { - htri_t sharable; /* Whether the attribute will be shared */ + htri_t shareable; /* Whether the attribute will be shared */ size_t raw_size = 0; /* Raw size of message */ - /* Check for attribute being sharable */ - if ((sharable = H5SM_can_share(loc->file, NULL, NULL, H5O_ATTR_ID, attr)) < 0) + /* Check for attribute being shareable */ + if ((shareable = H5SM_can_share(loc->file, NULL, NULL, H5O_ATTR_ID, attr)) < 0) HGOTO_ERROR(H5E_ATTR, H5E_BADMESG, FAIL, "can't determine attribute sharing status") - else if (sharable == FALSE) { + else if (shareable == FALSE) { /* Compute the size needed to encode the attribute */ raw_size = (H5O_MSG_ATTR->raw_size)(loc->file, FALSE, attr); } /* end if */ /* Check for condititions for switching to "dense" attribute storage are met */ - if (ainfo.nattrs == oh->max_compact || (!sharable && raw_size >= H5O_MESG_MAX_SIZE)) { + if (ainfo.nattrs == oh->max_compact || (!shareable && raw_size >= H5O_MESG_MAX_SIZE)) { H5O_iter_cvt_t udata; /* User data for callback */ H5O_mesg_operator_t op; /* Wrapper for operator */ diff --git a/src/H5Obogus.c b/src/H5Obogus.c index c7fcdb9..ea231d6 100644 --- a/src/H5Obogus.c +++ b/src/H5Obogus.c @@ -47,7 +47,7 @@ const H5O_msg_class_t H5O_MSG_BOGUS_VALID[1] = {{ H5O_BOGUS_VALID_ID, /*message id number */ "bogus valid", /*message name for debugging */ 0, /*native message size */ - H5O_SHARE_IS_SHARABLE, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE, /* messages are shareable? */ H5O__bogus_decode, /*decode message */ H5O__bogus_encode, /*encode message */ NULL, /*copy the native value */ @@ -71,7 +71,7 @@ const H5O_msg_class_t H5O_MSG_BOGUS_INVALID[1] = {{ H5O_BOGUS_INVALID_ID, /*message id number */ "bogus invalid", /*message name for debugging */ 0, /*native message size */ - H5O_SHARE_IS_SHARABLE, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE, /* messages are shareable? */ H5O__bogus_decode, /*decode message */ H5O__bogus_encode, /*encode message */ NULL, /*copy the native value */ diff --git a/src/H5Obtreek.c b/src/H5Obtreek.c index a783366..1eaf3e5 100644 --- a/src/H5Obtreek.c +++ b/src/H5Obtreek.c @@ -37,7 +37,7 @@ const H5O_msg_class_t H5O_MSG_BTREEK[1] = {{ H5O_BTREEK_ID, /*message id number */ "v1 B-tree 'K' values", /*message name for debugging */ sizeof(H5O_btreek_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__btreek_decode, /*decode message */ H5O__btreek_encode, /*encode message */ H5O__btreek_copy, /*copy the native value */ diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 8eed092..84e68a1 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -277,7 +277,7 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) * * Note that the object header is read with with a speculative read. * If the initial read is too small, make note of this fact and return - * without error. H5C__load_entry() will note the size discrepency + * without error. H5C__load_entry() will note the size discrepancy * and retry the deserialize operation with the correct size read. * * Return: Success: Pointer to in core representation diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c index 2603f71..8f60a0e 100644 --- a/src/H5Ocache_image.c +++ b/src/H5Ocache_image.c @@ -51,7 +51,7 @@ const H5O_msg_class_t H5O_MSG_MDCI[1] = {{ H5O_MDCI_MSG_ID, /* message id number */ "mdci", /* message name for debugging */ sizeof(H5O_mdci_t), /* native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__mdci_decode, /* decode message */ H5O__mdci_encode, /* encode message */ H5O__mdci_copy, /* copy method */ diff --git a/src/H5Ocont.c b/src/H5Ocont.c index 080225a..9dd04fe 100644 --- a/src/H5Ocont.c +++ b/src/H5Ocont.c @@ -47,7 +47,7 @@ const H5O_msg_class_t H5O_MSG_CONT[1] = {{ H5O_CONT_ID, /*message id number */ "hdr continuation", /*message name for debugging */ sizeof(H5O_cont_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__cont_decode, /*decode message */ H5O__cont_encode, /*encode message */ NULL, /*no copy method */ diff --git a/src/H5Ocopy.c b/src/H5Ocopy.c index 6716769..8367b93 100644 --- a/src/H5Ocopy.c +++ b/src/H5Ocopy.c @@ -962,7 +962,7 @@ H5O__copy_header(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, hid_t if (H5P_get(ocpy_plist, H5O_CPY_OPTION_NAME, &cpy_option) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get object copy flag") - /* Retrieve the marge committed datatype list */ + /* Retrieve the merge committed datatype list */ if (H5P_peek(ocpy_plist, H5O_CPY_MERGE_COMM_DT_LIST_NAME, &dt_list) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get merge committed datatype list") diff --git a/src/H5Ocopy_ref.c b/src/H5Ocopy_ref.c index e06e699..1cda3ea 100644 --- a/src/H5Ocopy_ref.c +++ b/src/H5Ocopy_ref.c @@ -170,31 +170,32 @@ H5O__copy_expand_ref_object1(H5O_loc_t *src_oloc, const void *buf_src, H5O_loc_t H5O_token_t tmp_token = {0}; /* If data is not initialized, copy zeros and skip */ - if (0 == HDmemcmp(src_buf, zeros, buf_size)) { + if (0 == HDmemcmp(src_buf, zeros, buf_size)) HDmemset(dst_buf, 0, buf_size); - continue; - } - - /* Set up for the object copy for the reference */ - if (H5R__decode_token_obj_compat(src_buf, &buf_size, &tmp_token, token_size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "unable to decode src object address") - if (H5VL_native_token_to_addr(src_oloc->file, H5I_FILE, tmp_token, &src_oloc->addr) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNSERIALIZE, FAIL, "can't deserialize object token into address") - - if (!H5F_addr_defined(src_oloc->addr) || src_oloc->addr == 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "undefined reference pointer") - dst_oloc->addr = HADDR_UNDEF; - - /* Attempt to copy object from source to destination file */ - if (H5O__copy_obj_by_ref(src_oloc, dst_oloc, dst_root_loc, cpy_info) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object") - - /* Set the object reference info for the destination file */ - if (H5VL_native_addr_to_token(dst_oloc->file, H5I_FILE, dst_oloc->addr, &tmp_token) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "can't serialize address into object token") - if (H5R__encode_token_obj_compat((const H5O_token_t *)&tmp_token, token_size, dst_buf, &buf_size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "unable to encode dst object address") - } /* end for */ + else { + /* Set up for the object copy for the reference */ + if (H5R__decode_token_obj_compat(src_buf, &buf_size, &tmp_token, token_size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "unable to decode src object address") + if (H5VL_native_token_to_addr(src_oloc->file, H5I_FILE, tmp_token, &src_oloc->addr) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNSERIALIZE, FAIL, + "can't deserialize object token into address") + + if (!H5F_addr_defined(src_oloc->addr) || src_oloc->addr == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "undefined reference pointer") + dst_oloc->addr = HADDR_UNDEF; + + /* Attempt to copy object from source to destination file */ + if (H5O__copy_obj_by_ref(src_oloc, dst_oloc, dst_root_loc, cpy_info) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object") + + /* Set the object reference info for the destination file */ + if (H5VL_native_addr_to_token(dst_oloc->file, H5I_FILE, dst_oloc->addr, &tmp_token) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "can't serialize address into object token") + if (H5R__encode_token_obj_compat((const H5O_token_t *)&tmp_token, token_size, dst_buf, + &buf_size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "unable to encode dst object address") + } /* end else */ + } /* end for */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -232,43 +233,42 @@ H5O__copy_expand_ref_region1(H5O_loc_t *src_oloc, const void *buf_src, H5O_loc_t uint8_t * q; /* If data is not initialized, copy zeros and skip */ - if (0 == HDmemcmp(src_buf, zeros, buf_size)) { + if (0 == HDmemcmp(src_buf, zeros, buf_size)) HDmemset(dst_buf, 0, buf_size); - continue; - } - - /* Read from heap */ - if (H5R__decode_heap(src_oloc->file, src_buf, &buf_size, &data, &data_size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "unable to decode dataset region information") - - /* Get object address */ - p = (const uint8_t *)data; - H5F_addr_decode(src_oloc->file, &p, &src_oloc->addr); - if (!H5F_addr_defined(src_oloc->addr) || src_oloc->addr == 0) { - H5MM_free(data); - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "undefined reference pointer") - } - dst_oloc->addr = HADDR_UNDEF; - - /* Attempt to copy object from source to destination file */ - if (H5O__copy_obj_by_ref(src_oloc, dst_oloc, dst_root_loc, cpy_info) < 0) { - H5MM_free(data); - HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object") - } /* end if */ - - /* Serialize object addr */ - q = (uint8_t *)data; - H5F_addr_encode(dst_oloc->file, &q, dst_oloc->addr); - - /* Write to heap */ - if (H5R__encode_heap(dst_oloc->file, dst_buf, &buf_size, data, (size_t)data_size) < 0) { + else { + /* Read from heap */ + if (H5R__decode_heap(src_oloc->file, src_buf, &buf_size, &data, &data_size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "unable to decode dataset region information") + + /* Get object address */ + p = (const uint8_t *)data; + H5F_addr_decode(src_oloc->file, &p, &src_oloc->addr); + if (!H5F_addr_defined(src_oloc->addr) || src_oloc->addr == 0) { + H5MM_free(data); + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "undefined reference pointer") + } + dst_oloc->addr = HADDR_UNDEF; + + /* Attempt to copy object from source to destination file */ + if (H5O__copy_obj_by_ref(src_oloc, dst_oloc, dst_root_loc, cpy_info) < 0) { + H5MM_free(data); + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object") + } /* end if */ + + /* Serialize object addr */ + q = (uint8_t *)data; + H5F_addr_encode(dst_oloc->file, &q, dst_oloc->addr); + + /* Write to heap */ + if (H5R__encode_heap(dst_oloc->file, dst_buf, &buf_size, data, (size_t)data_size) < 0) { + H5MM_free(data); + HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to encode dataset region information") + } + + /* Free the buffer allocated in H5R__decode_heap() */ H5MM_free(data); - HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to encode dataset region information") - } - - /* Free the buffer allocated in H5R__decode_heap() */ - H5MM_free(data); - } /* end for */ + } /* end else */ + } /* end for */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -288,21 +288,22 @@ H5O__copy_expand_ref_object2(H5O_loc_t *src_oloc, hid_t tid_src, const H5T_t *dt size_t nbytes_src, H5O_loc_t *dst_oloc, H5G_loc_t *dst_root_loc, void *buf_dst, size_t ref_count, H5O_copy_t *cpy_info) { - H5T_t * dt_mem = NULL; /* Memory datatype */ - H5T_t * dt_dst = NULL; /* Destination datatype */ - hid_t tid_mem = H5I_INVALID_HID; /* Datatype ID for memory datatype */ - hid_t tid_dst = H5I_INVALID_HID; /* Datatype ID for memory datatype */ - H5T_path_t *tpath_src_mem = NULL, *tpath_mem_dst = NULL; /* Datatype conversion paths */ - size_t i; /* Local index variable */ - hbool_t reg_tid_src = (tid_src == H5I_INVALID_HID); - hid_t dst_loc_id = H5I_INVALID_HID; - void * conv_buf = NULL; /* Buffer for converting data */ - size_t conv_buf_size = 0; /* Buffer size */ - void * reclaim_buf = NULL; /* Buffer for reclaiming data */ - H5S_t * buf_space = NULL; /* Dataspace describing buffer */ - hsize_t buf_dim[1] = {ref_count}; /* Dimension for buffer */ - size_t token_size = H5F_SIZEOF_ADDR(src_oloc->file); - herr_t ret_value = SUCCEED; + H5T_t * dt_mem = NULL; /* Memory datatype */ + H5T_t * dt_dst = NULL; /* Destination datatype */ + hid_t tid_mem = H5I_INVALID_HID; /* Datatype ID for memory datatype */ + hid_t tid_dst = H5I_INVALID_HID; /* Datatype ID for memory datatype */ + H5T_path_t * tpath_src_mem = NULL, *tpath_mem_dst = NULL; /* Datatype conversion paths */ + size_t i; /* Local index variable */ + hbool_t reg_tid_src = (tid_src == H5I_INVALID_HID); + hid_t dst_loc_id = H5I_INVALID_HID; + void * conv_buf = NULL; /* Buffer for converting data */ + size_t conv_buf_size = 0; /* Buffer size */ + void * reclaim_buf = NULL; /* Buffer for reclaiming data */ + H5S_t * buf_space = NULL; /* Dataspace describing buffer */ + hsize_t buf_dim[1] = {ref_count}; /* Dimension for buffer */ + size_t token_size = H5F_SIZEOF_ADDR(src_oloc->file); + const unsigned char zeros[H5R_REF_BUF_SIZE] = {0}; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -353,29 +354,34 @@ H5O__copy_expand_ref_object2(H5O_loc_t *src_oloc, hid_t tid_src, const H5T_t *dt /* Making equivalent references in the destination file */ for (i = 0; i < ref_count; i++) { - H5R_ref_t * ref_ptr = (H5R_ref_t *)conv_buf; - H5R_ref_priv_t *ref = (H5R_ref_priv_t *)&ref_ptr[i]; - H5O_token_t tmp_token = {0}; - - /* Get src object address */ - if (H5R__get_obj_token(ref, &tmp_token, &token_size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to get object token") - if (H5VL_native_token_to_addr(src_oloc->file, H5I_FILE, tmp_token, &src_oloc->addr) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNSERIALIZE, FAIL, "can't deserialize object token into address") - - /* Attempt to copy object from source to destination file */ - if (H5O__copy_obj_by_ref(src_oloc, dst_oloc, dst_root_loc, cpy_info) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object") - - /* Set dst object address */ - if (H5VL_native_addr_to_token(dst_oloc->file, H5I_FILE, dst_oloc->addr, &tmp_token) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "can't serialize address into object token") - if (H5R__set_obj_token(ref, (const H5O_token_t *)&tmp_token, token_size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "unable to set object token") - /* Do not set app_ref since references are released once the copy is done */ - if (H5R__set_loc_id(ref, dst_loc_id, TRUE, FALSE) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "unable to set destination loc id") - } /* end for */ + H5R_ref_t * ref_ptr = (H5R_ref_t *)conv_buf; + H5R_ref_priv_t *ref = (H5R_ref_priv_t *)&ref_ptr[i]; + + /* Check for null reference - only expand reference if it is not null */ + if (HDmemcmp(ref, zeros, H5R_REF_BUF_SIZE)) { + H5O_token_t tmp_token = {0}; + + /* Get src object address */ + if (H5R__get_obj_token(ref, &tmp_token, &token_size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to get object token") + if (H5VL_native_token_to_addr(src_oloc->file, H5I_FILE, tmp_token, &src_oloc->addr) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNSERIALIZE, FAIL, + "can't deserialize object token into address") + + /* Attempt to copy object from source to destination file */ + if (H5O__copy_obj_by_ref(src_oloc, dst_oloc, dst_root_loc, cpy_info) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object") + + /* Set dst object address */ + if (H5VL_native_addr_to_token(dst_oloc->file, H5I_FILE, dst_oloc->addr, &tmp_token) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "can't serialize address into object token") + if (H5R__set_obj_token(ref, (const H5O_token_t *)&tmp_token, token_size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "unable to set object token") + /* Do not set app_ref since references are released once the copy is done */ + if (H5R__set_loc_id(ref, dst_loc_id, TRUE, FALSE) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "unable to set destination loc id") + } /* end if */ + } /* end for */ /* Copy into another buffer, to reclaim memory later */ if (NULL == (reclaim_buf = H5FL_BLK_MALLOC(type_conv, conv_buf_size))) diff --git a/src/H5Odrvinfo.c b/src/H5Odrvinfo.c index 8bebc6d..69ab5b2 100644 --- a/src/H5Odrvinfo.c +++ b/src/H5Odrvinfo.c @@ -38,7 +38,7 @@ const H5O_msg_class_t H5O_MSG_DRVINFO[1] = {{ H5O_DRVINFO_ID, /*message id number */ "driver info", /*message name for debugging */ sizeof(H5O_drvinfo_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__drvinfo_decode, /*decode message */ H5O__drvinfo_encode, /*encode message */ H5O__drvinfo_copy, /*copy the native value */ diff --git a/src/H5Odtype.c b/src/H5Odtype.c index 1be9522..9af79f4 100644 --- a/src/H5Odtype.c +++ b/src/H5Odtype.c @@ -89,7 +89,7 @@ const H5O_msg_class_t H5O_MSG_DTYPE[1] = {{ H5O_DTYPE_ID, /* message id number */ "datatype", /* message name for debugging */ sizeof(H5T_t), /* native message size */ - H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are shareable? */ H5O__dtype_shared_decode, /* decode message */ H5O__dtype_shared_encode, /* encode message */ H5O__dtype_copy, /* copy the native value */ @@ -1731,7 +1731,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_NO_CLASS: case H5T_NCLASSES: default: - HDsprintf(buf, "H5T_CLASS_%d", (int)(dt->shared->type)); + HDsnprintf(buf, sizeof(buf), "H5T_CLASS_%d", (int)(dt->shared->type)); s = buf; break; } /* end switch */ @@ -1746,7 +1746,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, "Number of members:", dt->shared->u.compnd.nmembs); for (i = 0; i < dt->shared->u.compnd.nmembs; i++) { - HDsprintf(buf, "Member %u:", i); + HDsnprintf(buf, sizeof(buf), "Member %u:", i); HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth, buf, dt->shared->u.compnd.memb[i].name); HDfprintf(stream, "%*s%-*s %lu\n", indent + 3, "", MAX(0, fwidth - 3), "Byte offset:", (unsigned long)(dt->shared->u.compnd.memb[i].offset)); @@ -1759,7 +1759,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, "Number of members:", dt->shared->u.enumer.nmembs); for (i = 0; i < dt->shared->u.enumer.nmembs; i++) { - HDsprintf(buf, "Member %u:", i); + HDsnprintf(buf, sizeof(buf), "Member %u:", i); HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth, buf, dt->shared->u.enumer.name[i]); HDfprintf(stream, "%*s%-*s 0x", indent, "", fwidth, "Raw bytes of value:"); for (k = 0; k < dt->shared->parent->shared->size; k++) @@ -1799,13 +1799,14 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_CSET_RESERVED_13: case H5T_CSET_RESERVED_14: case H5T_CSET_RESERVED_15: - HDsprintf(buf, "H5T_CSET_RESERVED_%d", (int)(dt->shared->u.atomic.u.s.cset)); + HDsnprintf(buf, sizeof(buf), "H5T_CSET_RESERVED_%d", (int)(dt->shared->u.atomic.u.s.cset)); s = buf; break; case H5T_CSET_ERROR: default: - HDsprintf(buf, "Unknown character set: %d", (int)(dt->shared->u.atomic.u.s.cset)); + HDsnprintf(buf, sizeof(buf), "Unknown character set: %d", + (int)(dt->shared->u.atomic.u.s.cset)); s = buf; break; } /* end switch */ @@ -1837,13 +1838,14 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_STR_RESERVED_13: case H5T_STR_RESERVED_14: case H5T_STR_RESERVED_15: - HDsprintf(buf, "H5T_STR_RESERVED_%d", (int)(dt->shared->u.atomic.u.s.pad)); + HDsnprintf(buf, sizeof(buf), "H5T_STR_RESERVED_%d", (int)(dt->shared->u.atomic.u.s.pad)); s = buf; break; case H5T_STR_ERROR: default: - HDsprintf(buf, "Unknown string padding: %d", (int)(dt->shared->u.atomic.u.s.pad)); + HDsnprintf(buf, sizeof(buf), "Unknown string padding: %d", + (int)(dt->shared->u.atomic.u.s.pad)); s = buf; break; } /* end switch */ @@ -1862,7 +1864,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_VLEN_BADTYPE: case H5T_VLEN_MAXTYPE: default: - HDsprintf(buf, "H5T_VLEN_%d", dt->shared->u.vlen.type); + HDsnprintf(buf, sizeof(buf), "H5T_VLEN_%d", dt->shared->u.vlen.type); s = buf; break; } /* end switch */ @@ -1880,7 +1882,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_LOC_BADLOC: case H5T_LOC_MAXLOC: default: - HDsprintf(buf, "H5T_LOC_%d", (int)dt->shared->u.vlen.loc); + HDsnprintf(buf, sizeof(buf), "H5T_LOC_%d", (int)dt->shared->u.vlen.loc); s = buf; break; } /* end switch */ @@ -1911,13 +1913,13 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_CSET_RESERVED_13: case H5T_CSET_RESERVED_14: case H5T_CSET_RESERVED_15: - HDsprintf(buf, "H5T_CSET_RESERVED_%d", (int)(dt->shared->u.vlen.cset)); + HDsnprintf(buf, sizeof(buf), "H5T_CSET_RESERVED_%d", (int)(dt->shared->u.vlen.cset)); s = buf; break; case H5T_CSET_ERROR: default: - HDsprintf(buf, "Unknown character set: %d", (int)(dt->shared->u.vlen.cset)); + HDsnprintf(buf, sizeof(buf), "Unknown character set: %d", (int)(dt->shared->u.vlen.cset)); s = buf; break; } /* end switch */ @@ -1949,13 +1951,13 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_STR_RESERVED_13: case H5T_STR_RESERVED_14: case H5T_STR_RESERVED_15: - HDsprintf(buf, "H5T_STR_RESERVED_%d", (int)(dt->shared->u.vlen.pad)); + HDsnprintf(buf, sizeof(buf), "H5T_STR_RESERVED_%d", (int)(dt->shared->u.vlen.pad)); s = buf; break; case H5T_STR_ERROR: default: - HDsprintf(buf, "Unknown string padding: %d", (int)(dt->shared->u.vlen.pad)); + HDsnprintf(buf, sizeof(buf), "Unknown string padding: %d", (int)(dt->shared->u.vlen.pad)); s = buf; break; } /* end switch */ @@ -1995,7 +1997,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_ORDER_ERROR: default: - HDsprintf(buf, "H5T_ORDER_%d", dt->shared->u.atomic.order); + HDsnprintf(buf, sizeof(buf), "H5T_ORDER_%d", dt->shared->u.atomic.order); s = buf; break; } /* end switch */ @@ -2069,9 +2071,9 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_NPAD: default: if (dt->shared->u.atomic.u.f.pad < 0) - HDsprintf(buf, "H5T_PAD_%d", -(dt->shared->u.atomic.u.f.pad)); + HDsnprintf(buf, sizeof(buf), "H5T_PAD_%d", -(dt->shared->u.atomic.u.f.pad)); else - HDsprintf(buf, "bit-%d", dt->shared->u.atomic.u.f.pad); + HDsnprintf(buf, sizeof(buf), "bit-%d", dt->shared->u.atomic.u.f.pad); s = buf; break; } /* end switch */ @@ -2092,7 +2094,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_NORM_ERROR: default: - HDsprintf(buf, "H5T_NORM_%d", (int)(dt->shared->u.atomic.u.f.norm)); + HDsnprintf(buf, sizeof(buf), "H5T_NORM_%d", (int)(dt->shared->u.atomic.u.f.norm)); s = buf; } /* end switch */ HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth, "Normalization:", s); @@ -2129,7 +2131,7 @@ H5O__dtype_debug(H5F_t *f, const void *mesg, FILE *stream, int indent, int fwidt case H5T_SGN_ERROR: case H5T_NSGN: default: - HDsprintf(buf, "H5T_SGN_%d", (int)(dt->shared->u.atomic.u.i.sign)); + HDsnprintf(buf, sizeof(buf), "H5T_SGN_%d", (int)(dt->shared->u.atomic.u.i.sign)); s = buf; break; } /* end switch */ diff --git a/src/H5Oefl.c b/src/H5Oefl.c index d950249..931fe0b 100644 --- a/src/H5Oefl.c +++ b/src/H5Oefl.c @@ -41,7 +41,7 @@ const H5O_msg_class_t H5O_MSG_EFL[1] = {{ H5O_EFL_ID, /*message id number */ "external file list", /*message name for debugging */ sizeof(H5O_efl_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__efl_decode, /*decode message */ H5O__efl_encode, /*encode message */ H5O__efl_copy, /*copy native value */ diff --git a/src/H5Ofill.c b/src/H5Ofill.c index 5068039..d905352 100644 --- a/src/H5Ofill.c +++ b/src/H5Ofill.c @@ -107,7 +107,7 @@ const H5O_msg_class_t H5O_MSG_FILL[1] = {{ H5O_FILL_ID, /*message id number */ "fill", /*message name for debugging */ sizeof(H5O_fill_t), /*native message size */ - H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are shareable? */ H5O__fill_shared_decode, /*decode message */ H5O__fill_shared_encode, /*encode message */ H5O__fill_copy, /*copy the native value */ @@ -131,7 +131,7 @@ const H5O_msg_class_t H5O_MSG_FILL_NEW[1] = {{ H5O_FILL_NEW_ID, /*message id number */ "fill_new", /*message name for debugging */ sizeof(H5O_fill_t), /*native message size */ - H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are shareable? */ H5O__fill_new_shared_decode, /*decode message */ H5O__fill_new_shared_encode, /*encode message */ H5O__fill_copy, /*copy the native value */ diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c index b60f589..7253f3e 100644 --- a/src/H5Ofsinfo.c +++ b/src/H5Ofsinfo.c @@ -44,7 +44,7 @@ const H5O_msg_class_t H5O_MSG_FSINFO[1] = {{ H5O_FSINFO_ID, /* message id number */ "fsinfo", /* message name for debugging */ sizeof(H5O_fsinfo_t), /* native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__fsinfo_decode, /* decode message */ H5O__fsinfo_encode, /* encode message */ H5O__fsinfo_copy, /* copy the native value */ @@ -176,7 +176,7 @@ H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section threshold */ H5F_DECODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ - UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metdata threshold */ + UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metadata threshold */ H5F_addr_decode(f, &p, &(fsinfo->eoa_pre_fsm_fsalloc)); /* EOA before free-space header and section info */ diff --git a/src/H5Oginfo.c b/src/H5Oginfo.c index 8540cc4..304890c 100644 --- a/src/H5Oginfo.c +++ b/src/H5Oginfo.c @@ -43,7 +43,7 @@ const H5O_msg_class_t H5O_MSG_GINFO[1] = {{ H5O_GINFO_ID, /*message id number */ "ginfo", /*message name for debugging */ sizeof(H5O_ginfo_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__ginfo_decode, /*decode message */ H5O__ginfo_encode, /*encode message */ H5O__ginfo_copy, /*copy the native value */ diff --git a/src/H5Oint.c b/src/H5Oint.c index f24d0bb..ee79b0c 100644 --- a/src/H5Oint.c +++ b/src/H5Oint.c @@ -82,9 +82,6 @@ static herr_t H5O__reset_info2(H5O_info2_t *oinfo); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Header message ID to class mapping * * Remember to increment H5O_MSG_TYPES in H5Opkg.h when adding a new @@ -178,21 +175,20 @@ static const H5O_obj_class_t *const H5O_obj_class_g[] = { }; /*------------------------------------------------------------------------- - * Function: H5O__init_package - * - * Purpose: Initialize information specific to H5O interface. - * - * Return: Non-negative on success/Negative on failure + * Function: H5O_init * - * Programmer: Quincey Koziol - * Thursday, January 18, 2007 + * Purpose: Initialize the interface from some other layer. * + * Return: Success: non-negative + * Failure: negative *------------------------------------------------------------------------- */ herr_t -H5O__init_package(void) +H5O_init(void) { - FUNC_ENTER_PACKAGE_NOERR + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOERR /* H5O interface sanity checks */ HDcompile_assert(H5O_MSG_TYPES == NELMTS(H5O_msg_class_g)); @@ -200,8 +196,8 @@ H5O__init_package(void) HDcompile_assert(H5O_UNKNOWN_ID < H5O_MSG_TYPES); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5O__init_package() */ + FUNC_LEAVE_NOAPI(ret_value) +} /*------------------------------------------------------------------------- * Function: H5O__set_version @@ -567,7 +563,7 @@ H5O_open(H5O_loc_t *loc) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Check args */ HDassert(loc); @@ -584,7 +580,6 @@ H5O_open(H5O_loc_t *loc) else H5F_INCR_NOPEN_OBJS(loc->file); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_open() */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 651e317..c939e72 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -51,7 +51,7 @@ const H5O_msg_class_t H5O_MSG_LAYOUT[1] = {{ H5O_LAYOUT_ID, /* message id number */ "layout", /* message name for debugging */ sizeof(H5O_layout_t), /* native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__layout_decode, /* decode message */ H5O__layout_encode, /* encode message */ H5O__layout_copy, /* copy the native value */ diff --git a/src/H5Olinfo.c b/src/H5Olinfo.c index 711f199..eacc916 100644 --- a/src/H5Olinfo.c +++ b/src/H5Olinfo.c @@ -53,7 +53,7 @@ const H5O_msg_class_t H5O_MSG_LINFO[1] = {{ H5O_LINFO_ID, /*message id number */ "linfo", /*message name for debugging */ sizeof(H5O_linfo_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__linfo_decode, /*decode message */ H5O__linfo_encode, /*encode message */ H5O__linfo_copy, /*copy the native value */ diff --git a/src/H5Olink.c b/src/H5Olink.c index 75456f4..51c44a3 100644 --- a/src/H5Olink.c +++ b/src/H5Olink.c @@ -56,7 +56,7 @@ const H5O_msg_class_t H5O_MSG_LINK[1] = {{ H5O_LINK_ID, /*message id number */ "link", /*message name for debugging */ sizeof(H5O_link_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__link_decode, /*decode message */ H5O__link_encode, /*encode message */ H5O__link_copy, /*copy the native value */ diff --git a/src/H5Omessage.c b/src/H5Omessage.c index 15edf1e..fa20aa1 100644 --- a/src/H5Omessage.c +++ b/src/H5Omessage.c @@ -371,7 +371,7 @@ H5O__msg_write_real(H5F_t *f, H5O_t *oh, const H5O_msg_class_t *type, unsigned m HDassert(((H5O_shared_t *)idx_msg->native)->type != H5O_SHARE_TYPE_COMMITTED); /* Also, sanity check that a message doesn't switch status from being - * shared (or sharable) to being unsharable. (Which could cause + * shared (or shareable) to being unshareable. (Which could cause * a message to increase in size in the object header) */ HDassert(!(mesg_flags & H5O_MSG_FLAG_DONTSHARE)); @@ -1131,7 +1131,7 @@ done: * Nov 19 2004 * * Description: - * This function interates over the object headers of an object + * This function iterates over the object headers of an object * specified with 'loc' of type 'type_id'. For each object header of the * object, the 'op_data' and some additional information (specified below) are * passed to the 'op' function. @@ -1194,7 +1194,7 @@ done: * Sep 6 2005 * * Description: - * This function interates over the object headers of an object + * This function iterates over the object headers of an object * specified with 'ent' of type 'type_id'. For each object header of the * object, the 'op_data' and some additional information (specified below) are * passed to the 'op' function. @@ -1544,7 +1544,7 @@ H5O_msg_is_shared(unsigned type_id, const void *mesg) HDassert(type); HDassert(mesg); - /* If messages in a class aren't sharable, then obviously this message isn't shared! :-) */ + /* If messages in a class aren't shareable, then obviously this message isn't shared! :-) */ if (type->share_flags & H5O_SHARE_IS_SHARABLE) ret_value = H5O_IS_STORED_SHARED(((const H5O_shared_t *)mesg)->type); else diff --git a/src/H5Omtime.c b/src/H5Omtime.c index 163ea32..7d3c56a 100644 --- a/src/H5Omtime.c +++ b/src/H5Omtime.c @@ -43,7 +43,7 @@ const H5O_msg_class_t H5O_MSG_MTIME[1] = {{ H5O_MTIME_ID, /*message id number */ "mtime", /*message name for debugging */ sizeof(time_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__mtime_decode, /*decode message */ H5O__mtime_encode, /*encode message */ H5O__mtime_copy, /*copy the native value */ @@ -68,7 +68,7 @@ const H5O_msg_class_t H5O_MSG_MTIME_NEW[1] = {{ H5O_MTIME_NEW_ID, /*message id number */ "mtime_new", /*message name for debugging */ sizeof(time_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__mtime_new_decode, /*decode message */ H5O__mtime_new_encode, /*encode message */ H5O__mtime_copy, /*copy the native value */ diff --git a/src/H5Oname.c b/src/H5Oname.c index 1636a0e..35578a9 100644 --- a/src/H5Oname.c +++ b/src/H5Oname.c @@ -43,7 +43,7 @@ const H5O_msg_class_t H5O_MSG_NAME[1] = {{ H5O_NAME_ID, /*message id number */ "name", /*message name for debugging */ sizeof(H5O_name_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__name_decode, /*decode message */ H5O__name_encode, /*encode message */ H5O__name_copy, /*copy the native value */ diff --git a/src/H5Onull.c b/src/H5Onull.c index 9b377fb..0f3143c 100644 --- a/src/H5Onull.c +++ b/src/H5Onull.c @@ -31,7 +31,7 @@ const H5O_msg_class_t H5O_MSG_NULL[1] = {{ H5O_NULL_ID, /*message id number */ "null", /*message name for debugging */ 0, /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ NULL, /*no decode method */ NULL, /*no encode method */ NULL, /*no copy method */ diff --git a/src/H5Opkg.h b/src/H5Opkg.h index 331fcf6..ebfe636 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -378,7 +378,7 @@ typedef struct H5O_chunk_proxy_t { H5O_t * oh; /* Object header for this chunk */ unsigned chunkno; /* Chunk number for this chunk */ - /* Flush depencency parent information (not stored) + /* Flush dependency parent information (not stored) * * The following field is used to store a pointer * to the in-core representation of a new chunk proxy's flush dependency diff --git a/src/H5Opline.c b/src/H5Opline.c index 221e323..243f454 100644 --- a/src/H5Opline.c +++ b/src/H5Opline.c @@ -67,7 +67,7 @@ const H5O_msg_class_t H5O_MSG_PLINE[1] = {{ H5O_PLINE_ID, /* message id number */ "filter pipeline", /* message name for debugging */ sizeof(H5O_pline_t), /* native message size */ - H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are shareable? */ H5O__pline_shared_decode, /* decode message */ H5O__pline_shared_encode, /* encode message */ H5O__pline_copy, /* copy the native value */ @@ -86,7 +86,7 @@ const H5O_msg_class_t H5O_MSG_PLINE[1] = {{ H5O__pline_shared_debug /* debug the message */ }}; -/* Format version bounds for filter pipleline */ +/* Format version bounds for filter pipeline */ const unsigned H5O_pline_ver_bounds[] = { H5O_PLINE_VERSION_1, /* H5F_LIBVER_EARLIEST */ H5O_PLINE_VERSION_2, /* H5F_LIBVER_V18 */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 72306df..d5d1972 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -251,7 +251,7 @@ typedef struct H5O_copy_t { #define H5O_SHARE_TYPE_UNSHARED 0 /* Message is not shared */ #define H5O_SHARE_TYPE_SOHM 1 /* Message is stored in SOHM heap */ #define H5O_SHARE_TYPE_COMMITTED 2 /* Message is stored in another object header */ -#define H5O_SHARE_TYPE_HERE 3 /* Message is stored in this object header, but is sharable */ +#define H5O_SHARE_TYPE_HERE 3 /* Message is stored in this object header, but is shareable */ /* Detect messages that aren't stored in message's object header */ #define H5O_IS_STORED_SHARED(T) \ diff --git a/src/H5Opublic.h b/src/H5Opublic.h index d9d0500..70f451e 100644 --- a/src/H5Opublic.h +++ b/src/H5Opublic.h @@ -80,7 +80,7 @@ #define H5O_SHMESG_MAX_LIST_SIZE 5000 /* Flags for H5Oget_info. - * Theses flags determine which fields will be filled in in the H5O_info_t + * These flags determine which fields will be filled in the H5O_info_t * struct. */ #define H5O_INFO_BASIC 0x0001u /**< Fill in the fileno, addr, type, and rc fields */ @@ -90,8 +90,8 @@ //! /** - * Flags for H5Oget_native_info(). Theses flags determine which fields will be - * filled in in the \ref H5O_native_info_t struct. + * Flags for H5Oget_native_info(). These flags determine which fields will be + * filled in the \ref H5O_native_info_t struct. */ #define H5O_NATIVE_INFO_HDR 0x0008u /**< Fill in the hdr field */ #define H5O_NATIVE_INFO_META_SIZE 0x0010u /**< Fill in the meta_size field */ @@ -510,7 +510,7 @@ H5_DLL herr_t H5Oget_info3(hid_t loc_id, H5O_info2_t *oinfo, unsigned fields); * location and relative name * * \fgdta_loc_obj_id{loc_id} - * \param[in] name Name of group, relative to \p loc_id + * \param[in] name Name of object, relative to \p loc_id * \param[out] oinfo Buffer in which to return object information * \param[in] fields Flags specifying the fields to include in \p oinfo * \lapl_id @@ -1166,7 +1166,7 @@ H5_DLL ssize_t H5Oget_comment_by_name(hid_t loc_id, const char *name, char *comm * best effort setting. If the application passes in * a value indicating iteration in creation order and a group is * encountered that was not tracked in creation order, that group - * will be iterated over in alpha-numeric order by name, or + * will be iterated over in alphanumeric order by name, or * name order. (Name order is the native order * used by the HDF5 library and is always available.) * @@ -1265,7 +1265,7 @@ H5_DLL herr_t H5Ovisit3(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order * best effort setting. If the application passes in a * value indicating iteration in creation order and a group is * encountered that was not tracked in creation order, that group - * will be iterated over in alpha-numeric order by name, or + * will be iterated over in alphanumeric order by name, or * name order. (Name order is the native order * used by the HDF5 library and is always available.) * @@ -1834,7 +1834,7 @@ H5_DLL herr_t H5Oget_info1(hid_t loc_id, H5O_info1_t *oinfo); * by location and relative name * * \fgdta_loc_obj_id{loc_id} - * \param[in] name Name of group, relative to \p loc_id + * \param[in] name Name of object, relative to \p loc_id * \param[out] oinfo Buffer in which to return object information * \lapl_id * @@ -1960,7 +1960,7 @@ H5_DLL herr_t H5Oget_info2(hid_t loc_id, H5O_info1_t *oinfo, unsigned fields); * by location and relative name * * \fgdta_loc_obj_id{loc_id} - * \param[in] name Name of group, relative to \p loc_id + * \param[in] name Name of object, relative to \p loc_id * \param[out] oinfo Buffer in which to return object information * \param[in] fields Flags specifying the fields to include in \p oinfo * \lapl_id @@ -1978,7 +1978,7 @@ H5_DLL herr_t H5Oget_info2(hid_t loc_id, H5O_info1_t *oinfo, unsigned fields); * in the H5Oget_info1() function entry. * * The \p fields parameter contains flags to determine which fields - * will be filled in in the H5O_info1_t \c struct returned in + * will be filled in the H5O_info1_t \c struct returned in * \p oinfo. These flags are defined in the H5Opublic.h file: * * \obj_info_fields @@ -2096,7 +2096,7 @@ H5_DLL herr_t H5Oget_info_by_idx2(hid_t loc_id, const char *group_name, H5_index * best effort setting. If the application passes in * a value indicating iteration in creation order and a group is * encountered that was not tracked in creation order, that group - * will be iterated over in alpha-numeric order by name, or + * will be iterated over in alphanumeric order by name, or * name order. (Name order is the native order * used by the HDF5 library and is always available.) * @@ -2188,7 +2188,7 @@ H5_DLL herr_t H5Ovisit1(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order * best effort setting. If the application passes in a * value indicating iteration in creation order and a group is * encountered that was not tracked in creation order, that group - * will be iterated over in alpha-numeric order by name, or + * will be iterated over in alphanumeric order by name, or * name order. (Name order is the native order * used by the HDF5 library and is always available.) * @@ -2291,7 +2291,7 @@ H5_DLL herr_t H5Ovisit_by_name1(hid_t loc_id, const char *obj_name, H5_index_t i * best effort setting. If the application passes in * a value indicating iteration in creation order and a group is * encountered that was not tracked in creation order, that group - * will be iterated over in alpha-numeric order by name, or + * will be iterated over in alphanumeric order by name, or * name order. (Name order is the native order * used by the HDF5 library and is always available.) * @@ -2385,7 +2385,7 @@ H5_DLL herr_t H5Ovisit2(hid_t obj_id, H5_index_t idx_type, H5_iter_order_t order * best effort setting. If the application passes in a * value indicating iteration in creation order and a group is * encountered that was not tracked in creation order, that group - * will be iterated over in alpha-numeric order by name, or + * will be iterated over in alphanumeric order by name, or * name order. (Name order is the native order * used by the HDF5 library and is always available.) * diff --git a/src/H5Orefcount.c b/src/H5Orefcount.c index 52eee14..500ab0f 100644 --- a/src/H5Orefcount.c +++ b/src/H5Orefcount.c @@ -45,7 +45,7 @@ const H5O_msg_class_t H5O_MSG_REFCOUNT[1] = {{ H5O_REFCOUNT_ID, /*message id number */ "refcount", /*message name for debugging */ sizeof(H5O_refcount_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__refcount_decode, /*decode message */ H5O__refcount_encode, /*encode message */ H5O__refcount_copy, /*copy the native value */ diff --git a/src/H5Osdspace.c b/src/H5Osdspace.c index dab989f..574103d 100644 --- a/src/H5Osdspace.c +++ b/src/H5Osdspace.c @@ -61,7 +61,7 @@ const H5O_msg_class_t H5O_MSG_SDSPACE[1] = {{ H5O_SDSPACE_ID, /* message id number */ "dataspace", /* message name for debugging */ sizeof(H5S_extent_t), /* native message size */ - H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are sharable? */ + H5O_SHARE_IS_SHARABLE | H5O_SHARE_IN_OHDR, /* messages are shareable? */ H5O__sdspace_shared_decode, /* decode message */ H5O__sdspace_shared_encode, /* encode message */ H5O__sdspace_copy, /* copy the native value */ diff --git a/src/H5Oshmesg.c b/src/H5Oshmesg.c index 1c894a5..a4e8b8f 100644 --- a/src/H5Oshmesg.c +++ b/src/H5Oshmesg.c @@ -37,7 +37,7 @@ const H5O_msg_class_t H5O_MSG_SHMESG[1] = {{ H5O_SHMESG_ID, /*message id number */ "shared message table", /*message name for debugging */ sizeof(H5O_shmesg_table_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__shmesg_decode, /*decode message */ H5O__shmesg_encode, /*encode message */ H5O__shmesg_copy, /*copy the native value */ diff --git a/src/H5Ostab.c b/src/H5Ostab.c index fa030bd..213b8a0 100644 --- a/src/H5Ostab.c +++ b/src/H5Ostab.c @@ -51,7 +51,7 @@ const H5O_msg_class_t H5O_MSG_STAB[1] = {{ H5O_STAB_ID, /*message id number */ "stab", /*message name for debugging */ sizeof(H5O_stab_t), /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ H5O__stab_decode, /*decode message */ H5O__stab_encode, /*encode message */ H5O__stab_copy, /*copy the native value */ diff --git a/src/H5Ounknown.c b/src/H5Ounknown.c index 2b8a7bb..dc43a6c 100644 --- a/src/H5Ounknown.c +++ b/src/H5Ounknown.c @@ -37,7 +37,7 @@ const H5O_msg_class_t H5O_MSG_UNKNOWN[1] = {{ H5O_UNKNOWN_ID, /*message id number */ "unknown", /*message name for debugging */ 0, /*native message size */ - 0, /* messages are sharable? */ + 0, /* messages are shareable? */ NULL, /*decode message */ NULL, /*encode message */ NULL, /*copy the native value */ diff --git a/src/H5P.c b/src/H5P.c index a2d9cfc..c671ea0 100644 --- a/src/H5P.c +++ b/src/H5P.c @@ -54,9 +54,6 @@ typedef struct { /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -678,7 +675,7 @@ done: NAME H5Pexist PURPOSE - Routine to query the existance of a property in a property object. + Routine to query the existence of a property in a property object. USAGE htri_t H5P_exist(id, name) hid_t id; IN: Property object ID to check @@ -712,7 +709,7 @@ H5Pexist(hid_t id, const char *name) if (!name || !*name) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid property name"); - /* Check for the existance of the property in the list or class */ + /* Check for the existence of the property in the list or class */ if (H5I_GENPROP_LST == H5I_get_type(id)) { if (NULL == (plist = (H5P_genplist_t *)H5I_object(id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list"); @@ -858,7 +855,7 @@ done: Failure: H5I_INVALID_HID (negative) DESCRIPTION Decodes a property list from a binary buffer. The contents of the buffer - contain the values for the correponding properties of the plist. The decode + contain the values for the corresponding properties of the plist. The decode callback of a certain property decodes its value from the buffer and sets it in the property list. GLOBAL VARIABLES @@ -1299,7 +1296,7 @@ done: Returns non-negative on success, negative on failure. DESCRIPTION Removes a property from a property list. Both properties which were - in existance when the property list was created (i.e. properties registered + in existence when the property list was created (i.e. properties registered with H5Pregister2) and properties added to the list after it was created (i.e. added with H5Pinsert2) may be removed from a property list. Properties do not need to be removed a property list before the list itself diff --git a/src/H5PB.c b/src/H5PB.c index ef93bd7..c8bce40 100644 --- a/src/H5PB.c +++ b/src/H5PB.c @@ -109,9 +109,6 @@ static herr_t H5PB__write_raw(H5F_shared_t *, H5FD_mem_t, haddr_t, size_t, const /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -119,7 +116,6 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /*******************/ /* Local Variables */ /*******************/ - /* Declare a free list to manage the H5PB_t struct */ H5FL_DEFINE_STATIC(H5PB_t); @@ -297,7 +293,7 @@ H5PB_print_stats(const H5PB_t *page_buf) (page_buf->bypasses[0] + page_buf->bypasses[1] + page_buf->bypasses[2]), page_buf->bypasses[0], page_buf->bypasses[1], page_buf->bypasses[2]); - HDfprintf(stdout, "acesses = %" PRIi64 " (%" PRIi64 "/%" PRIi64 "/%" PRIi64 ")\n", + HDfprintf(stdout, "accesses = %" PRIi64 " (%" PRIi64 "/%" PRIi64 "/%" PRIi64 ")\n", (page_buf->accesses[0] + page_buf->accesses[1] + page_buf->accesses[2]), page_buf->accesses[0], page_buf->accesses[1], page_buf->accesses[2]); @@ -924,7 +920,7 @@ H5PB_count_meta_access_by_size(H5PB_t *pb, size_t size) * max_pages), simply read from the HDF5 file and return. * * The free space manager guarantees that allocations larger - * than one page will be page alligned, and that allocations + * than one page will be page aligned, and that allocations * of size less than or equal to page size will not cross page * boundaries. Further, unlike raw data, metadata is always * written and read atomically. @@ -1052,7 +1048,7 @@ H5PB_count_meta_access_by_size(H5PB_t *pb, size_t size) * Changes: Updated for discovery of the fact that the fixed and * extensible array data structures allocate multiple * metadata cache entries in a single block, and thus - * violate that invarient that metadata entries either + * violate that invariant that metadata entries either * do not cross page boundaries, or are page aligned. * * JRM -- 3/28/20 @@ -1205,7 +1201,7 @@ H5PB_read(H5F_shared_t *shared, H5FD_mem_t type, haddr_t addr, size_t size, void * empty. * * Since we know that the read crosses at least one - * page boundary, and we have aleady filtered out the + * page boundary, and we have already filtered out the * body only case, at least two of the above must be * non-empty. */ @@ -1542,7 +1538,7 @@ done: * In all contexts, there is no guarantee that the page buffer * will contain any of the possible entries. * - * Space allocations larger than one page must be page alligned. + * Space allocations larger than one page must be page aligned. * Further, any space between the end of a multi-page allocation * and the next page boundary will remain un-allocated until after * the original allocation is freed. This implies that: @@ -1900,7 +1896,7 @@ H5PB_vfd_swmr__release_tick_list(H5F_shared_t *shared) } } /* if the entry is not a multi-page metadata entry, it must already - * be on either the replacment policy or the delayed write list. + * be on either the replacement policy or the delayed write list. * In either case, it will be flush when possible and necessary. */ } @@ -2236,7 +2232,7 @@ done: * max_pages), simply write to the HDF5 file and return. * * The free space manager guarantees that allocations larger - * than one page will be page alligned, and that allocations + * than one page will be page aligned, and that allocations * of size less than or equal to page size will not cross page * boundaries. Further, unlike raw data, metadata is always * written and read atomically. @@ -2256,7 +2252,7 @@ done: * structures, it complicates the metadata cache and the page * buffer. * - * From the page buffer perspective, it breaks the invarient + * From the page buffer perspective, it breaks the invariant * that metadata entries of less than page size don't cross * page boundaries, and those of size greater than or equal * to page size start on page boundaries -- which is important @@ -2511,7 +2507,7 @@ H5PB_write(H5F_shared_t *shared, H5FD_mem_t type, haddr_t addr, size_t size, con * empty. * * Since we know that the write crosses at least one - * page boundary, and we have aleady filtered out the + * page boundary, and we have already filtered out the * body only case, at least two of the above must be * non-empty. */ @@ -3139,7 +3135,7 @@ H5PB__flush_entry(H5F_shared_t *shared, H5PB_t *page_buf, H5PB_entry_t *const en * true in all cases. */ - /* Verify that the base addresss of the page is within the EOA. If it + /* Verify that the base address of the page is within the EOA. If it * isn't, the associated page has been discarded and should have been * removed from the page buffer. This is a bug in the HDF5 library, so * an assertion is adequate here. @@ -3354,7 +3350,7 @@ done: * * evict the entry and test to see if page_buf->curr_pages < * page_buf->max_pages. If it is, return. Otherwise, continue - * the scan until either the above condidtion is fulfilled, + * the scan until either the above condition is fulfilled, * or the head of the LRU is reach. * * Under normal circumstances, it should always be possible @@ -3725,7 +3721,7 @@ done: * Y | Y | MPMDE | N | X | Error (case 9) * ------+------+--------+------+---------+------------------------------------- * Y | Y | MPMDE | Y | X | Clip read to MPE size if required. - * | | | | | Satify read from MPE (case 9) + * | | | | | Satisfy read from MPE (case 9) * ------+------+--------+------+---------+------------------------------------- * | | | | | * ------+------+--------+------+---------+------------------------------------- @@ -4195,7 +4191,7 @@ H5PB__read_raw(H5F_shared_t *shared, H5FD_mem_t type, haddr_t addr, size_t size, else { /* this is an internal page -- copy it in its - * entireity. + * entirety. */ offset = @@ -4375,7 +4371,7 @@ H5PB__write_meta(H5F_shared_t *shared, H5FD_mem_t type, haddr_t addr, size_t siz H5PB_t * page_buf; /* Page buffer for this file */ H5PB_entry_t *entry_ptr; /* Pointer to page buffer entry */ uint64_t page; /* page offset of addr */ - haddr_t page_addr; /* page containg addr */ + haddr_t page_addr; /* page containing addr */ size_t offset; /* offset of write in page */ herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5PBpkg.h b/src/H5PBpkg.h index f06206b..03c75bd 100644 --- a/src/H5PBpkg.h +++ b/src/H5PBpkg.h @@ -153,7 +153,7 @@ /* successor at head of list -- prepend */ \ H5PB__DLL_PREPEND(entry_ptr, head_ptr, tail_ptr, len, Size, fail_val) \ \ - else /* sucessor in body of list -- insert before it */ \ + else /* successor in body of list -- insert before it */ \ { \ H5PB__DLL_PRE_INSERT_SC(entry_ptr, head_ptr, tail_ptr, len, Size, fail_val) \ HDassert(suc_ptr->prev->magic == H5PB__H5PB_ENTRY_T_MAGIC); \ @@ -278,7 +278,7 @@ /* successor at head of list -- prepend */ \ H5PB__IL_DLL_PREPEND(entry_ptr, head_ptr, tail_ptr, len, Size, fail_val) \ \ - else /* sucessor in body of list -- insert before it */ \ + else /* successor in body of list -- insert before it */ \ { \ H5PB__IL_DLL_PRE_INSERT_SC(entry_ptr, head_ptr, tail_ptr, len, Size, fail_val) \ HDassert(suc_ptr->il_prev->magic == H5PB__H5PB_ENTRY_T_MAGIC); \ @@ -1458,7 +1458,7 @@ * Macro: H5PB__INSERT_IN_DWL * * Insert the supplied page buffer entry in the delayed write list - * maintaining the invarient: + * maintaining the invariant: * * entry_ptr->next == NULL || * entry_ptr->delay_write_until >= entry_ptr->next->delay_write_until @@ -1599,7 +1599,7 @@ * Fields supporting the hash table: * * Entries in the page buffer are indexed by a more or less conventional - * hash table with chaining (see header comment on H5PB_t for futher details). + * hash table with chaining (see header comment on H5PB_t for further details). * If there are multiple entries in any hash bin, they are stored in a doubly * linked list. * @@ -1647,7 +1647,7 @@ * Fields supporting VFD SWMR: * * is_mpmde: Boolean flag that is set to TRUE iff the entry - * is a multi-page metadata entry. In the absense of VFD + * is a multi-page metadata entry. In the absence of VFD * SWMR, the field should always be set to FALSE. * * Observe that: @@ -1668,15 +1668,15 @@ * * delay_write_until: Unsigned 64 bit integer containing the first tick * in which the entry may be written to file, or 0 if there - * is no such constraint. It should be set ot 0 when VFD + * is no such constraint. It should be set to 0 when VFD * is not enabled. * * tl_next: Next pointer on the list of entries modified in the current - * tick, If the enty is not on the tick list, or if there is + * tick, If the entry is not on the tick list, or if there is * no next entry on the list, this field should be set to NULL. * * tl_prev: Prev pointer on the list of entries modified in the current - * tick, If the enty is not on the tick list, or if there is + * tick, If the entry is not on the tick list, or if there is * no previous entry on the list, this field should be set to * NULL. * diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h index 15ee749..c23336e 100644 --- a/src/H5PBprivate.h +++ b/src/H5PBprivate.h @@ -308,7 +308,7 @@ typedef struct H5PB_entry_t H5PB_entry_t; * * To facilitate identification of entries that must be removed from the * DWL during the end of tick scan, the list always observes the following - * invarient for any entry on the list: + * invariant for any entry on the list: * * entry_ptr->next == NULL || * entry_ptr->delay_write_until >= entry_ptr->next->delay_write_until @@ -528,7 +528,7 @@ typedef struct H5PB_entry_t H5PB_entry_t; * max_dwl_size: int64_t containing the maximum value of dwl_size. * * total_dwl_ins_depth: int64_t containing the total insertion depth - * required to maintain the odering invarient on the + * required to maintain the odering invariant on the * delayed write list. * * md_read_splits: int64_t containing the number of metadata reads that diff --git a/src/H5PL.c b/src/H5PL.c index 30b6c52..b052078 100644 --- a/src/H5PL.c +++ b/src/H5PL.c @@ -67,7 +67,7 @@ * the special "::" string. * * Return: Success: Non-negative - * Failture: Negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -99,7 +99,7 @@ done: * Positive if one or more of the plugin types are enabled * * Return: Success: Non-negative - * Failture: Negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -128,7 +128,7 @@ done: * Purpose: Insert a plugin search path at the end of the list. * * Return: Success: Non-negative - * Failture: Negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -160,7 +160,7 @@ done: * Purpose: Insert a plugin search path at the beginning of the list. * * Return: Success: Non-negative - * Failture: Negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -234,7 +234,7 @@ done: * other paths after the index. * * Return: Success: Non-negative - * Failture: Negative + * Failure: Negative * *------------------------------------------------------------------------- */ @@ -270,11 +270,11 @@ done: /*------------------------------------------------------------------------- * Function: H5PLremove * - * Purpose: Remove the plugin path at the specifed index and compact + * Purpose: Remove the plugin path at the specified index and compact * the list. * * Return: Success: Non-negative - * Failture: Negative + * Failure: Negative * * Return: Non-negative or success. * diff --git a/src/H5PLint.c b/src/H5PLint.c index d20401e..fe9d3c1 100644 --- a/src/H5PLint.c +++ b/src/H5PLint.c @@ -46,9 +46,6 @@ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -123,28 +120,27 @@ H5PL__set_plugin_control_mask(unsigned int mask) } /* end H5PL__set_plugin_control_mask() */ /*------------------------------------------------------------------------- - * Function: H5PL__init_package - * - * Purpose: Initialize any package-specific data and call any init - * routines for the package. + * Function: H5PL_init * - * Return: SUCCEED/FAIL + * Purpose: Initialize the interface from some other layer. * + * Return: Success: non-negative + * Failure: negative *------------------------------------------------------------------------- */ herr_t -H5PL__init_package(void) +H5PL_init(void) { char * env_var = NULL; herr_t ret_value = SUCCEED; - FUNC_ENTER_PACKAGE + FUNC_ENTER_NOAPI(FAIL) /* Check the environment variable to determine if the user wants * to ignore plugins. The special symbol H5PL_NO_PLUGIN (defined in * H5PLpublic.h) means we don't want to load plugins. */ - if (NULL != (env_var = HDgetenv("HDF5_PLUGIN_PRELOAD"))) + if (NULL != (env_var = HDgetenv(HDF5_PLUGIN_PRELOAD))) if (!HDstrcmp(env_var, H5PL_NO_PLUGIN)) { H5PL_plugin_control_mask_g = 0; H5PL_allow_plugins_g = FALSE; @@ -160,7 +156,7 @@ H5PL__init_package(void) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__init_package() */ +} /*------------------------------------------------------------------------- * Function: H5PL_term_package @@ -183,24 +179,17 @@ H5PL_term_package(void) FUNC_ENTER_NOAPI_NOINIT - if (H5_PKG_INIT_VAR) { - - /* Close the plugin cache. - * We need to bump the return value if we did any real work here. - */ - if (H5PL__close_plugin_cache(&already_closed) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache") - if (!already_closed) - ret_value++; - - /* Close the search path table and free the paths */ - if (H5PL__close_path_table() < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table") + /* Close the plugin cache. + * We need to bump the return value if we did any real work here. + */ + if (H5PL__close_plugin_cache(&already_closed) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache") + if (!already_closed) + ret_value++; - /* Mark the interface as uninitialized */ - if (0 == ret_value) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Close the search path table and free the paths */ + if (H5PL__close_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table") done: FUNC_LEAVE_NOAPI(ret_value) @@ -243,6 +232,11 @@ H5PL_load(H5PL_type_t type, const H5PL_key_t *key) "Virtual Object Layer (VOL) driver plugins disabled") break; + case H5PL_TYPE_VFD: + if ((H5PL_plugin_control_mask_g & H5PL_VFD_PLUGIN) == 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "Virtual File Driver (VFD) plugins disabled") + break; + case H5PL_TYPE_ERROR: case H5PL_TYPE_NONE: default: @@ -275,9 +269,31 @@ done: * * Purpose: Opens a plugin. * - * The success parameter will be set to TRUE and the plugin_info - * parameter will be filled in on success. Otherwise, they - * will be FALSE and NULL, respectively. + * `path` specifies the path to the plugin library file. + * + * `type` specifies the type of plugin being searched for and + * will be used to verify that a loaded plugin matches the + * type requested. H5PL_TYPE_NONE may be passed, in which case + * no plugin type verification is performed. This is most + * useful when iterating over available plugins without regard + * to their types. + * + * `key` specifies the information that will be used to find a + * specific plugin. For filter plugins, this is typically an + * integer identifier. For VOL connector and VFD plugins, this + * is typically either an integer identifier or a name string. + * After a plugin has been opened, this information will be + * compared against the relevant information provided by the + * plugin to ensure that the plugin is a match. If + * H5PL_TYPE_NONE is provided for `type`, then `key` should be + * NULL. + * + * On successful open of a plugin, the `success` parameter + * will be set to TRUE and the `plugin_type` and `plugin_info` + * parameters will be filled appropriately. On failure, the + * `success` parameter will be set to FALSE, the `plugin_type` + * parameter will be set to H5PL_TYPE_ERROR and the + * `plugin_info` parameter will be set to NULL. * * Return: SUCCEED/FAIL * @@ -308,6 +324,8 @@ H5PL__open(const char *path, H5PL_type_t type, const H5PL_key_t *key, hbool_t *s /* Check args - Just assert on package functions */ HDassert(path); + if (type == H5PL_TYPE_NONE) + HDassert(!key); HDassert(success); HDassert(plugin_info); @@ -396,6 +414,34 @@ H5PL__open(const char *path, H5PL_type_t type, const H5PL_key_t *key, hbool_t *s break; } + case H5PL_TYPE_VFD: { + const void *cls; + + /* Get the plugin info */ + if (NULL == (cls = (const void *)(*get_plugin_info)())) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get VFD info from plugin") + + /* Setup temporary plugin key if one wasn't supplied */ + if (!key) { + tmp_key.vfd.kind = H5FD_GET_DRIVER_BY_NAME; + tmp_key.vfd.u.name = ((const H5FD_class_t *)cls)->name; + key = &tmp_key; + } + + /* Ask VFD interface if this class is the one we are looking for and is compatible, etc */ + if (H5FD_check_plugin_load(cls, key, success) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, FAIL, "VFD compatibility check failed") + + /* Check for finding the correct plugin */ + if (*success) { + if (plugin_type) + *plugin_type = H5PL_TYPE_VFD; + *plugin_info = cls; + } + + break; + } + case H5PL_TYPE_ERROR: case H5PL_TYPE_NONE: default: @@ -453,10 +499,9 @@ H5PL_iterate(H5PL_iterate_type_t iter_type, H5PL_iterate_t iter_op, void *op_dat { herr_t ret_value = H5_ITER_CONT; - FUNC_ENTER_NOAPI(H5_ITER_ERROR) + FUNC_ENTER_NOAPI_NOERR ret_value = H5PL__path_table_iterate(iter_type, iter_op, op_data); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5PL_iterate() */ diff --git a/src/H5PLpath.c b/src/H5PLpath.c index 39a7d0d..b86fd6e 100644 --- a/src/H5PLpath.c +++ b/src/H5PLpath.c @@ -248,7 +248,7 @@ H5PL__create_path_table(void) /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it * or from the default paths if it isn't set. */ - env_var = HDgetenv("HDF5_PLUGIN_PATH"); + env_var = HDgetenv(HDF5_PLUGIN_PATH); if (NULL == env_var) paths = H5MM_strdup(H5PL_DEFAULT_PATH); else @@ -322,7 +322,7 @@ H5PL__close_path_table(void) * Purpose: Gets the number of plugin paths that have been stored. * * Return: Success: The number of paths - * Failture: Can't fail + * Failure: Can't fail *------------------------------------------------------------------------- */ unsigned @@ -571,12 +571,12 @@ H5PL__path_table_iterate(H5PL_iterate_type_t iter_type, H5PL_iterate_t iter_op, FUNC_ENTER_PACKAGE - for (u = 0; (u < H5PL_num_paths_g) && (ret_value == H5_ITER_CONT); u++) - ret_value = H5PL__path_table_iterate_process_path(H5PL_paths_g[u], iter_type, iter_op, op_data); - - if (ret_value < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_BADITER, H5_ITER_ERROR, "can't iterate over plugins in plugin path '%s'", - H5PL_paths_g[u]); + for (u = 0; (u < H5PL_num_paths_g) && (ret_value == H5_ITER_CONT); u++) { + if ((ret_value = + H5PL__path_table_iterate_process_path(H5PL_paths_g[u], iter_type, iter_op, op_data)) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_BADITER, H5_ITER_ERROR, + "can't iterate over plugins in plugin path '%s'", H5PL_paths_g[u]); + } done: FUNC_LEAVE_NOAPI(ret_value); @@ -615,9 +615,9 @@ H5PL__path_table_iterate_process_path(const char *plugin_path, H5PL_iterate_type HDassert(plugin_path); HDassert(iter_op); - /* Open the directory */ + /* Open the directory - skip the path if the directory can't be opened */ if (!(dirp = HDopendir(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, H5_ITER_ERROR, "can't open directory: %s", plugin_path) + HGOTO_DONE(H5_ITER_CONT) /* Iterate through all entries in the directory */ while (NULL != (dp = HDreaddir(dirp))) { @@ -663,7 +663,8 @@ H5PL__path_table_iterate_process_path(const char *plugin_path, H5PL_iterate_type /* Determine if we should process this plugin */ plugin_matches = (iter_type == H5PL_ITER_TYPE_ALL) || ((iter_type == H5PL_ITER_TYPE_FILTER) && (plugin_type == H5PL_TYPE_FILTER)) || - ((iter_type == H5PL_ITER_TYPE_VOL) && (plugin_type == H5PL_TYPE_VOL)); + ((iter_type == H5PL_ITER_TYPE_VOL) && (plugin_type == H5PL_TYPE_VOL)) || + ((iter_type == H5PL_ITER_TYPE_VFD) && (plugin_type == H5PL_TYPE_VFD)); /* If the plugin was successfully loaded, call supplied callback function on plugin */ if (plugin_loaded && plugin_matches && (ret_value = iter_op(plugin_type, plugin_info, op_data))) @@ -706,10 +707,11 @@ H5PL__path_table_iterate_process_path(const char *plugin_path, H5PL_iterate_type HDassert(plugin_path); HDassert(iter_op); - /* Specify a file mask. *.* = We want everything! */ - HDsprintf(service, "%s\\*.dll", plugin_path); + /* Specify a file mask. *.* = We want everything! - + * skip the path if the directory can't be opened */ + HDsnprintf(service, sizeof(service), "%s\\*.dll", plugin_path); if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, H5_ITER_ERROR, "can't open directory") + HGOTO_DONE(H5_ITER_CONT) /* Loop over all the files */ do { @@ -740,7 +742,8 @@ H5PL__path_table_iterate_process_path(const char *plugin_path, H5PL_iterate_type /* Determine if we should process this plugin */ plugin_matches = (iter_type == H5PL_ITER_TYPE_ALL) || ((iter_type == H5PL_ITER_TYPE_FILTER) && (plugin_type == H5PL_TYPE_FILTER)) || - ((iter_type == H5PL_ITER_TYPE_VOL) && (plugin_type == H5PL_TYPE_VOL)); + ((iter_type == H5PL_ITER_TYPE_VOL) && (plugin_type == H5PL_TYPE_VOL)) || + ((iter_type == H5PL_ITER_TYPE_VFD) && (plugin_type == H5PL_TYPE_VFD)); /* If the plugin was successfully loaded, call supplied callback function on plugin */ if (plugin_loaded && plugin_matches && (ret_value = iter_op(plugin_type, plugin_info, op_data))) @@ -931,7 +934,7 @@ H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *fo *found = FALSE; /* Specify a file mask. *.* = We want everything! */ - HDsprintf(service, "%s\\*.dll", dir); + HDsnprintf(service, sizeof(service), "%s\\*.dll", dir); if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory") diff --git a/src/H5PLplugin_cache.c b/src/H5PLplugin_cache.c index b7cdac0..25e373f 100644 --- a/src/H5PLplugin_cache.c +++ b/src/H5PLplugin_cache.c @@ -263,59 +263,99 @@ H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *f /* Loop over all the plugins, looking for one that matches */ for (u = 0; u < H5PL_num_plugins_g; u++) { - /* If the plugin type (filter, VOL connector, etc.) match, proceed */ - if (search_params->type == H5PL_cache_g[u].type) { - hbool_t matched = FALSE; /* Whether cached plugin info matches */ + hbool_t matched = FALSE; /* Whether cached plugin info matches */ - switch (search_params->type) { - case H5PL_TYPE_FILTER: - if (search_params->key->id == H5PL_cache_g[u].key.id) + /* Determine if the plugin types match */ + if (search_params->type != H5PL_cache_g[u].type) + continue; + + /* Determine if cache entry matches based on type-specific information */ + switch (search_params->type) { + case H5PL_TYPE_FILTER: + /* Check if specified filter plugin ID matches cache entry's ID */ + if (search_params->key->id == H5PL_cache_g[u].key.id) + matched = TRUE; + + break; + + case H5PL_TYPE_VOL: + if (search_params->key->vol.kind == H5VL_GET_CONNECTOR_BY_NAME) { + /* Make sure the plugin cache entry key type matches our search key type */ + if (H5PL_cache_g[u].key.vol.kind != H5VL_GET_CONNECTOR_BY_NAME) + continue; + + /* Check if specified VOL connector name matches cache entry's name */ + if (!HDstrcmp(search_params->key->vol.u.name, H5PL_cache_g[u].key.vol.u.name)) + matched = TRUE; + } + else { + HDassert(search_params->key->vol.kind == H5VL_GET_CONNECTOR_BY_VALUE); + + /* Make sure the plugin cache entry key type matches our search key type */ + if (H5PL_cache_g[u].key.vol.kind != H5VL_GET_CONNECTOR_BY_VALUE) + continue; + + /* Check if specified VOL connector ID matches cache entry's ID */ + if (search_params->key->vol.u.value == H5PL_cache_g[u].key.vol.u.value) + matched = TRUE; + } + + break; + + case H5PL_TYPE_VFD: + if (search_params->key->vfd.kind == H5FD_GET_DRIVER_BY_NAME) { + /* Make sure the plugin cache entry key type matches our search key type */ + if (H5PL_cache_g[u].key.vfd.kind != H5FD_GET_DRIVER_BY_NAME) + continue; + + /* Check if specified VFD name matches cache entry's name */ + if (!HDstrcmp(search_params->key->vfd.u.name, H5PL_cache_g[u].key.vfd.u.name)) matched = TRUE; - break; - - case H5PL_TYPE_VOL: - if (search_params->key->vol.kind == H5VL_GET_CONNECTOR_BY_VALUE) { - if (search_params->key->vol.u.value == H5PL_cache_g[u].key.vol.u.value) - matched = TRUE; - } /* end if */ - else if (search_params->key->vol.kind == H5VL_GET_CONNECTOR_BY_NAME) { - if (0 == HDstrcmp(search_params->key->vol.u.name, H5PL_cache_g[u].key.vol.u.name)) - matched = TRUE; - } /* end else-if */ - else - HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "bad VOL plugin search key type") - break; - - case H5PL_TYPE_ERROR: - case H5PL_TYPE_NONE: - default: - HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "bad plugin type") - } /* end switch */ - - /* If the plugin type (filter, VOL connector, etc.) and key match, query the plugin for its info - */ - if (matched) { - H5PL_get_plugin_info_t get_plugin_info_function; - const void * info; - - /* Get the "get plugin info" function from the plugin. */ - if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC( - (H5PL_cache_g[u]).handle, "H5PLget_plugin_info"))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") - - /* Call the "get plugin info" function */ - if (NULL == (info = (*get_plugin_info_function)())) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") - - /* Set output parameters */ - *found = TRUE; - *plugin_info = info; - - /* No need to continue processing */ + } + else { + HDassert(search_params->key->vfd.kind == H5FD_GET_DRIVER_BY_VALUE); + + /* Make sure the plugin cache entry key type matches our search key type */ + if (H5PL_cache_g[u].key.vfd.kind != H5FD_GET_DRIVER_BY_VALUE) + continue; + + /* Check if specified VFD ID matches cache entry's ID */ + if (search_params->key->vfd.u.value == H5PL_cache_g[u].key.vfd.u.value) + matched = TRUE; + } + break; - } /* end if */ - } /* end if */ - } /* end for */ + + case H5PL_TYPE_ERROR: + case H5PL_TYPE_NONE: + default: + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "Invalid plugin type specified") + } + + /* If the plugin type (filter, VOL connector, VFD plugin, etc.) and key match, + * query the plugin for its info. + */ + if (matched) { + H5PL_get_plugin_info_t get_plugin_info_function; + const void * info; + + /* Get the "get plugin info" function from the plugin. */ + if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC( + H5PL_cache_g[u].handle, "H5PLget_plugin_info"))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") + + /* Call the "get plugin info" function */ + if (NULL == (info = (*get_plugin_info_function)())) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") + + /* Set output parameters */ + *found = TRUE; + *plugin_info = info; + + /* No need to continue processing */ + break; + } + } /* end for */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5PLprivate.h b/src/H5PLprivate.h index 7eae513..83cc8a5 100644 --- a/src/H5PLprivate.h +++ b/src/H5PLprivate.h @@ -22,6 +22,7 @@ /* Private headers needed by this file */ #include "H5private.h" /* Generic Functions */ +#include "H5FDprivate.h" /* File Drivers */ #include "H5VLprivate.h" /* Virtual Object Layer */ /**************************/ @@ -32,16 +33,29 @@ /* Library Private Typedefs */ /****************************/ +/* Key used to find VOL connector plugins */ +typedef struct H5PL_vol_key_t { + H5VL_get_connector_kind_t kind; /* Kind of VOL lookup to do */ + union { + H5VL_class_value_t value; /* VOL connector value */ + const char * name; /* VOL connector name */ + } u; +} H5PL_vol_key_t; + +/* Key used to find VFD plugins */ +typedef struct H5PL_vfd_key_t { + H5FD_get_driver_kind_t kind; /* Kind of VFD lookup to do */ + union { + H5FD_class_value_t value; /* VFD value */ + const char * name; /* VFD name */ + } u; +} H5PL_vfd_key_t; + /* The key that will be used to find the plugin */ typedef union H5PL_key_t { - int id; /* I/O filters */ - struct { - H5VL_get_connector_kind_t kind; /* Kind of VOL lookup to do */ - union { - H5VL_class_value_t value; /* VOL connector value */ - const char * name; /* VOL connector name */ - } u; - } vol; + int id; /* I/O filters */ + H5PL_vol_key_t vol; + H5PL_vfd_key_t vfd; } H5PL_key_t; /* Enum dictating the type of plugins to process @@ -50,6 +64,7 @@ typedef union H5PL_key_t { typedef enum { H5PL_ITER_TYPE_FILTER, H5PL_ITER_TYPE_VOL, + H5PL_ITER_TYPE_VFD, H5PL_ITER_TYPE_ALL, } H5PL_iterate_type_t; @@ -67,5 +82,6 @@ typedef herr_t (*H5PL_iterate_t)(H5PL_type_t plugin_type, const void *plugin_inf /* Internal API routines */ H5_DLL const void *H5PL_load(H5PL_type_t plugin_type, const H5PL_key_t *key); H5_DLL herr_t H5PL_iterate(H5PL_iterate_type_t iter_type, H5PL_iterate_t iter_op, void *op_data); +H5_DLL herr_t H5PL_init(void); #endif /* H5PLprivate_H */ diff --git a/src/H5PLpublic.h b/src/H5PLpublic.h index 55ff594..ab24bd5 100644 --- a/src/H5PLpublic.h +++ b/src/H5PLpublic.h @@ -35,14 +35,16 @@ typedef enum H5PL_type_t { H5PL_TYPE_ERROR = -1, /**< Error */ H5PL_TYPE_FILTER = 0, /**< Filter */ - H5PL_TYPE_VOL = 1, /**< VOL driver */ - H5PL_TYPE_NONE = 2 /**< Sentinel: This must be last! */ + H5PL_TYPE_VOL = 1, /**< VOL connector */ + H5PL_TYPE_VFD = 2, /**< VFD */ + H5PL_TYPE_NONE = 3 /**< Sentinel: This must be last! */ } H5PL_type_t; //! /* Common dynamic plugin type flags used by the set/get_loading_state functions */ #define H5PL_FILTER_PLUGIN 0x0001 #define H5PL_VOL_PLUGIN 0x0002 +#define H5PL_VFD_PLUGIN 0x0004 #define H5PL_ALL_PLUGIN 0xFFFF #ifdef __cplusplus diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 46dc94c..c901931 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -1991,7 +1991,7 @@ done: /*------------------------------------------------------------------------- * Function: H5Pget_mpio_actual_io_mode * - * Purpose: Retrieves the type of I/O actually preformed when collective I/O + * Purpose: Retrieves the type of I/O actually performed when collective I/O * is requested. * * Return: Non-negative on success/Negative on failure @@ -2213,10 +2213,18 @@ H5P__dxfr_dset_io_hyp_sel_cmp(const void *_space1, const void *_space2, size_t H if (TRUE != H5S_extent_equal(*space1, *space2)) HGOTO_DONE(-1); - /* Compare the selection "shape" of the dataspaces */ - /* (Error & not-equal count the same) */ - if (TRUE != H5S_select_shape_same(*space1, *space2)) + /* Compare the selection "shape" of the dataspaces + * (Error & not-equal count the same) + * + * Since H5S_select_shape_same() can result in the dataspaces being + * rebuilt, the parameters are not const which makes it impossible + * to match the cmp prototype. Since we need to compare them, + * we quiet the const warning. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") + if (TRUE != H5S_select_shape_same((H5S_t *)*space1, (H5S_t *)*space2)) HGOTO_DONE(-1); + H5_GCC_CLANG_DIAG_ON("cast-qual") } /* end if */ done: diff --git a/src/H5Pencdec.c b/src/H5Pencdec.c index e2a97f8..01573d1 100644 --- a/src/H5Pencdec.c +++ b/src/H5Pencdec.c @@ -663,7 +663,7 @@ done: on failure. DESCRIPTION Decodes a property list from a binary buffer. The contents of the buffer - contain the values for the correponding properties of the plist. The decode + contain the values for the corresponding properties of the plist. The decode callback of a certain property decodes its value from the buffer and sets it in the property list. GLOBAL VARIABLES diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index ff9ff8a..65d426b 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -41,8 +41,28 @@ #include "H5VMprivate.h" /* Vector Functions */ /* Includes needed to set default file driver */ -#include "H5FDsec2.h" /* POSIX unbuffered I/O */ +#include "H5FDsec2.h" /* POSIX unbuffered I/O */ +#include "H5FDcore.h" +#include "H5FDlog.h" +#include "H5FDfamily.h" +#include "H5FDmulti.h" #include "H5FDstdio.h" /* Standard C buffered I/O */ +#include "H5FDsplitter.h" +#ifdef H5_HAVE_PARALLEL +#include "H5FDmpio.h" +#endif +#ifdef H5_HAVE_DIRECT +#include "H5FDdirect.h" +#endif +#ifdef H5_HAVE_MIRROR_VFD +#include "H5FDmirror.h" +#endif +#ifdef H5_HAVE_LIBHDFS +#include "H5FDhdfs.h" +#endif +#ifdef H5_HAVE_ROS3_VFD +#include "H5FDros3.h" +#endif #ifdef H5_HAVE_WINDOWS #include "H5FDwindows.h" /* Win32 I/O */ #endif @@ -114,7 +134,7 @@ #define H5F_ACS_FILE_DRV_SIZE sizeof(H5FD_driver_prop_t) #define H5F_ACS_FILE_DRV_DEF \ { \ - H5_DEFAULT_VFD, NULL \ + H5_DEFAULT_VFD, NULL, NULL \ } #define H5F_ACS_FILE_DRV_CRT H5P__facc_file_driver_create #define H5F_ACS_FILE_DRV_SET H5P__facc_file_driver_set @@ -416,6 +436,9 @@ static int H5P__facc_mpi_info_cmp(const void *value1, const void *value2, siz static herr_t H5P__facc_mpi_info_close(const char *name, size_t size, void *value); #endif /* H5_HAVE_PARALLEL */ +/* Internal routines */ +static herr_t H5P__facc_set_def_driver_check_predefined(const char *driver_name, hid_t *driver_id); + /*********************/ /* Package Variables */ /*********************/ @@ -518,7 +541,7 @@ static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ static const unsigned H5F_def_page_buf_min_raw_perc_g = - H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */ + H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer minimum raw data size */ static const hbool_t H5F_def_use_file_locking_g = H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ static const hbool_t H5F_def_ignore_disabled_file_locks_g = @@ -851,6 +874,208 @@ done: } /* end H5P__facc_reg_prop() */ /*------------------------------------------------------------------------- + * Function: H5P__facc_set_def_driver + * + * Purpose: Parses a string that contains the name of the default VFL + * driver for the default FAPL. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5P__facc_set_def_driver(void) +{ + const char *driver_env_var; + hbool_t driver_ref_inc = FALSE; + hid_t driver_id = H5I_INVALID_HID; /* VFL driver ID */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Check if VFL driver environment variable is set */ + driver_env_var = HDgetenv(HDF5_DRIVER); + + /* Only parse VFL driver string if it's set */ + if (driver_env_var && *driver_env_var) { + H5FD_driver_prop_t driver_prop; + H5P_genplist_t * def_fapl; /* Default file access property list */ + H5P_genclass_t * def_fapclass; /* Default file access property class */ + const char * driver_config_env_var; + htri_t driver_is_registered; + + /* Check to see if the driver is already registered */ + if ((driver_is_registered = H5FD_is_driver_registered_by_name(driver_env_var, &driver_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't check if VFL driver is already registered") + if (driver_is_registered) { + HDassert(driver_id >= 0); + + if (H5I_inc_ref(driver_id, TRUE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINC, FAIL, "unable to increment ref count on VFD") + driver_ref_inc = TRUE; + } /* end else-if */ + else { + /* Check for VFL drivers that ship with the library */ + if (H5P__facc_set_def_driver_check_predefined(driver_env_var, &driver_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't check for predefined VFL driver name") + else if (driver_id > 0) { + if (H5I_inc_ref(driver_id, TRUE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINC, FAIL, "can't increment VFL driver refcount") + driver_ref_inc = TRUE; + } + else { + /* Register the VFL driver */ + if ((driver_id = H5FD_register_driver_by_name(driver_env_var, TRUE)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTREGISTER, FAIL, "can't register VFL driver") + driver_ref_inc = TRUE; + } /* end else */ + } /* end else */ + + /* Retrieve driver configuration string from environment variable, if set. */ + driver_config_env_var = HDgetenv(HDF5_DRIVER_CONFIG); + + driver_prop.driver_id = driver_id; + driver_prop.driver_info = NULL; + driver_prop.driver_config_str = driver_config_env_var; + + /* Get default file access pclass */ + if (NULL == (def_fapclass = (H5P_genclass_t *)H5I_object(H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_VFL, H5E_BADID, FAIL, + "can't find object for default file access property class ID") + + /* Set new default VFL driver for default file access pclass */ + if (H5P__class_set(def_fapclass, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, + "can't set default VFL driver for default file access property list class") + + /* Get default file access plist */ + if (NULL == (def_fapl = (H5P_genplist_t *)H5I_object(H5P_FILE_ACCESS_DEFAULT))) + HGOTO_ERROR(H5E_VFL, H5E_BADID, FAIL, "can't find object for default fapl ID") + + /* Set new default VFL driver for default FAPL */ + if (H5P_set_driver(def_fapl, driver_prop.driver_id, driver_prop.driver_info, + driver_prop.driver_config_str) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't set default VFL driver for default FAPL") + } + +done: + /* Clean up on error */ + if (ret_value < 0) { + if (driver_id >= 0 && driver_ref_inc && H5I_dec_app_ref(driver_id) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTDEC, FAIL, "unable to unregister VFL driver") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_set_def_driver() */ + +/*------------------------------------------------------------------------- + * Function: H5P__facc_set_def_driver_check_predefined + * + * Purpose: Checks a given driver name against a list of predefined + * names for VFL drivers that are internal to HDF5. If a name + * is matched, the ID for that driver is returned through + * `driver_id`. Otherwise, `driver_id` is set to + * H5I_INVALID_HID. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__facc_set_def_driver_check_predefined(const char *driver_name, hid_t *driver_id) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(driver_name); + HDassert(driver_id); + + if (!HDstrcmp(driver_name, "sec2")) { + if ((*driver_id = H5FD_SEC2) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize sec2 VFD") + } + else if (!HDstrcmp(driver_name, "core") || !HDstrcmp(driver_name, "core_paged")) { + if ((*driver_id = H5FD_CORE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize core VFD") + } + else if (!HDstrcmp(driver_name, "log")) { + if ((*driver_id = H5FD_LOG) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize log VFD") + } + else if (!HDstrcmp(driver_name, "family")) { + if ((*driver_id = H5FD_FAMILY) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize family VFD") + } + else if (!HDstrcmp(driver_name, "multi") || !HDstrcmp(driver_name, "split")) { + if ((*driver_id = H5FD_MULTI) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize multi VFD") + } + else if (!HDstrcmp(driver_name, "stdio")) { + if ((*driver_id = H5FD_STDIO) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize stdio VFD") + } + else if (!HDstrcmp(driver_name, "splitter")) { + if ((*driver_id = H5FD_SPLITTER) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize splitter VFD") + } + else if (!HDstrcmp(driver_name, "mpio")) { +#ifdef H5_HAVE_PARALLEL + if ((*driver_id = H5FD_MPIO) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize MPI I/O VFD") +#else + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "MPI-I/O VFD is not enabled") +#endif + } + else if (!HDstrcmp(driver_name, "direct")) { +#ifdef H5_HAVE_DIRECT + if ((*driver_id = H5FD_DIRECT) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize Direct I/O VFD") +#else + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "Direct I/O VFD is not enabled") +#endif + } + else if (!HDstrcmp(driver_name, "mirror")) { +#ifdef H5_HAVE_MIRROR_VFD + if ((*driver_id = H5FD_MIRROR) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize mirror VFD") +#else + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "Mirror VFD is not enabled") +#endif + } + else if (!HDstrcmp(driver_name, "hdfs")) { +#ifdef H5_HAVE_LIBHDFS + if ((*driver_id = H5FD_HDFS) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize HDFS VFD") +#else + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "HDFS VFD is not enabled") +#endif + } + else if (!HDstrcmp(driver_name, "ros3")) { +#ifdef H5_HAVE_ROS3_VFD + if ((*driver_id = H5FD_ROS3) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize ROS3 VFD") +#else + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "ROS3 VFD is not enabled") +#endif + } + else if (!HDstrcmp(driver_name, "windows")) { +#ifdef H5_HAVE_WINDOWS + if ((*driver_id = H5FD_WINDOWS) < 0) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, "couldn't initialize Windows VFD") +#else + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "Windows VFD is not enabled") +#endif + } + else { + *driver_id = H5I_INVALID_HID; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__facc_set_def_driver_check_predefined() */ + +/*------------------------------------------------------------------------- * Function: H5Pset_alignment * * Purpose: Sets the alignment properties of a file access property list @@ -941,42 +1166,52 @@ done: } /* end H5Pget_alignment() */ /*------------------------------------------------------------------------- - * Function: H5P_set_driver + * Function: H5P_set_driver * - * Purpose: Set the file driver (DRIVER_ID) for a file access - * property list (PLIST_ID) and supply an optional - * struct containing the driver-specific properites - * (DRIVER_INFO). The driver properties will be copied into the - * property list and the reference count on the driver will be - * incremented, allowing the caller to close the driver ID but - * still use the property list. + * Purpose: Set the file driver (NEW_DRIVER_ID) for a file access + * property list (PLIST). A struct (NEW_DRIVER_INFO) or string + * (NEW_DRIVER_CONFIG_STR) containing the driver-specific + * properties can optionally be supplied. The driver properties + * (struct or string) will be copied into the property list and + * the reference count on the driver will be incremented, + * allowing the caller to close the driver ID but still use the + * property list. * - * Return: Success: Non-negative - * Failure: Negative + * Note: Only one of either NEW_DRIVER_INFO or NEW_DRIVER_CONFIG_STR + * should be specified, but not both. + * + * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke - * Tuesday, August 3, 1999 + * Tuesday, August 3, 1999 * *------------------------------------------------------------------------- */ herr_t -H5P_set_driver(H5P_genplist_t *plist, hid_t new_driver_id, const void *new_driver_info) +H5P_set_driver(H5P_genplist_t *plist, hid_t new_driver_id, const void *new_driver_info, + const char *new_driver_config_str) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) + /* If VFD configuration information is supplied, ensure that either binary + * configuration data or a configuration string is supplied, but not both. + */ + HDassert(!new_driver_info || !new_driver_config_str); + if (NULL == H5I_object_verify(new_driver_id, H5I_VFL)) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver ID") if (TRUE == H5P_isa_class(plist->plist_id, H5P_FILE_ACCESS)) { - H5FD_driver_prop_t driver_prop; /* Property for driver ID & info */ + H5FD_driver_prop_t driver_prop; /* Property for driver ID, info & config. string */ /* Prepare the driver property */ - driver_prop.driver_id = new_driver_id; - driver_prop.driver_info = new_driver_info; + driver_prop.driver_id = new_driver_id; + driver_prop.driver_info = new_driver_info; + driver_prop.driver_config_str = new_driver_config_str; - /* Set the driver ID & info property */ + /* Set the driver ID, info & config. string property */ if (H5P_set(plist, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set driver ID & info") } /* end if */ @@ -1022,7 +1257,7 @@ H5Pset_driver(hid_t plist_id, hid_t new_driver_id, const void *new_driver_info) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver ID") /* Set the driver */ - if (H5P_set_driver(plist, new_driver_id, new_driver_info) < 0) + if (H5P_set_driver(plist, new_driver_id, new_driver_info, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set driver info") done: @@ -1030,6 +1265,180 @@ done: } /* end H5Pset_driver() */ /*------------------------------------------------------------------------- + * Function: H5P_set_driver_by_name + * + * Purpose: Set the file driver name (DRIVER_NAME) for a file access + * property list (PLIST) and supply an optional string + * containing the driver-specific properties (DRIVER_CONFIG). + * The driver properties string will be copied into the + * property list. + * + * If the file driver specified by DRIVER_NAME is not + * currently registered, an attempt will be made to load the + * driver as a plugin. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5P_set_driver_by_name(H5P_genplist_t *plist, const char *driver_name, const char *driver_config, + hbool_t app_ref) +{ + hid_t new_driver_id = H5I_INVALID_HID; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + assert(plist); + assert(driver_name); + + /* Register the driver */ + if ((new_driver_id = H5FD_register_driver_by_name(driver_name, app_ref)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTREGISTER, FAIL, "unable to register VFD") + + /* Set the driver */ + if (H5P_set_driver(plist, new_driver_id, NULL, driver_config) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set driver info") + +done: + if (ret_value < 0) { + if (new_driver_id >= 0 && H5I_dec_app_ref(new_driver_id) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTDEC, FAIL, "can't decrement count on VFD ID") + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P_set_driver_by_name() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_driver_by_name + * + * Purpose: Set the file driver name (DRIVER_NAME) for a file access + * property list (PLIST_ID) and supply an optional string + * containing the driver-specific properties (DRIVER_CONFIG). + * The driver properties string will be copied into the + * property list. + * + * If the file driver specified by DRIVER_NAME is not + * currently registered, an attempt will be made to load the + * driver as a plugin. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_driver_by_name(hid_t plist_id, const char *driver_name, const char *driver_config) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*s*s", plist_id, driver_name, driver_config); + + /* Check arguments */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(plist_id, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + if (!driver_name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "driver_name parameter cannot be NULL") + if (!*driver_name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "driver_name parameter cannot be an empty string") + + /* Set the driver */ + if (H5P_set_driver_by_name(plist, driver_name, driver_config, TRUE) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set driver info") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_driver_by_name() */ + +/*------------------------------------------------------------------------- + * Function: H5P_set_driver_by_value + * + * Purpose: Set the file driver value (DRIVER_VALUE) for a file access + * property list (PLIST) and supply an optional string + * containing the driver-specific properties (DRIVER_CONFIG). + * The driver properties string will be copied into the + * property list. + * + * If the file driver specified by DRIVER_VALUE is not + * currently registered, an attempt will be made to load the + * driver as a plugin. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5P_set_driver_by_value(H5P_genplist_t *plist, H5FD_class_value_t driver_value, const char *driver_config, + hbool_t app_ref) +{ + hid_t new_driver_id = H5I_INVALID_HID; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + assert(plist); + assert(driver_value >= 0); + + /* Register the driver */ + if ((new_driver_id = H5FD_register_driver_by_value(driver_value, app_ref)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTREGISTER, FAIL, "unable to register VFD") + + /* Set the driver */ + if (H5P_set_driver(plist, new_driver_id, NULL, driver_config) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set driver info") + +done: + if (ret_value < 0) { + if (new_driver_id >= 0 && H5I_dec_app_ref(new_driver_id) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTDEC, FAIL, "can't decrement count on VFD ID") + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P_set_driver_by_value() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_driver_by_value + * + * Purpose: Set the file driver value (DRIVER_VALUE) for a file access + * property list (PLIST_ID) and supply an optional string + * containing the driver-specific properties (DRIVER_CONFIG). + * The driver properties string will be copied into the + * property list. + * + * If the file driver specified by DRIVER_VALUE is not + * currently registered, an attempt will be made to load the + * driver as a plugin. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_driver_by_value(hid_t plist_id, H5FD_class_value_t driver_value, const char *driver_config) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "iDV*s", plist_id, driver_value, driver_config); + + /* Check arguments */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(plist_id, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + if (driver_value < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "negative VFD value is disallowed") + + /* Set the driver */ + if (H5P_set_driver_by_value(plist, driver_value, driver_config, TRUE) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set driver info") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_driver_by_value() */ + +/*------------------------------------------------------------------------- * Function: H5P_peek_driver * * Purpose: Return the ID of the low-level file driver. PLIST_ID should @@ -1056,7 +1465,7 @@ H5P_peek_driver(H5P_genplist_t *plist) /* Get the current driver ID */ if (TRUE == H5P_isa_class(plist->plist_id, H5P_FILE_ACCESS)) { - H5FD_driver_prop_t driver_prop; /* Property for driver ID & info */ + H5FD_driver_prop_t driver_prop; /* Property for driver ID, info & configuration string */ if (H5P_peek(plist, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get driver ID") @@ -1140,7 +1549,7 @@ H5P_peek_driver_info(H5P_genplist_t *plist) /* Get the current driver info */ if (TRUE == H5P_isa_class(plist->plist_id, H5P_FILE_ACCESS)) { - H5FD_driver_prop_t driver_prop; /* Property for driver ID & info */ + H5FD_driver_prop_t driver_prop; /* Property for driver ID, info & configuration string */ if (H5P_peek(plist, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get driver info") @@ -1193,13 +1602,104 @@ done: } /* end H5Pget_driver_info() */ /*------------------------------------------------------------------------- + * Function: H5P_peek_driver_config_str + * + * Purpose: Returns a pointer directly to the file driver configuration + * string of a file access property list. + * + * Return: Success: Ptr to *uncopied* driver configuration string, if + * any. + * + * Failure: NULL. NULL is also returned if the driver has not + * been configured with a driver configuration string. + * + *------------------------------------------------------------------------- + */ +const char * +H5P_peek_driver_config_str(H5P_genplist_t *plist) +{ + const char *ret_value = NULL; /* Return value */ + + FUNC_ENTER_NOAPI(NULL) + + /* Get the current driver configuration string */ + if (TRUE == H5P_isa_class(plist->plist_id, H5P_FILE_ACCESS)) { + H5FD_driver_prop_t driver_prop; /* Property for driver ID, info & configuration string */ + + if (H5P_peek(plist, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get driver configuration string") + ret_value = driver_prop.driver_config_str; + } /* end if */ + else + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, NULL, "not a file access property list") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P_peek_driver_config_str() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_driver_config_str + * + * Purpose: Retrieves a string representation of the configuration for + * the driver set on the given FAPL. The returned string can + * be used to configure the same driver in an identical way. + * + * `config_buf` may be NULL, in which case the length of the + * driver configuration string is simply returned. The caller + * can then allocate a buffer of the appropriate size and call + * this routine again. + * + * Return: Length of the driver configuration string on success (not + * including the NUL terminator) + * Negative on failure + * + *------------------------------------------------------------------------- + */ +ssize_t +H5Pget_driver_config_str(hid_t fapl_id, char *config_buf, size_t buf_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + const char * config_str = NULL; + ssize_t ret_value = -1; + + FUNC_ENTER_API((-1)) + H5TRACE3("Zs", "i*sz", fapl_id, config_buf, buf_size); + + /* Check arguments */ + if (!config_buf && buf_size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, (-1), "config_buf cannot be NULL if buf_size is non-zero") + + /* Get the plist structure */ + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ID, H5E_BADID, (-1), "can't find object for ID") + + /* Retrieve configuration string property */ + if ((config_str = H5P_peek_driver_config_str(plist))) { + size_t config_str_len = HDstrlen(config_str); + + if (config_buf) { + HDstrncpy(config_buf, config_str, MIN(config_str_len + 1, buf_size)); + if (config_str_len >= buf_size) + config_buf[buf_size - 1] = '\0'; + } + + ret_value = (ssize_t)config_str_len; + } + else + ret_value = 0; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_driver_config_str() */ + +/*------------------------------------------------------------------------- * Function: H5P__file_driver_copy * * Purpose: Copy file driver ID & info. * * Note: This is an "in-place" copy, since this routine gets called * after the top-level copy has been performed and this routine - * finishes the "deep" part of the copy. + * finishes the "deep" part of the copy. * * Return: Success: Non-negative * Failure: Negative @@ -1250,6 +1750,15 @@ H5P__file_driver_copy(void *value) /* Set the driver info for the copy */ info->driver_info = new_pl; } /* end if */ + + /* Copy driver configuration string, if it exists */ + if (info->driver_config_str) { + char *new_config_str = NULL; + + if (NULL == (new_config_str = H5MM_strdup(info->driver_config_str))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "driver configuration string copy failed") + info->driver_config_str = new_config_str; + } /* end if */ } /* end if */ } /* end if */ @@ -1280,7 +1789,6 @@ H5P__file_driver_free(void *value) if (value) { H5FD_driver_prop_t *info = (H5FD_driver_prop_t *)value; /* Driver ID & info struct */ - /* Copy the driver & info, if there is one */ if (info->driver_id > 0) { /* Free the driver info, if it exists */ @@ -1288,6 +1796,9 @@ H5P__file_driver_free(void *value) if (H5FD_free_driver_info(info->driver_id, info->driver_info) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "driver info free request failed") + /* Free the driver configuration string, if it exists */ + H5MM_xfree_const(info->driver_config_str); + /* Decrement reference count for driver */ if (H5I_dec_ref(info->driver_id) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTDEC, FAIL, "can't decrement reference count for driver ID") @@ -1507,6 +2018,16 @@ H5P__facc_file_driver_cmp(const void *_info1, const void *_info2, size_t H5_ATTR HGOTO_DONE(cmp_value); } /* end if */ + /* Compare driver configuration strings */ + if (info1->driver_config_str == NULL && info2->driver_config_str != NULL) + HGOTO_DONE(-1); + if (info1->driver_config_str != NULL && info2->driver_config_str == NULL) + HGOTO_DONE(1); + if (info1->driver_config_str) { + if (0 != (cmp_value = HDstrcmp(info1->driver_config_str, info2->driver_config_str))) + HGOTO_DONE(cmp_value); + } + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_file_driver_cmp() */ @@ -2390,7 +2911,7 @@ done: * * Currently, the only two valid combinations for this routine are: * LOW = H5F_FORMAT_EARLIEST and HIGH = H5F_FORMAT_LATEST (the default - * setting, which creates objects with the ealiest version possible for + * setting, which creates objects with the earliest version possible for * each object, but no upper limit on the version allowed to be created if * a newer version of an object's format is required to support a feature * requested with an HDF5 library API routine), and LOW = H5F_FORMAT_LATEST @@ -2406,7 +2927,7 @@ done: * Note: Eventually we want to add more values to the H5F_libver_t * enumerated type that indicate library release values where the file * format was changed (like "H5F_FORMAT_1_2_0" for the file format changes - * in the 1.2.x release branch and possily even "H5F_FORMAT_1_4_2" for + * in the 1.2.x release branch and possibly even "H5F_FORMAT_1_4_2" for * a change mid-way through the 1.4.x release branch, etc). * * Adding more values will allow applications to make settings like the @@ -2852,7 +3373,7 @@ H5Pset_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callback HDassert(callbacks_ptr->udata_copy); HDassert(callbacks_ptr->udata_free); if ((info.callbacks.udata = callbacks_ptr->udata_copy(callbacks_ptr->udata)) == NULL) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't copy the suppplied udata") + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't copy the supplied udata") } /* end if */ /* Set values */ @@ -4916,15 +5437,14 @@ H5P__decode_coll_md_read_flag_t(const void **_pp, void *_value) * Function: H5Pset_all_coll_metadata_ops * * Purpose: Tell the library whether the metadata read operations will - * be done collectively (1) or not (0). Default is independent. - * With collective mode, the library will optimize access to - * metadata operations on the file. + * be done collectively (1) or not (0). Default is independent. + * With collective mode, the library will optimize access to + * metadata operations on the file. * * Note: This routine accepts file access property lists, link - * access property lists, attribute access property lists, - * dataset access property lists, group access property lists, - * named datatype access property lists, - * and dataset transfer property lists. + * access property lists, attribute access property lists, + * dataset access property lists, group access property lists + * and named datatype access property lists. * * Return: Non-negative on success/Negative on failure * @@ -4944,7 +5464,7 @@ H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective) H5TRACE2("e", "ib", plist_id, is_collective); /* Compare the property list's class against the other class */ - /* (Dataset, group, attribute, and named datype access property lists + /* (Dataset, group, attribute, and named datatype access property lists * are sub-classes of link access property lists -QAK) */ if (TRUE != H5P_isa_class(plist_id, H5P_LINK_ACCESS) && TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS)) @@ -4974,10 +5494,9 @@ done: * Purpose: Gets information about collective metadata read mode. * * Note: This routine accepts file access property lists, link - * access property lists, attribute access property lists, - * dataset access property lists, group access property lists, - * named datatype access property lists, - * and dataset transfer property lists. + * access property lists, attribute access property lists, + * dataset access property lists, group access property lists, + * and named datatype access property lists. * * Return: Non-negative on success/Negative on failure * @@ -4995,7 +5514,7 @@ H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective /*out*/) H5TRACE2("e", "ix", plist_id, is_collective); /* Compare the property list's class against the other class */ - /* (Dataset, group, attribute, and named datype access property lists + /* (Dataset, group, attribute, and named datatype access property lists * are sub-classes of link access property lists -QAK) */ if (TRUE != H5P_isa_class(plist_id, H5P_LINK_ACCESS) && TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS)) @@ -5764,7 +6283,7 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr) if (H5P_set(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, config_ptr) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set metadata cache initial config") - if (H5P_set_driver(plist, H5FD_VFD_SWMR, NULL) < 0) + if (H5P_set_driver(plist, H5FD_VFD_SWMR, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set VFD SWMR driver info"); done: diff --git a/src/H5Pint.c b/src/H5Pint.c index 90fd1c2..c0f5ee1 100644 --- a/src/H5Pint.c +++ b/src/H5Pint.c @@ -123,77 +123,78 @@ static herr_t H5P__free_del_name_cb(void *item, void H5_ATTR_UNUSED *key, void H /* * Predefined property list classes. These are initialized at runtime by - * H5P__init_package() in this source file. + * H5P_init() in this source file. */ -hid_t H5P_CLS_ROOT_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_ROOT_g = NULL; -hid_t H5P_CLS_OBJECT_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_OBJECT_CREATE_g = NULL; -hid_t H5P_CLS_FILE_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_FILE_CREATE_g = NULL; -hid_t H5P_CLS_FILE_ACCESS_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_FILE_ACCESS_g = NULL; -hid_t H5P_CLS_DATASET_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_DATASET_CREATE_g = NULL; +hid_t H5P_CLS_ROOT_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_ROOT_g = NULL; + +hid_t H5P_CLS_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_ATTRIBUTE_ACCESS_g = NULL; +hid_t H5P_CLS_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_ATTRIBUTE_CREATE_g = NULL; hid_t H5P_CLS_DATASET_ACCESS_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_DATASET_ACCESS_g = NULL; +hid_t H5P_CLS_DATASET_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_DATASET_CREATE_g = NULL; hid_t H5P_CLS_DATASET_XFER_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_DATASET_XFER_g = NULL; +hid_t H5P_CLS_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_DATATYPE_ACCESS_g = NULL; +hid_t H5P_CLS_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_DATATYPE_CREATE_g = NULL; +hid_t H5P_CLS_FILE_ACCESS_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_FILE_ACCESS_g = NULL; +hid_t H5P_CLS_FILE_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_FILE_CREATE_g = NULL; hid_t H5P_CLS_FILE_MOUNT_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_FILE_MOUNT_g = NULL; -hid_t H5P_CLS_GROUP_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_GROUP_CREATE_g = NULL; hid_t H5P_CLS_GROUP_ACCESS_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_GROUP_ACCESS_g = NULL; -hid_t H5P_CLS_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_DATATYPE_CREATE_g = NULL; -hid_t H5P_CLS_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_DATATYPE_ACCESS_g = NULL; -hid_t H5P_CLS_MAP_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_MAP_CREATE_g = NULL; +hid_t H5P_CLS_GROUP_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_GROUP_CREATE_g = NULL; +hid_t H5P_CLS_LINK_ACCESS_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_LINK_ACCESS_g = NULL; +hid_t H5P_CLS_LINK_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_LINK_CREATE_g = NULL; hid_t H5P_CLS_MAP_ACCESS_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_MAP_ACCESS_g = NULL; -hid_t H5P_CLS_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_ATTRIBUTE_CREATE_g = NULL; -hid_t H5P_CLS_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_ATTRIBUTE_ACCESS_g = NULL; +hid_t H5P_CLS_MAP_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_MAP_CREATE_g = NULL; hid_t H5P_CLS_OBJECT_COPY_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_OBJECT_COPY_g = NULL; -hid_t H5P_CLS_LINK_CREATE_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_LINK_CREATE_g = NULL; -hid_t H5P_CLS_LINK_ACCESS_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_LINK_ACCESS_g = NULL; +hid_t H5P_CLS_OBJECT_CREATE_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_OBJECT_CREATE_g = NULL; +hid_t H5P_CLS_REFERENCE_ACCESS_ID_g = H5I_INVALID_HID; +H5P_genclass_t *H5P_CLS_REFERENCE_ACCESS_g = NULL; hid_t H5P_CLS_STRING_CREATE_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_STRING_CREATE_g = NULL; hid_t H5P_CLS_VOL_INITIALIZE_ID_g = H5I_INVALID_HID; H5P_genclass_t *H5P_CLS_VOL_INITIALIZE_g = NULL; -hid_t H5P_CLS_REFERENCE_ACCESS_ID_g = H5I_INVALID_HID; -H5P_genclass_t *H5P_CLS_REFERENCE_ACCESS_g = NULL; /* * Predefined property lists for each predefined class. These are initialized - * at runtime by H5P__init_package() in this source file. + * at runtime by H5P_init() in this source file. */ -hid_t H5P_LST_FILE_CREATE_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_FILE_ACCESS_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_FILE_ACCESS_ANY_VFD_g = H5I_INVALID_HID; -hid_t H5P_LST_DATASET_CREATE_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_DATASET_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_DATASET_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_DATASET_XFER_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_FILE_ACCESS_ANY_VFD_g = H5I_INVALID_HID; +hid_t H5P_LST_FILE_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_FILE_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_FILE_MOUNT_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_GROUP_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_GROUP_ACCESS_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_MAP_CREATE_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_GROUP_CREATE_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_LINK_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_LINK_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_MAP_ACCESS_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_MAP_CREATE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_OBJECT_COPY_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_LINK_CREATE_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_LINK_ACCESS_ID_g = H5I_INVALID_HID; -hid_t H5P_LST_VOL_INITIALIZE_ID_g = H5I_INVALID_HID; hid_t H5P_LST_REFERENCE_ACCESS_ID_g = H5I_INVALID_HID; +hid_t H5P_LST_VOL_INITIALIZE_ID_g = H5I_INVALID_HID; /* Root property list class library initialization object */ const H5P_libclass_t H5P_CLS_ROOT[1] = {{ @@ -413,48 +414,29 @@ static const H5I_class_t H5I_GENPROPLST_CLS[1] = {{ }}; /*------------------------------------------------------------------------- - * Function: H5P_init + * Function: H5P_init_phase1 * - * Purpose: Initialize the interface from some other layer. + * Purpose: Initialize the interface from some other layer. This should + * be followed with a call to H5P_init_phase2 after the H5P + * interface is completely setup. * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Saturday, March 4, 2000 * *------------------------------------------------------------------------- */ herr_t -H5P_init(void) +H5P_init_phase1(void) { + size_t tot_init = 0; /* Total # of classes initialized */ + size_t pass_init; /* # of classes initialized in each pass */ + size_t u; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5P_init() */ - -/*-------------------------------------------------------------------------- -NAME - H5P__init_package -- Initialize interface-specific information -USAGE - herr_t H5P__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. ---------------------------------------------------------------------------*/ -herr_t -H5P__init_package(void) -{ - size_t tot_init; /* Total # of classes initialized */ - size_t pass_init; /* # of classes initialized in each pass */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE /* Sanity check */ HDcompile_assert(H5P_TYPE_REFERENCE_ACCESS == (H5P_TYPE_MAX_TYPE - 1)); @@ -473,8 +455,6 @@ H5P__init_package(void) */ tot_init = 0; do { - size_t u; /* Local index variable */ - /* Reset pass initialization counter */ pass_init = 0; @@ -524,8 +504,59 @@ H5P__init_package(void) HDassert(tot_init == NELMTS(init_class)); done: + if (ret_value < 0 && tot_init > 0) { + /* First uninitialize all default property lists */ + H5I_clear_type(H5I_GENPROP_LST, FALSE, FALSE); + + /* Then uninitialize any initialized libclass */ + for (u = 0; u < NELMTS(init_class); u++) { + H5P_libclass_t const *lib_class = init_class[u]; /* Current class to operate on */ + + HDassert(lib_class->class_id); + if (*lib_class->class_id >= 0) { + /* Close the class ID */ + if (H5I_dec_ref(*lib_class->class_id) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CLOSEERROR, FAIL, "unable to close property list class ID") + } + else if (lib_class->pclass && *lib_class->pclass) { + /* Close a half-initialized pclass */ + if (H5P__close_class(*lib_class->pclass) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CLOSEERROR, FAIL, "unable to close property list class") + } + } + } + FUNC_LEAVE_NOAPI(ret_value) -} /* end H5P__init_package() */ +} + +/*------------------------------------------------------------------------- + * Function: H5P_init_phase2 + * + * Purpose: Finish initializing the interface from some other package. + * + * Note: This is broken out as a separate routine so that the + * library's default VFL driver can be chosen and initialized + * after the entire H5P interface has been initialized. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +herr_t +H5P_init_phase2(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Set up the default VFL driver */ + if (H5P__facc_set_def_driver() < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "unable to set default VFL driver") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P_init_phase2() */ /*-------------------------------------------------------------------------- NAME @@ -551,77 +582,109 @@ H5P_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - int64_t nlist, nclass; - - /* Destroy HDF5 library property classes & lists */ - - /* Check if there are any open property list classes or lists */ - nclass = H5I_nmembers(H5I_GENPROP_CLS); - nlist = H5I_nmembers(H5I_GENPROP_LST); - - /* If there are any open classes or groups, attempt to get rid of them. */ - if ((nclass + nlist) > 0) { - /* Clear the lists */ - if (nlist > 0) { - (void)H5I_clear_type(H5I_GENPROP_LST, FALSE, FALSE); - - /* Reset the default property lists, if they've been closed */ - if (H5I_nmembers(H5I_GENPROP_LST) == 0) { - H5P_LST_FILE_CREATE_ID_g = H5P_LST_FILE_ACCESS_ID_g = H5P_LST_DATASET_CREATE_ID_g = - H5P_LST_DATASET_ACCESS_ID_g = H5P_LST_DATASET_XFER_ID_g = H5P_LST_GROUP_CREATE_ID_g = - H5P_LST_GROUP_ACCESS_ID_g = H5P_LST_DATATYPE_CREATE_ID_g = - H5P_LST_DATATYPE_ACCESS_ID_g = H5P_LST_MAP_CREATE_ID_g = - H5P_LST_MAP_ACCESS_ID_g = H5P_LST_ATTRIBUTE_CREATE_ID_g = - H5P_LST_ATTRIBUTE_ACCESS_ID_g = H5P_LST_OBJECT_COPY_ID_g = - H5P_LST_LINK_CREATE_ID_g = H5P_LST_LINK_ACCESS_ID_g = - H5P_LST_VOL_INITIALIZE_ID_g = H5P_LST_REFERENCE_ACCESS_ID_g = - H5P_LST_FILE_MOUNT_ID_g = H5I_INVALID_HID; - } /* end if */ - } /* end if */ - - /* Only attempt to close the classes after all the lists are closed */ - if (nlist == 0 && nclass > 0) { - (void)H5I_clear_type(H5I_GENPROP_CLS, FALSE, FALSE); - - /* Reset the default property classes, if they've been closed */ - if (H5I_nmembers(H5I_GENPROP_CLS) == 0) { - H5P_CLS_ROOT_g = H5P_CLS_OBJECT_CREATE_g = H5P_CLS_FILE_CREATE_g = H5P_CLS_FILE_ACCESS_g = - H5P_CLS_DATASET_CREATE_g = H5P_CLS_DATASET_ACCESS_g = H5P_CLS_DATASET_XFER_g = - H5P_CLS_GROUP_CREATE_g = H5P_CLS_GROUP_ACCESS_g = H5P_CLS_DATATYPE_CREATE_g = - H5P_CLS_DATATYPE_ACCESS_g = H5P_CLS_MAP_CREATE_g = H5P_CLS_MAP_ACCESS_g = - H5P_CLS_STRING_CREATE_g = H5P_CLS_ATTRIBUTE_CREATE_g = - H5P_CLS_ATTRIBUTE_ACCESS_g = H5P_CLS_OBJECT_COPY_g = - H5P_CLS_LINK_CREATE_g = H5P_CLS_LINK_ACCESS_g = - H5P_CLS_VOL_INITIALIZE_g = H5P_CLS_REFERENCE_ACCESS_g = - H5P_CLS_FILE_MOUNT_g = NULL; - - H5P_CLS_ROOT_ID_g = H5P_CLS_OBJECT_CREATE_ID_g = H5P_CLS_FILE_CREATE_ID_g = - H5P_CLS_FILE_ACCESS_ID_g = H5P_CLS_DATASET_CREATE_ID_g = H5P_CLS_DATASET_ACCESS_ID_g = - H5P_CLS_DATASET_XFER_ID_g = H5P_CLS_GROUP_CREATE_ID_g = - H5P_CLS_GROUP_ACCESS_ID_g = H5P_CLS_DATATYPE_CREATE_ID_g = - H5P_CLS_DATATYPE_ACCESS_ID_g = H5P_CLS_MAP_CREATE_ID_g = - H5P_CLS_MAP_ACCESS_ID_g = H5P_CLS_STRING_CREATE_ID_g = - H5P_CLS_ATTRIBUTE_CREATE_ID_g = H5P_CLS_ATTRIBUTE_ACCESS_ID_g = - H5P_CLS_OBJECT_COPY_ID_g = H5P_CLS_LINK_CREATE_ID_g = - H5P_CLS_LINK_ACCESS_ID_g = H5P_CLS_VOL_INITIALIZE_ID_g = - H5P_CLS_REFERENCE_ACCESS_ID_g = - H5P_CLS_FILE_MOUNT_ID_g = H5I_INVALID_HID; - } /* end if */ - } /* end if */ + int64_t nlist, nclass; + + /* Destroy HDF5 library property classes & lists */ + + /* Check if there are any open property list classes or lists */ + nclass = H5I_nmembers(H5I_GENPROP_CLS); + nlist = H5I_nmembers(H5I_GENPROP_LST); + + /* If there are any open classes or groups, attempt to get rid of them. */ + if ((nclass + nlist) > 0) { + /* Clear the lists */ + if (nlist > 0) { + (void)H5I_clear_type(H5I_GENPROP_LST, FALSE, FALSE); + + /* Reset the default property lists, if they've been closed */ + if (H5I_nmembers(H5I_GENPROP_LST) == 0) { + H5P_LST_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_DATASET_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_DATASET_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_DATASET_XFER_ID_g = H5I_INVALID_HID; + H5P_LST_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_FILE_ACCESS_ANY_VFD_g = H5I_INVALID_HID; + H5P_LST_FILE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_FILE_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_FILE_MOUNT_ID_g = H5I_INVALID_HID; + H5P_LST_GROUP_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_GROUP_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_LINK_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_LINK_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_MAP_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_MAP_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_OBJECT_COPY_ID_g = H5I_INVALID_HID; + H5P_LST_REFERENCE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_LST_VOL_INITIALIZE_ID_g = H5I_INVALID_HID; + } + } - n++; /*H5I*/ + /* Only attempt to close the classes after all the lists are closed */ + if (nlist == 0 && nclass > 0) { + (void)H5I_clear_type(H5I_GENPROP_CLS, FALSE, FALSE); + + /* Reset the default property classes and IDs if they've been closed */ + if (H5I_nmembers(H5I_GENPROP_CLS) == 0) { + H5P_CLS_ROOT_g = NULL; + + H5P_CLS_ATTRIBUTE_ACCESS_g = NULL; + H5P_CLS_ATTRIBUTE_CREATE_g = NULL; + H5P_CLS_DATASET_ACCESS_g = NULL; + H5P_CLS_DATASET_CREATE_g = NULL; + H5P_CLS_DATASET_XFER_g = NULL; + H5P_CLS_DATATYPE_ACCESS_g = NULL; + H5P_CLS_DATATYPE_CREATE_g = NULL; + H5P_CLS_FILE_ACCESS_g = NULL; + H5P_CLS_FILE_CREATE_g = NULL; + H5P_CLS_FILE_MOUNT_g = NULL; + H5P_CLS_GROUP_ACCESS_g = NULL; + H5P_CLS_GROUP_CREATE_g = NULL; + H5P_CLS_LINK_ACCESS_g = NULL; + H5P_CLS_LINK_CREATE_g = NULL; + H5P_CLS_MAP_ACCESS_g = NULL; + H5P_CLS_MAP_CREATE_g = NULL; + H5P_CLS_OBJECT_COPY_g = NULL; + H5P_CLS_OBJECT_CREATE_g = NULL; + H5P_CLS_REFERENCE_ACCESS_g = NULL; + H5P_CLS_STRING_CREATE_g = NULL; + H5P_CLS_VOL_INITIALIZE_g = NULL; + + H5P_CLS_ROOT_ID_g = H5I_INVALID_HID; + + H5P_CLS_ATTRIBUTE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_ATTRIBUTE_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_DATASET_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_DATASET_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_DATASET_XFER_ID_g = H5I_INVALID_HID; + H5P_CLS_DATATYPE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_DATATYPE_CREATE_ID_g = H5I_INVALID_HID; + H5P_LST_FILE_ACCESS_ANY_VFD_g = H5I_INVALID_HID; + H5P_CLS_FILE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_FILE_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_FILE_MOUNT_ID_g = H5I_INVALID_HID; + H5P_CLS_GROUP_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_GROUP_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_LINK_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_LINK_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_MAP_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_MAP_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_OBJECT_COPY_ID_g = H5I_INVALID_HID; + H5P_CLS_OBJECT_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_REFERENCE_ACCESS_ID_g = H5I_INVALID_HID; + H5P_CLS_STRING_CREATE_ID_g = H5I_INVALID_HID; + H5P_CLS_VOL_INITIALIZE_ID_g = H5I_INVALID_HID; + } } - else { - /* Destroy the property list and class id groups */ - n += (H5I_dec_type_ref(H5I_GENPROP_LST) > 0); - n += (H5I_dec_type_ref(H5I_GENPROP_CLS) > 0); - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ + n++; /*H5I*/ + } + else { + /* Destroy the property list and class id groups */ + n += (H5I_dec_type_ref(H5I_GENPROP_LST) > 0); + n += (H5I_dec_type_ref(H5I_GENPROP_CLS) > 0); + } /* end else */ FUNC_LEAVE_NOAPI(n) } /* end H5P_term_package() */ @@ -3243,7 +3306,7 @@ done: NAME H5P_exist_plist PURPOSE - Internal routine to query the existance of a property in a property list. + Internal routine to query the existence of a property in a property list. USAGE htri_t H5P_exist_plist(plist, name) const H5P_genplist_t *plist; IN: Property list to check @@ -3302,7 +3365,7 @@ done: NAME H5P__exist_pclass PURPOSE - Internal routine to query the existance of a property in a property class. + Internal routine to query the existence of a property in a property class. USAGE herr_t H5P__exist_pclass(pclass, name) H5P_genclass_t *pclass; IN: Property class to check @@ -3502,7 +3565,7 @@ H5P_get_nprops_pclass(const H5P_genclass_t *pclass, size_t *nprops, hbool_t recu { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(pclass); HDassert(nprops); @@ -3517,7 +3580,6 @@ H5P_get_nprops_pclass(const H5P_genclass_t *pclass, size_t *nprops, hbool_t recu *nprops += pclass->nprops; } /* end while */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* H5P_get_nprops_pclass() */ @@ -3818,7 +3880,7 @@ H5P__cmp_plist_cb(H5P_genprop_t *prop, void *_udata) /* Check if the property exists in the second property list */ if ((prop2_exist = H5P_exist_plist(udata->plist2, prop->name)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_NOTFOUND, H5_ITER_ERROR, "can't lookup existance of property?") + HGOTO_ERROR(H5E_PLIST, H5E_NOTFOUND, H5_ITER_ERROR, "can't lookup existence of property?") if (prop2_exist) { const H5P_genprop_t *prop2; /* Pointer to property in second plist */ @@ -3948,7 +4010,7 @@ H5P_class_isa(const H5P_genclass_t *pclass1, const H5P_genclass_t *pclass2) { htri_t ret_value = FAIL; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(pclass1); HDassert(pclass2); @@ -4769,7 +4831,7 @@ done: Returns non-negative on success, negative on failure. DESCRIPTION Removes a property from a property list. Both properties which were - in existance when the property list was created (i.e. properties registered + in existence when the property list was created (i.e. properties registered with H5Pregister2) and properties added to the list after it was created (i.e. added with H5Pinsert2) may be removed from a property list. Properties do not need to be removed a property list before the list itself @@ -4850,7 +4912,7 @@ H5P__copy_prop_plist(hid_t dst_id, hid_t src_id, const char *name) NULL == (dst_plist = (H5P_genplist_t *)H5I_object(dst_id))) HGOTO_ERROR(H5E_PLIST, H5E_NOTFOUND, FAIL, "property object doesn't exist") - /* If the property exists in the destination alread */ + /* If the property exists in the destination already */ if (NULL != H5P__find_prop_plist(dst_plist, name)) { /* Delete the property from the destination list, calling the 'close' callback if necessary */ if (H5P_remove(dst_plist, name) < 0) @@ -5250,14 +5312,13 @@ H5P_get_class_name(H5P_genclass_t *pclass) { char *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI(NULL) + FUNC_ENTER_NOAPI_NOERR HDassert(pclass); /* Get class name */ ret_value = H5MM_xstrdup(pclass->name); -done: FUNC_LEAVE_NOAPI(ret_value) } /* H5P_get_class_name() */ diff --git a/src/H5Pmodule.h b/src/H5Pmodule.h index 6e92e66..66a9574 100644 --- a/src/H5Pmodule.h +++ b/src/H5Pmodule.h @@ -111,7 +111,8 @@ * * \defgroup GAPL General Access Properties * \ingroup H5P - * \todo Should this be as standalone page? + * The functions in this section can be applied to different kinds of property + * lists. * * \defgroup GCPL Group Creation Properties * \ingroup H5P diff --git a/src/H5Pocpl.c b/src/H5Pocpl.c index e442030..06d9f85 100644 --- a/src/H5Pocpl.c +++ b/src/H5Pocpl.c @@ -482,7 +482,7 @@ done: * transfer property list. The FLAGS argument specifies certain * general properties of the filter and is documented below. * The CD_VALUES is an array of CD_NELMTS integers which are - * auxiliary data for the filter. The integer vlues will be + * auxiliary data for the filter. The integer values will be * stored in the dataset object header as part of the filter * information. * @@ -543,7 +543,7 @@ done: * transfer property list. The FLAGS argument specifies certain * general properties of the filter and is documented below. * The CD_VALUES is an array of CD_NELMTS integers which are - * auxiliary data for the filter. The integer vlues will be + * auxiliary data for the filter. The integer values will be * stored in the dataset object header as part of the filter * information. * @@ -609,7 +609,7 @@ done: * creation property list. The FLAGS argument specifies certain * general properties of the filter and is documented below. * The CD_VALUES is an array of CD_NELMTS integers which are - * auxiliary data for the filter. The integer vlues will be + * auxiliary data for the filter. The integer values will be * stored in the dataset object header as part of the filter * information. * @@ -671,7 +671,7 @@ done: * creation property list. The FLAGS argument specifies certain * general properties of the filter and is documented below. * The CD_VALUES is an array of CD_NELMTS integers which are - * auxiliary data for the filter. The integer vlues will be + * auxiliary data for the filter. The integer values will be * stored in the dataset object header as part of the filter * information. * @@ -782,7 +782,7 @@ done: * CD_NELMTS indicates the number of entries in the CD_VALUES * array allocated by the caller while on exit it contains the * number of values defined by the filter. FILTER_CONFIG is a bit - * field contaning encode/decode flags from H5Zpublic.h. The IDX + * field containing encode/decode flags from H5Zpublic.h. The IDX * should be a value between zero and N-1 as described for * H5Pget_nfilters() and the function will return failure if the * filter number is out of range. @@ -867,7 +867,7 @@ done: * CD_NELMTS indicates the number of entries in the CD_VALUES * array allocated by the caller while on exit it contains the * number of values defined by the filter. FILTER_CONFIG is a bit - * field contaning encode/decode flags from H5Zpublic.h. The ID + * field containing encode/decode flags from H5Zpublic.h. The ID * should be the filter ID to retrieve the parameters for. If the * filter is not set for the property list, an error will be returned. * @@ -916,7 +916,7 @@ done: * CD_NELMTS indicates the number of entries in the CD_VALUES * array allocated by the caller while on exit it contains the * number of values defined by the filter. FILTER_CONFIG is a bit - * field contaning encode/decode flags from H5Zpublic.h. The ID + * field containing encode/decode flags from H5Zpublic.h. The ID * should be the filter ID to retrieve the parameters for. If the * filter is not set for the property list, an error will be returned. * diff --git a/src/H5Ppkg.h b/src/H5Ppkg.h index dd43f76..2946931 100644 --- a/src/H5Ppkg.h +++ b/src/H5Ppkg.h @@ -188,6 +188,9 @@ H5_DLL herr_t H5P__decode_double(const void **_pp, void *value); H5_DLL herr_t H5P__encode_coll_md_read_flag_t(const void *value, void **_pp, size_t *size); H5_DLL herr_t H5P__decode_coll_md_read_flag_t(const void **_pp, void *value); +/* Private FAPL routines */ +H5_DLL herr_t H5P__facc_set_def_driver(void); + /* Private OCPL routines */ H5_DLL herr_t H5P__get_filter(const struct H5Z_filter_info_t *filter, unsigned int *flags, size_t *cd_nelmts, unsigned cd_values[], size_t namelen, char name[], unsigned *filter_config); diff --git a/src/H5Pprivate.h b/src/H5Pprivate.h index 6ebe5d3..7cbb397 100644 --- a/src/H5Pprivate.h +++ b/src/H5Pprivate.h @@ -156,8 +156,9 @@ struct H5O_fill_t; struct H5T_t; struct H5VL_connector_prop_t; -/* Package initialization routine */ -H5_DLL herr_t H5P_init(void); +/* Package initialization routines */ +H5_DLL herr_t H5P_init_phase1(void); +H5_DLL herr_t H5P_init_phase2(void); /* Internal versions of API routines */ H5_DLL herr_t H5P_close(H5P_genplist_t *plist); @@ -181,7 +182,13 @@ H5_DLL char * H5P_get_class_name(H5P_genclass_t *pclass); H5_DLL herr_t H5P_get_nprops_pclass(const H5P_genclass_t *pclass, size_t *nprops, hbool_t recurse); H5_DLL hid_t H5P_peek_driver(H5P_genplist_t *plist); H5_DLL const void *H5P_peek_driver_info(H5P_genplist_t *plist); -H5_DLL herr_t H5P_set_driver(H5P_genplist_t *plist, hid_t new_driver_id, const void *new_driver_info); +H5_DLL const char *H5P_peek_driver_config_str(H5P_genplist_t *plist); +H5_DLL herr_t H5P_set_driver(H5P_genplist_t *plist, hid_t new_driver_id, const void *new_driver_info, + const char *new_driver_config_str); +H5_DLL herr_t H5P_set_driver_by_name(H5P_genplist_t *plist, const char *driver_name, + const char *driver_config, hbool_t app_ref); +H5_DLL herr_t H5P_set_driver_by_value(H5P_genplist_t *plist, H5FD_class_value_t driver_value, + const char *driver_config, hbool_t app_ref); H5_DLL herr_t H5P_set_vol(H5P_genplist_t *plist, hid_t vol_id, const void *vol_info); H5_DLL herr_t H5P_reset_vol_class(const H5P_genclass_t *pclass, const struct H5VL_connector_prop_t *vol_prop); H5_DLL herr_t H5P_set_vlen_mem_manager(H5P_genplist_t *plist, H5MM_allocate_t alloc_func, void *alloc_info, diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index ed61646..ed9db10 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -117,21 +117,70 @@ extern "C" { /* Define property list class callback function pointer types */ //! /** - * \todo Document me! + * \brief Callback function for H5Pcreate_class() + * + * \param[in] prop_id The identifier of the property list class being created + * \param[in] create_data User pointer to any class creation data required + * \return \herr_t + * + * \details This function is called when a new property list of the class + * with which this function was registered is being created. The + * function is called after any registered parent create function is + * called for each property value. + * + * If the create function returns a negative value, the new list is not + * returned to the user and the property list creation routine returns + * an error value. + * + * \since 1.4.0 + * */ typedef herr_t (*H5P_cls_create_func_t)(hid_t prop_id, void *create_data); //! //! /** - * \todo Document me! + * \brief Callback function for H5Pcreate_class() + * + * \param[in] new_prop_id The identifier of the property list copy + * \param[in] old_prop_id The identifier of the property list being copied + * \param[in] copy_data User pointer to any copy data required + * \return \herr_t + * + * \details This function is called when an existing property list of this + * class is copied. The copy callback function is called after any + * registered parent copy callback function is called for each property + * value. + * + * If the copy routine returns a negative value, the new list is not + * returned to the user and the property list copy function returns an + * error value. + * + * \since 1.4.0 + * */ typedef herr_t (*H5P_cls_copy_func_t)(hid_t new_prop_id, hid_t old_prop_id, void *copy_data); //! //! /** - * \todo Document me! + * \brief Callback function for H5Pcreate_class() + * + * \param[in] prop_id The identifier of the property list class being created + * \param[in] close_data User pointer to any close data required + * \return \herr_t + * + * \details This function is called when a property list of the class + * with which this function was registered is being closed. The + * function is called after any registered parent close function is + * called for each property value. + * + * If the close function returns a negative value, the new list is not + * returned to the user and the property list close routine returns + * an error value. + * + * \since 1.4.0 + * */ typedef herr_t (*H5P_cls_close_func_t)(hid_t prop_id, void *close_data); //! @@ -146,8 +195,8 @@ typedef herr_t (*H5P_cls_close_func_t)(hid_t prop_id, void *close_data); * \param[in,out] value The value for the property * \return \herr_t * - * \details The H5P_prp_cb1_t() describes the parameters used by the - * property create,copy and close callback functions. + * \details The H5P_prp_cb1_t() function describes the parameters used by the + * property create, copy and close callback functions. */ typedef herr_t (*H5P_prp_cb1_t)(const char *name, size_t size, void *value); //! @@ -162,8 +211,8 @@ typedef herr_t (*H5P_prp_cb1_t)(const char *name, size_t size, void *value); * \param[in] value The value for the property * \return \herr_t * - * \details The H5P_prp_cb2_t() describes the parameters used by the - * property set ,copy and delete callback functions. + * \details The H5P_prp_cb2_t() function describes the parameters used by the + * property set, copy and delete callback functions. */ typedef herr_t (*H5P_prp_cb2_t)(hid_t prop_id, const char *name, size_t size, void *value); //! @@ -173,13 +222,28 @@ typedef H5P_prp_cb2_t H5P_prp_set_func_t; typedef H5P_prp_cb2_t H5P_prp_get_func_t; //! /** - * \todo Document me! + * \brief Callback function for encoding property values + * + * \param[in] value The property value to be encoded + * \param[out] buf The encoded property value + * \param[out] size The size of \p buf + * \return \herr_t + * + * \note There is currently no public API which exposes a callback of this type. + * */ typedef herr_t (*H5P_prp_encode_func_t)(const void *value, void **buf, size_t *size); //! //! /** - * \todo Document me! + * \brief Callback function for decoding property values + * + * \param[in] buf A buffer containing an encoded property value + * \param[out] value The decoded property value + * \return \herr_t + * + * \note There is currently no public API which exposes a callback of this type. + * */ typedef herr_t (*H5P_prp_decode_func_t)(const void **buf, void *value); //! @@ -188,7 +252,16 @@ typedef H5P_prp_cb1_t H5P_prp_copy_func_t; //! /** - * \todo Document me! + * \brief Callback function for comparing property values + * + * \param[in] value1 A property value + * \param[in] value2 A property value + * \param[in] size The size of the \p value1 and \p value2 buffers + * \return Returns a positive value if \c value1 is greater than \c value2, a + * negative value if \c value2 is greater than \c value1 and zero if + * \c value1 and \c value2 are equal. + * + * \see H5Pregister(), H5Pinsert() */ typedef int (*H5P_prp_compare_func_t)(const void *value1, const void *value2, size_t size); //! @@ -198,7 +271,19 @@ typedef H5P_prp_cb1_t H5P_prp_close_func_t; /* Define property list iteration function type */ //! /** - * \todo Document me! + * \brief Callback function for H5Piterate() + * + * \param[in] id The identifier of a property list or property list class + * \param[in] name The name of the current property + * \param[in,out] iter_data The user context passed to H5Piterate() + * \return \herr_t_iter + * + * \details This function is called for each property encountered when + * iterating over a property list or property list class + * via H5Piterate(). + * + * \since 1.4.0 + * */ typedef herr_t (*H5P_iterate_t)(hid_t id, const char *name, void *iter_data); //! @@ -265,15 +350,15 @@ typedef enum H5D_mpio_no_collective_cause_t { H5D_MPIO_DATA_TRANSFORMS = 0x04, /**< Collective I/O was not performed because data transforms needed to be applied */ H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED = 0x08, - /**< \todo FIXME! */ + /**< Collective I/O was disabled by environment variable (\Code{HDF5_MPI_OPT_TYPES}) */ H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES = 0x10, /**< Collective I/O was not performed because one of the dataspaces was neither simple nor scalar */ H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET = 0x20, /**< Collective I/O was not performed because the dataset was neither contiguous nor chunked */ H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED = 0x40, - /**< \todo FIXME! */ + /**< Collective I/O was not performed because parallel filtered writes are disabled */ H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE = 0x80, - /**< \todo FIXME! */ + /**< Error */ H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE = 0x100 /**< Sentinel */ } H5D_mpio_no_collective_cause_t; @@ -579,77 +664,12 @@ H5_DLL hid_t H5Pcreate(hid_t cls_id); * those existing properties, only add or remove their own class * properties. Property list classes defined and supported in the * HDF5 library distribution are listed and briefly described in - * H5Pcreate(). The \p create routine is called when a new property - * list of this class is being created. The #H5P_cls_create_func_t - * callback function is defined as follows: + * H5Pcreate(). The \p create, \p copy, \p close functions are called + * when a property list of the new class is created, copied, or closed, + * respectively. * - * \snippet this H5P_cls_create_func_t_snip - * - * The parameters to this callback function are defined as follows: - * - * - * - * - * - * - * - * - * - *
    \ref hid_t \c prop_idIN: The identifier of the property list being created
    \Code{void * create_data}IN: User pointer to any class creation data required
    - * - * The \p create routine is called after any registered - * \p create function is called for each property value. If the - * \p create routine returns a negative value, the new list is not - * returned to the user and the property list creation routine returns - * an error value. - * - * The \p copy routine is called when an existing property - * list of this class is copied. The #H5P_cls_copy_func_t callback - * function is defined as follows: - * \snippet this H5P_cls_copy_func_t_snip - * - * The parameters to this callback function are defined as follows: - * - * - * - * - * - * - * - * - * - *
    \ref hid_t \c prop_idIN: The identifier of the property list created by copying
    \Code{void * copy_data}IN: User pointer to any class copy data required
    - * - * The \p copy routine is called after any registered \p copy function - * is called for each property value. If the \p copy routine returns a - * negative value, the new list is not returned to the user and the - * property list \p copy routine returns an error value. - * - * The \p close routine is called when a property list of this class - * is being closed. The #H5P_cls_close_func_t callback function is - * defined as follows: - * \snippet this H5P_cls_close_func_t_snip - * - * The parameters to this callback function are defined as follows: - * - * - * - * - * - * - * - * - * - *
    \ref hid_t \c prop_idIN: The identifier of the property list being closed
    \Code{void * close_data}IN: User pointer to any class close data required
    - * - * The \p close routine is called before any registered \p close - * function is called for each property value. If the \p close routine - * returns a negative value, the property list close routine returns - * an error value but the property list is still closed. - * - * H5Pclose_class() can be used to release the property list class - * identifier returned by this function so that resources leaks will - * not develop. + * H5Pclose_class() must be used to release the property list class + * identifier returned by this function. * * \since 1.4.0 * @@ -1378,35 +1398,12 @@ H5_DLL htri_t H5Pisa_class(hid_t plist_id, hid_t pclass_id); * returned in this case, the iterator cannot be restarted if * one of the calls to its operator returns non-zero. * - * The prototype for the #H5P_iterate_t operator is as follows: - * \snippet this H5P_iterate_t_snip - * - * The operation receives the property list or class + * The operation \p iter_func receives the property list or class * identifier for the object being iterated over, \p id, the * name of the current property within the object, \p name, * and the pointer to the operator data passed in to H5Piterate(), - * \p iter_data. The valid return values from an operator are - * as follows: + * \p iter_data. * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
    ZeroCauses the iterator to continue, returning zero when all - * properties have been processed
    PositiveCauses the iterator to immediately return that positive - * value, indicating short-circuit success. The iterator - * can be restarted at the index of the next property
    NegativeCauses the iterator to immediately return that value, - * indicating failure. The iterator can be restarted at the - * index of the next property
    * H5Piterate() assumes that the properties in the object * identified by \p id remain unchanged through the iteration. * If the membership changes during the iteration, the function's @@ -1879,9 +1876,6 @@ H5_DLL herr_t H5Pget_attr_phase_change(hid_t plist_id, unsigned *max_compact, un * * \brief Returns information about a filter in a pipeline * - * \todo Signature for H5Pget_filter2 is different in H5Pocpl.c than in - * H5Ppublic.h - * * \ocpl_id{plist_id} * \param[in] idx Sequence number within the filter pipeline of the filter * for which information is sought @@ -3450,6 +3444,34 @@ H5_DLL const void *H5Pget_driver_info(hid_t plist_id); /** * \ingroup FAPL * + * \brief Retrieves a string representation of the configuration for + * the driver set on the given FAPL. The returned string can + * be used to configure the same driver in an identical way. + * + * \fapl_id + * \param[out] config_buf Driver configuration string output buffer + * \param[in] buf_size Size of driver configuration string output buffer + * + * \return Returns the length of the driver configuration string on + * success (not including the NUL terminator). Returns negative + * on failure. + * + * \details H5Pget_driver_config_str() retrieves a string representation + * of the configuration for the driver set on the given FAPL. The + * returned string can be used to configure the same driver in + * an identical way. + * + * If \p config_buf is NULL, the length of the driver configuration + * string is simply returned. The caller can then allocate a buffer + * of the appropriate size and call this routine again. + * + * \version 1.12.1 Function publicized in this release. + * + */ +H5_DLL ssize_t H5Pget_driver_config_str(hid_t fapl_id, char *config_buf, size_t buf_size); +/** + * \ingroup FAPL + * * \brief Retrieves the size of the external link open file cache * * \fapl_id{plist_id} @@ -3886,13 +3908,13 @@ H5_DLL herr_t H5Pget_meta_block_size(hid_t fapl_id, hsize_t *size); * * The second example illustrates the two cases for retrieving the * number of read attempts from the file access property list of a file - * opened with SWMR acccess. + * opened with SWMR access. * * \include H5Pget_metadata_read_attempts.2.c * * The third example illustrates the two cases for retrieving the number * of read attempts from the file access property list of a file opened - * with non-SWMR acccess. + * with non-SWMR access. * * \include H5Pget_metadata_read_attempts.3.c * @@ -4183,17 +4205,14 @@ H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold, hsize_t alignme * * \note Note: Raw dataset chunk caching is not currently * supported when using the MPI I/O and MPI POSIX file drivers - * in read/write mode; see H5Pset_fapl_mpio() and - * H5Pset_fapl_mpiposix(), respectively. When using one of these - * file drivers, all calls to H5Dread() and H5Dwrite() will access + * in read/write mode; see H5Pset_fapl_mpio(). When using this + * file driver, all calls to H5Dread() and H5Dwrite() will access * the disk directly, and H5Pset_cache() will have no effect on * performance. * * \note Raw dataset chunk caching is supported when these drivers are * used in read-only mode. * - * \todo Check on H5Pset_fapl_mpio() and H5Pset_fapl_mpiposix(). - * * \version 1.8.0 The use of the \p mdc_nelmts parameter was discontinued. * Metadata cache configuration is managed with * H5Pset_mdc_config() and H5Pget_mdc_config(). @@ -4293,6 +4312,57 @@ H5_DLL herr_t H5Pset_driver(hid_t plist_id, hid_t driver_id, const void *driver_ /** * \ingroup FAPL * + * \brief Sets a file driver according to a given driver name + * + * \plist_id + * \param[in] driver_name The new driver name + * \param[in] driver_config Optional string containing driver properties + * + * \return \herr_t + * + * \details H5Pset_driver_by_name() sets the file driver, by the name + * driver_name, for a file access or data transfer property list, + * \p plist_id, and supplies an optional string containing the + * driver-specific properties, \p driver_config. The driver + * properties string will be copied into the property list. + * + * If the driver specified by \p driver_name is not currently + * registered, an attempt will be made to load the driver as a + * plugin. + * + * \version 1.12.1 Function publicized in this release. + * + */ +H5_DLL herr_t H5Pset_driver_by_name(hid_t plist_id, const char *driver_name, const char *driver_config); +/** + * \ingroup FAPL + * + * \brief Sets a file driver according to a given driver value (ID). + * + * \plist_id + * \param[in] driver_value The new driver value (ID) + * \param[in] driver_config Optional string containing driver properties + * + * \return \herr_t + * + * \details H5Pset_driver_by_value() sets the file driver, by the value + * driver_value, for a file access or data transfer property list, + * \p plist_id, and supplies an optional string containing the + * driver-specific properties, \p driver_config. The driver + * properties string will be copied into the property list. + * + * If the driver specified by \p driver_value is not currently + * registered, an attempt will be made to load the driver as a + * plugin. + * + * \version 1.12.1 Function publicized in this release. + * + */ +H5_DLL herr_t H5Pset_driver_by_value(hid_t plist_id, H5FD_class_value_t driver_value, + const char *driver_config); +/** + * \ingroup FAPL + * * \brief Sets the number of files that can be held open in an external * link open file cache * @@ -4301,7 +4371,7 @@ H5_DLL herr_t H5Pset_driver(hid_t plist_id, hid_t driver_id, const void *driver_ * The external link open file cache holds files open after * they have been accessed via an external link. This cache reduces * the number of times such files are opened when external links are - * accessed repeatedly and can siginificantly improves performance in + * accessed repeatedly and can significantly improves performance in * certain heavy-use situations and when low-level file opens or closes * are expensive. * @@ -5410,12 +5480,38 @@ H5_DLL herr_t H5Pset_coll_metadata_write(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective); /** - * \todo Add missing documentation + * \ingroup FAPL + * + * \brief Get the MPI communicator and info + * + * \fapl_id + * \param[out] comm MPI communicator + * \param[out] info MPI info object + * \return \herr_t + * + * \details H5Pget_mpi_params() gets the MPI communicator and info stored in + * the file access property list \p fapl_id. + * + * \todo When was this introduced? + * */ H5_DLL herr_t H5Pget_mpi_params(hid_t fapl_id, MPI_Comm *comm, MPI_Info *info); /** - * \todo Add missing documentation + * \ingroup FAPL + * + * \brief Set the MPI communicator and info + * + * \fapl_id + * \param[in] comm MPI communicator + * \param[in] info MPI info object + * \return \herr_t + * + * \details H5Pset_mpi_params() sets the MPI communicator and info stored in + * the file access property list \p fapl_id. + * + * \todo When was this introduced? + * */ H5_DLL herr_t H5Pset_mpi_params(hid_t fapl_id, MPI_Comm comm, MPI_Info info); #endif /* H5_HAVE_PARALLEL */ @@ -7031,9 +7127,6 @@ H5_DLL herr_t H5Pget_virtual_printf_gap(hid_t dapl_id, hsize_t *gap_size); * * \dapl_id * \param[out] view The flag specifying the view of the virtual dataset. - * Valid values are: - * \li #H5D_VDS_FIRST_MISSING - * \li #H5D_VDS_LAST_AVAILABLE * * \return \herr_t * @@ -7387,11 +7480,7 @@ H5_DLL herr_t H5Pset_virtual_printf_gap(hid_t dapl_id, hsize_t gap_size); * * \dapl_id * \param[in] view Flag specifying the extent of the data to be included - * in the view. Valid values are: - * \li #H5D_VDS_FIRST_MISSING: View includes all data - * before the first missing mapped data - * \li #H5D_VDS_LAST_AVAILABLE View includes all - * available mapped data + * in the view. * * \return \herr_t * @@ -7559,8 +7648,11 @@ H5_DLL herr_t H5Pget_hyper_vector_size(hid_t fapl_id, size_t *size /*out*/); * \details H5Pget_preserve() checks the status of the dataset transfer * property list. * + * \since 1.0.0 + * * \version 1.6.0 The flag parameter was changed from INTEGER to LOGICAL to * better match the C API. (Fortran 90) + * \version 1.8.2 Deprecated. * */ H5_DLL int H5Pget_preserve(hid_t plist_id); @@ -7588,6 +7680,8 @@ H5_DLL int H5Pget_preserve(hid_t plist_id); * * Please refer to the function H5Pset_type_conv_cb() for more details. * + * \since 1.8.0 + * */ H5_DLL herr_t H5Pget_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t *op, void **operate_data); /** @@ -7611,6 +7705,8 @@ H5_DLL herr_t H5Pget_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t *op, voi * H5Pset_vlen_mem_manager(), returning the parameters set by * that function. * + * \since 1.0.0 + * */ H5_DLL herr_t H5Pget_vlen_mem_manager(hid_t plist_id, H5MM_allocate_t *alloc_func, void **alloc_info, H5MM_free_t *free_func, void **free_info); @@ -7854,8 +7950,9 @@ H5_DLL herr_t H5Pset_hyper_vector_size(hid_t plist_id, size_t size); * I/O pipeline treats the destination datapoints as completely * uninitialized. * - * \todo Add missing version information: introduction, deprecation, etc. - * Why is the declaration not in the deprecated section? + * \since 1.0.0 + * + * \version 1.8.2 Deprecated. * */ H5_DLL herr_t H5Pset_preserve(hid_t plist_id, hbool_t status); @@ -7883,7 +7980,7 @@ H5_DLL herr_t H5Pset_preserve(hid_t plist_id, hbool_t status); * function prototype is as follows: * \snippet H5Tpublic.h H5T_conv_except_func_t_snip * - * \todo Add version information. + * \since 1.8.0 * */ H5_DLL herr_t H5Pset_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t op, void *operate_data); @@ -7933,7 +8030,8 @@ H5_DLL herr_t H5Pset_type_conv_cb(hid_t dxpl_id, H5T_conv_except_func_t op, void * set to \c NULL and the \p alloc_info and \p free_info parameters are * ignored. * - * \todo Add version information. + * \since 1.0.0 + * */ H5_DLL herr_t H5Pset_vlen_mem_manager(hid_t plist_id, H5MM_allocate_t alloc_func, void *alloc_info, H5MM_free_t free_func, void *free_info); diff --git a/src/H5RS.c b/src/H5RS.c index 65cfd38..16c2356 100644 --- a/src/H5RS.c +++ b/src/H5RS.c @@ -69,9 +69,6 @@ static herr_t H5RS__resize_for_append(H5RS_str_t *rs, size_t len); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -353,7 +350,7 @@ done: */ /* Disable warning for "format not a string literal" here -QAK */ /* - * This pragma only needs to surround the sprintf() calls with + * This pragma only needs to surround the snprintf() calls with * format_templ in the code below, but early (4.4.7, at least) gcc only * allows diagnostic pragmas to be toggled outside of functions. */ diff --git a/src/H5Rint.c b/src/H5Rint.c index 7ee4ecc..65bd485 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -141,9 +141,6 @@ static herr_t H5R__decode_string(const unsigned char *buf, size_t *nbytes, char /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -152,106 +149,27 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /* Local Variables */ /*******************/ -/* Flag indicating "top" of interface has been initialized */ -static hbool_t H5R_top_package_initialize_s = FALSE; - -/*-------------------------------------------------------------------------- -NAME - H5R__init_package -- Initialize interface-specific information -USAGE - herr_t H5R__init_package() - -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ +/*------------------------------------------------------------------------- + * Function: H5R_init + * + * Purpose: Initialize the interface from some other layer. + * + * Return: Success: non-negative + * Failure: negative + *------------------------------------------------------------------------- + */ herr_t -H5R__init_package(void) +H5R_init(void) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + herr_t ret_value = SUCCEED; - /* Mark "top" of interface as initialized */ - H5R_top_package_initialize_s = TRUE; + FUNC_ENTER_NOAPI_NOINIT_NOERR /* Sanity check, if assert fails, H5R_REF_BUF_SIZE must be increased */ HDcompile_assert(sizeof(H5R_ref_priv_t) <= H5R_REF_BUF_SIZE); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5R__init_package() */ - -/*-------------------------------------------------------------------------- - NAME - H5R_top_term_package - PURPOSE - Terminate various H5R objects - USAGE - void H5R_top_term_package() - RETURNS - void - DESCRIPTION - Release IDs for the ID group, deferring full interface shutdown - until later (in H5R_term_package). - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - Can't report errors... - EXAMPLES - REVISION LOG ---------------------------------------------------------------------------*/ -int -H5R_top_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Mark closed if initialized */ - if (H5R_top_package_initialize_s) - if (0 == n) - H5R_top_package_initialize_s = FALSE; - - FUNC_LEAVE_NOAPI(n) -} /* end H5R_top_term_package() */ - -/*-------------------------------------------------------------------------- - NAME - H5R_term_package - PURPOSE - Terminate various H5R objects - USAGE - void H5R_term_package() - RETURNS - void - DESCRIPTION - Release the ID group and any other resources allocated. - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - Can't report errors... - - Finishes shutting down the interface, after H5R_top_term_package() - is called - EXAMPLES - REVISION LOG ---------------------------------------------------------------------------*/ -int -H5R_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if (H5_PKG_INIT_VAR) { - /* Sanity checks */ - HDassert(FALSE == H5R_top_package_initialize_s); - - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } - - FUNC_LEAVE_NOAPI(n) -} /* end H5R_term_package() */ + FUNC_LEAVE_NOAPI(ret_value) +} /*------------------------------------------------------------------------- * Function: H5R__create_object diff --git a/src/H5Rprivate.h b/src/H5Rprivate.h index 6d6fcd5..59d2ed2 100644 --- a/src/H5Rprivate.h +++ b/src/H5Rprivate.h @@ -39,4 +39,6 @@ /* Library Private Prototypes */ /******************************/ +H5_DLL herr_t H5R_init(void); + #endif /* H5Rprivate_H */ diff --git a/src/H5S.c b/src/H5S.c index 13d32cf..75d3399 100644 --- a/src/H5S.c +++ b/src/H5S.c @@ -55,9 +55,6 @@ static htri_t H5S__is_simple(const H5S_t *sdim); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Format version bounds for dataspace */ const unsigned H5O_sdspace_ver_bounds[] = { H5O_SDSPACE_VERSION_1, /* H5F_LIBVER_EARLIEST */ @@ -96,9 +93,6 @@ static const H5I_class_t H5I_SPACE_SEL_ITER_CLS[1] = {{ (H5I_free_t)H5S__sel_iter_close_cb /* Callback routine for closing objects of this class */ }}; -/* Flag indicating "top" of interface has been initialized */ -static hbool_t H5S_top_package_initialize_s = FALSE; - /*------------------------------------------------------------------------- * Function: H5S_init * @@ -114,44 +108,19 @@ H5S_init(void) herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5S_init() */ - -/*-------------------------------------------------------------------------- -NAME - H5S__init_package -- Initialize interface-specific information -USAGE - herr_t H5S__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. ---------------------------------------------------------------------------*/ -herr_t -H5S__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE /* Initialize the ID group for the dataspace IDs */ if (H5I_register_type(H5I_DATASPACE_CLS) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize dataspace ID class") - /* Initialize the ID group for the dataspace selction iterator IDs */ + /* Initialize the ID group for the dataspace selection iterator IDs */ if (H5I_register_type(H5I_SPACE_SEL_ITER_CLS) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize dataspace selection iterator ID class") - /* Mark "top" of interface as initialized, too */ - H5S_top_package_initialize_s = TRUE; - done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5S__init_package() */ +} /* end H5S_init() */ /*-------------------------------------------------------------------------- NAME @@ -178,21 +147,14 @@ H5S_top_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5S_top_package_initialize_s) { - if (H5I_nmembers(H5I_DATASPACE) > 0) { - (void)H5I_clear_type(H5I_DATASPACE, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - - if (H5I_nmembers(H5I_SPACE_SEL_ITER) > 0) { - (void)H5I_clear_type(H5I_SPACE_SEL_ITER, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - - /* Mark "top" of interface as closed */ - if (0 == n) - H5S_top_package_initialize_s = FALSE; - } /* end if */ + if (H5I_nmembers(H5I_DATASPACE) > 0) { + (void)H5I_clear_type(H5I_DATASPACE, FALSE, FALSE); + n++; + } + if (H5I_nmembers(H5I_SPACE_SEL_ITER) > 0) { + (void)H5I_clear_type(H5I_SPACE_SEL_ITER, FALSE, FALSE); + n++; + } FUNC_LEAVE_NOAPI(n) } /* end H5S_top_term_package() */ @@ -224,22 +186,15 @@ H5S_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Sanity checks */ - HDassert(0 == H5I_nmembers(H5I_DATASPACE)); - HDassert(0 == H5I_nmembers(H5I_SPACE_SEL_ITER)); - HDassert(FALSE == H5S_top_package_initialize_s); - - /* Destroy the dataspace object id group */ - n += (H5I_dec_type_ref(H5I_DATASPACE) > 0); + /* Sanity checks */ + HDassert(0 == H5I_nmembers(H5I_DATASPACE)); + HDassert(0 == H5I_nmembers(H5I_SPACE_SEL_ITER)); - /* Destroy the dataspace selection iterator object id group */ - n += (H5I_dec_type_ref(H5I_SPACE_SEL_ITER) > 0); + /* Destroy the dataspace object id group */ + n += (H5I_dec_type_ref(H5I_DATASPACE) > 0); - /* Mark interface as closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Destroy the dataspace selection iterator object id group */ + n += (H5I_dec_type_ref(H5I_SPACE_SEL_ITER) > 0); FUNC_LEAVE_NOAPI(n) } /* end H5S_term_package() */ @@ -759,7 +714,7 @@ H5S_get_simple_extent_npoints(const H5S_t *ds) { hssize_t ret_value = -1; /* Return value */ - FUNC_ENTER_NOAPI(-1) + FUNC_ENTER_NOAPI_NOERR /* check args */ HDassert(ds); @@ -767,7 +722,6 @@ H5S_get_simple_extent_npoints(const H5S_t *ds) /* Get the number of elements in extent */ ret_value = (hssize_t)ds->extent.nelem; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_get_simple_extent_npoints() */ @@ -1740,13 +1694,12 @@ H5S_get_simple_extent_type(const H5S_t *space) { H5S_class_t ret_value = H5S_NO_CLASS; /* Return value */ - FUNC_ENTER_NOAPI(H5S_NO_CLASS) + FUNC_ENTER_NOAPI_NOERR HDassert(space); ret_value = H5S_GET_EXTENT_TYPE(space); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_get_simple_extent_type() */ diff --git a/src/H5SL.c b/src/H5SL.c index b4fbf99..e12e21a 100644 --- a/src/H5SL.c +++ b/src/H5SL.c @@ -239,7 +239,7 @@ } /* Macro used to reduce the level of a node by 1. Does not update the head node - * "current level". PREV is the previous node of the currrent height of X. */ + * "current level". PREV is the previous node of the current height of X. */ #define H5SL_DEMOTE(X, PREV) \ { \ size_t _lvl = X->level; \ @@ -255,7 +255,7 @@ #define H5SL_INSERT(CMP, SLIST, X, TYPE, KEY, HASHVAL) \ { \ H5SL_node_t *_last = X; /* Lowest node in the current gap */ \ - H5SL_node_t *_next = NULL; /* Highest node in the currect gap */ \ + H5SL_node_t *_next = NULL; /* Highest node in the current gap */ \ H5SL_node_t *_drop; /* Low node of the gap to drop into */ \ int _count; /* Number of nodes in the current gap */ \ int _i; \ @@ -311,7 +311,7 @@ { \ H5SL_node_t *_last = X; /* Lowest node in the current gap */ \ H5SL_node_t *_llast = X; /* Lowest node in the previous gap */ \ - H5SL_node_t *_next = NULL; /* Highest node in the currect gap */ \ + H5SL_node_t *_next = NULL; /* Highest node in the current gap */ \ H5SL_node_t *_drop = NULL; /* Low node of the gap to drop into */ \ H5SL_node_t *_ldrop = NULL; /* Low node of gap before the one to drop into */ \ H5SL_node_t *_head = SLIST->header; /* Head of the skip list */ \ @@ -510,9 +510,6 @@ static H5SL_node_t *H5SL__insert_common(H5SL_t *slist, void *item, const void *k static herr_t H5SL__release_common(H5SL_t *slist, H5SL_operator_t op, void *op_data); static herr_t H5SL__close_common(H5SL_t *slist, H5SL_operator_t op, void *op_data); -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Declare a free list to manage the H5SL_t struct */ H5FL_DEFINE_STATIC(H5SL_t); @@ -524,26 +521,21 @@ static H5FL_fac_head_t **H5SL_fac_g; static size_t H5SL_fac_nused_g; static size_t H5SL_fac_nalloc_g; -/*-------------------------------------------------------------------------- - NAME - H5SL__init_package - PURPOSE - Initialize interface-specific information - USAGE - herr_t H5SL__init_package() - RETURNS - Non-negative on success/Negative on failure - DESCRIPTION - Initializes any interface-specific data or routines. - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - EXAMPLES - REVISION LOG ---------------------------------------------------------------------------*/ +/*------------------------------------------------------------------------- + * Function: H5SL_init + * + * Purpose: Initialize the interface from some other layer. + * + * Return: Success: non-negative + * Failure: negative + *------------------------------------------------------------------------- + */ herr_t -H5SL__init_package(void) +H5SL_init(void) { - FUNC_ENTER_PACKAGE_NOERR + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOERR /* Allocate space for array of factories */ H5SL_fac_g = (H5FL_fac_head_t **)H5MM_malloc(sizeof(H5FL_fac_head_t *)); @@ -555,8 +547,8 @@ H5SL__init_package(void) HDassert(H5SL_fac_g[0]); H5SL_fac_nused_g = 1; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5SL__init_package() */ + FUNC_LEAVE_NOAPI(ret_value) +} /*-------------------------------------------------------------------------- NAME @@ -584,32 +576,26 @@ H5SL_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Terminate all the factories */ - if (H5SL_fac_nused_g > 0) { - size_t i; - herr_t H5_ATTR_NDEBUG_UNUSED ret; - - for (i = 0; i < H5SL_fac_nused_g; i++) { - ret = H5FL_fac_term(H5SL_fac_g[i]); - HDassert(ret >= 0); - } - H5SL_fac_nused_g = 0; + /* Terminate all the factories */ + if (H5SL_fac_nused_g > 0) { + size_t i; + herr_t H5_ATTR_NDEBUG_UNUSED ret; - n++; + for (i = 0; i < H5SL_fac_nused_g; i++) { + ret = H5FL_fac_term(H5SL_fac_g[i]); + HDassert(ret >= 0); } + H5SL_fac_nused_g = 0; - /* Free the list of factories */ - if (H5SL_fac_g) { - H5SL_fac_g = (H5FL_fac_head_t **)H5MM_xfree((void *)H5SL_fac_g); - H5SL_fac_nalloc_g = 0; + n++; + } - n++; - } + /* Free the list of factories */ + if (H5SL_fac_g) { + H5SL_fac_g = (H5FL_fac_head_t **)H5MM_xfree((void *)H5SL_fac_g); + H5SL_fac_nalloc_g = 0; - /* Mark the interface as uninitialized */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; + n++; } FUNC_LEAVE_NOAPI(n) @@ -1943,7 +1929,7 @@ H5SL_next(H5SL_node_t *slist_node) NAME H5SL_prev PURPOSE - Gets a pointer to the previos node in a skip list + Gets a pointer to the previous node in a skip list USAGE H5SL_node_t *H5SL_prev(slist_node) H5SL_node_t *slist_node; IN: Pointer to skip list node diff --git a/src/H5SLprivate.h b/src/H5SLprivate.h index be6f7b6..f4f7506 100644 --- a/src/H5SLprivate.h +++ b/src/H5SLprivate.h @@ -85,6 +85,7 @@ H5_DLL herr_t H5SL_release(H5SL_t *slist); H5_DLL herr_t H5SL_free(H5SL_t *slist, H5SL_operator_t op, void *op_data); H5_DLL herr_t H5SL_close(H5SL_t *slist); H5_DLL herr_t H5SL_destroy(H5SL_t *slist, H5SL_operator_t op, void *op_data); +H5_DLL herr_t H5SL_init(void); H5_DLL int H5SL_term_interface(void); #endif /* H5SLprivate_H */ diff --git a/src/H5SM.c b/src/H5SM.c index 03fb08c..1e92c65 100644 --- a/src/H5SM.c +++ b/src/H5SM.c @@ -76,9 +76,6 @@ static herr_t H5SM__read_mesg(H5F_t *f, const H5SM_sohm_t *mesg, H5HF_t *fheap, /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - H5FL_DEFINE(H5SM_master_table_t); H5FL_ARR_DEFINE(H5SM_index_header_t, H5O_SHMESG_MAX_NINDEXES); H5FL_DEFINE(H5SM_list_t); @@ -117,7 +114,7 @@ H5SM_init(H5F_t *f, H5P_genplist_t *fc_plist, const H5O_loc_t *ext_loc) haddr_t table_addr = HADDR_UNDEF; /* Address of SOHM master table in file */ unsigned list_max, btree_min; /* Phase change limits for SOHM indices */ unsigned index_type_flags[H5O_SHMESG_MAX_NINDEXES]; /* Messages types stored in each index */ - unsigned minsizes[H5O_SHMESG_MAX_NINDEXES]; /* Message size sharing threshhold for each index */ + unsigned minsizes[H5O_SHMESG_MAX_NINDEXES]; /* Message size sharing threshold for each index */ unsigned type_flags_used; /* Message type flags used, for sanity checking */ unsigned x; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1599,7 +1596,7 @@ H5SM_delete(H5F_t *f, H5O_t *open_oh, H5O_shared_t *sh_mesg) */ if (H5SM__delete_from_index(f, open_oh, &(table->indexes[index_num]), sh_mesg, &cache_flags, &mesg_size, &mesg_buf) < 0) - HGOTO_ERROR(H5E_SOHM, H5E_CANTDELETE, FAIL, "unable to delete mesage from SOHM index") + HGOTO_ERROR(H5E_SOHM, H5E_CANTDELETE, FAIL, "unable to delete message from SOHM index") /* Release the master SOHM table */ if (H5AC_unprotect(f, H5AC_SOHM_TABLE, H5F_SOHM_ADDR(f), table, cache_flags) < 0) diff --git a/src/H5Sall.c b/src/H5Sall.c index 35ddf3d..4dcc83a 100644 --- a/src/H5Sall.c +++ b/src/H5Sall.c @@ -49,22 +49,22 @@ static herr_t H5S__all_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); static herr_t H5S__all_release(H5S_t *space); static htri_t H5S__all_is_valid(const H5S_t *space); -static hssize_t H5S__all_serial_size(const H5S_t *space); -static herr_t H5S__all_serialize(const H5S_t *space, uint8_t **p); +static hssize_t H5S__all_serial_size(H5S_t *space); +static herr_t H5S__all_serialize(H5S_t *space, uint8_t **p); static herr_t H5S__all_deserialize(H5S_t **space, const uint8_t **p); static herr_t H5S__all_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S__all_offset(const H5S_t *space, hsize_t *off); static int H5S__all_unlim_dim(const H5S_t *space); static htri_t H5S__all_is_contiguous(const H5S_t *space); static htri_t H5S__all_is_single(const H5S_t *space); -static htri_t H5S__all_is_regular(const H5S_t *space); -static htri_t H5S__all_shape_same(const H5S_t *space1, const H5S_t *space2); -static htri_t H5S__all_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end); +static htri_t H5S__all_is_regular(H5S_t *space); +static htri_t H5S__all_shape_same(H5S_t *space1, H5S_t *space2); +static htri_t H5S__all_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end); static herr_t H5S__all_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__all_adjust_s(H5S_t *space, const hssize_t *offset); static herr_t H5S__all_project_scalar(const H5S_t *space, hsize_t *offset); static herr_t H5S__all_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); -static herr_t H5S__all_iter_init(const H5S_t *space, H5S_sel_iter_t *iter); +static herr_t H5S__all_iter_init(H5S_t *space, H5S_sel_iter_t *iter); /* Selection iteration callbacks */ static herr_t H5S__all_iter_coords(const H5S_sel_iter_t *iter, hsize_t *coords); @@ -144,7 +144,7 @@ static const H5S_sel_iter_class_t H5S_sel_iter_all[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5S__all_iter_init(const H5S_t H5_ATTR_UNUSED *space, H5S_sel_iter_t *iter) +H5S__all_iter_init(H5S_t H5_ATTR_UNUSED *space, H5S_sel_iter_t *iter) { FUNC_ENTER_STATIC_NOERR @@ -557,7 +557,7 @@ H5S__all_is_valid(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static hssize_t -H5S__all_serial_size(const H5S_t H5_ATTR_UNUSED *space) +H5S__all_serial_size(H5S_t H5_ATTR_UNUSED *space) { FUNC_ENTER_STATIC_NOERR @@ -577,7 +577,7 @@ H5S__all_serial_size(const H5S_t H5_ATTR_UNUSED *space) Serialize the current selection into a user-provided buffer. USAGE herr_t H5S__all_serialize(space, p) - const H5S_t *space; IN: Dataspace with selection to serialize + H5S_t *space; IN: Dataspace with selection to serialize uint8_t **p; OUT: Pointer to buffer to put serialized selection. Will be advanced to end of serialized selection. @@ -592,7 +592,7 @@ H5S__all_serial_size(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S__all_serialize(const H5S_t *space, uint8_t **p) +H5S__all_serialize(H5S_t *space, uint8_t **p) { uint8_t *pp = (*p); /* Local pointer for decoding */ @@ -864,7 +864,7 @@ H5S__all_is_single(const H5S_t H5_ATTR_UNUSED *space) Check if a "all" selection is "regular" USAGE htri_t H5S__all_is_regular(space) - const H5S_t *space; IN: Dataspace pointer to check + H5S_t *space; IN: Dataspace pointer to check RETURNS TRUE/FALSE/FAIL DESCRIPTION @@ -877,7 +877,7 @@ H5S__all_is_single(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__all_is_regular(const H5S_t H5_ATTR_UNUSED *space) +H5S__all_is_regular(H5S_t H5_ATTR_UNUSED *space) { FUNC_ENTER_STATIC_NOERR @@ -894,8 +894,8 @@ H5S__all_is_regular(const H5S_t H5_ATTR_UNUSED *space) Check if a two "all" selections are the same shape USAGE htri_t H5S__all_shape_same(space1, space2) - const H5S_t *space1; IN: First dataspace to check - const H5S_t *space2; IN: Second dataspace to check + H5S_t *space1; IN: First dataspace to check + H5S_t *space2; IN: Second dataspace to check RETURNS TRUE / FALSE / FAIL DESCRIPTION @@ -907,7 +907,7 @@ H5S__all_is_regular(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__all_shape_same(const H5S_t *space1, const H5S_t *space2) +H5S__all_shape_same(H5S_t *space1, H5S_t *space2) { int space1_dim; /* Current dimension in first dataspace */ int space2_dim; /* Current dimension in second dataspace */ @@ -957,7 +957,7 @@ done: Detect intersections of selection with block USAGE htri_t H5S__all_intersect_block(space, start, end) - const H5S_t *space; IN: Dataspace with selection to use + H5S_t *space; IN: Dataspace with selection to use const hsize_t *start; IN: Starting coordinate for block const hsize_t *end; IN: Ending coordinate for block RETURNS @@ -970,7 +970,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ htri_t -H5S__all_intersect_block(const H5S_t H5_ATTR_UNUSED *space, const hsize_t H5_ATTR_UNUSED *start, +H5S__all_intersect_block(H5S_t H5_ATTR_UNUSED *space, const hsize_t H5_ATTR_UNUSED *start, const hsize_t H5_ATTR_UNUSED *end) { FUNC_ENTER_STATIC_NOERR diff --git a/src/H5Shyper.c b/src/H5Shyper.c index a5b9b6c..7284846 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -173,8 +173,8 @@ static herr_t H5S__hyper_copy(H5S_t *dst, const H5S_t *src, hbool_t share_sele static herr_t H5S__hyper_release(H5S_t *space); static htri_t H5S__hyper_is_valid(const H5S_t *space); static hsize_t H5S__hyper_span_nblocks(H5S_hyper_span_info_t *spans); -static hssize_t H5S__hyper_serial_size(const H5S_t *space); -static herr_t H5S__hyper_serialize(const H5S_t *space, uint8_t **p); +static hssize_t H5S__hyper_serial_size(H5S_t *space); +static herr_t H5S__hyper_serialize(H5S_t *space, uint8_t **p); static herr_t H5S__hyper_deserialize(H5S_t **space, const uint8_t **p); static herr_t H5S__hyper_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S__hyper_offset(const H5S_t *space, hsize_t *offset); @@ -182,14 +182,14 @@ static int H5S__hyper_unlim_dim(const H5S_t *space); static herr_t H5S__hyper_num_elem_non_unlim(const H5S_t *space, hsize_t *num_elem_non_unlim); static htri_t H5S__hyper_is_contiguous(const H5S_t *space); static htri_t H5S__hyper_is_single(const H5S_t *space); -static htri_t H5S__hyper_is_regular(const H5S_t *space); -static htri_t H5S__hyper_shape_same(const H5S_t *space1, const H5S_t *space2); -static htri_t H5S__hyper_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end); +static htri_t H5S__hyper_is_regular(H5S_t *space); +static htri_t H5S__hyper_shape_same(H5S_t *space1, H5S_t *space2); +static htri_t H5S__hyper_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end); static herr_t H5S__hyper_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__hyper_adjust_s(H5S_t *space, const hssize_t *offset); static herr_t H5S__hyper_project_scalar(const H5S_t *space, hsize_t *offset); static herr_t H5S__hyper_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); -static herr_t H5S__hyper_iter_init(const H5S_t *space, H5S_sel_iter_t *iter); +static herr_t H5S__hyper_iter_init(H5S_t *space, H5S_sel_iter_t *iter); /* Selection iteration callbacks */ static herr_t H5S__hyper_iter_coords(const H5S_sel_iter_t *iter, hsize_t *coords); @@ -296,12 +296,12 @@ H5S__hyper_print_spans_helper(FILE *f, const H5S_hyper_span_t *span, unsigned de FUNC_ENTER_STATIC_NOERR while (span) { - HDfprintf(f, "%s: %*sdepth=%u, span=%p, (%Hu, %Hu), next=%p\n", __func__, depth * 2, "", depth, span, - span->low, span->high, span->next); + HDfprintf(f, "%s: %*sdepth=%u, span=%p, (%" PRIuHSIZE ", %" PRIuHSIZE "), next=%p\n", __func__, + depth * 2, "", depth, (void *)span, span->low, span->high, (void *)span->next); if (span->down) { - HDfprintf(f, "%s: %*sspans=%p, count=%u, bounds[0]={%Hu, %Hu}, head=%p\n", __func__, - (depth + 1) * 2, "", span->down, span->down->count, span->down->low_bounds[0], - span->down->high_bounds[0], span->down->head); + HDfprintf(f, "%s: %*sspans=%p, count=%u, bounds[0]={%" PRIuHSIZE ", %" PRIuHSIZE "}, head=%p\n", + __func__, (depth + 1) * 2, "", (void *)span->down, span->down->count, + span->down->low_bounds[0], span->down->high_bounds[0], (void *)span->down->head); H5S__hyper_print_spans_helper(f, span->down->head, depth + 1); } /* end if */ span = span->next; @@ -316,8 +316,9 @@ H5S__hyper_print_spans(FILE *f, const H5S_hyper_span_info_t *span_lst) FUNC_ENTER_STATIC_NOERR if (span_lst != NULL) { - HDfprintf(f, "%s: spans=%p, count=%u, bounds[0]={%Hu, %Hu}, head=%p\n", __func__, span_lst, - span_lst->count, span_lst->low_bounds[0], span_lst->high_bounds[0], span_lst->head); + HDfprintf(f, "%s: spans=%p, count=%u, bounds[0]={%" PRIuHSIZE ", %" PRIuHSIZE "}, head=%p\n", + __func__, (void *)span_lst, span_lst->count, span_lst->low_bounds[0], + span_lst->high_bounds[0], (void *)span_lst->head); H5S__hyper_print_spans_helper(f, span_lst->head, 0); } /* end if */ @@ -344,16 +345,16 @@ H5S__hyper_print_diminfo_helper(FILE *f, const char *field, unsigned ndims, cons if (dinfo != NULL) { HDfprintf(f, "%s: %s: start=[", __func__, field); for (u = 0; u < ndims; u++) - HDfprintf(f, "%Hd%s", dinfo[u].start, (u < (ndims - 1) ? ", " : "]\n")); + HDfprintf(f, "%" PRIuHSIZE "%s", dinfo[u].start, (u < (ndims - 1) ? ", " : "]\n")); HDfprintf(f, "%s: %s: stride=[", __func__, field); for (u = 0; u < ndims; u++) - HDfprintf(f, "%Hu%s", dinfo[u].stride, (u < (ndims - 1) ? ", " : "]\n")); + HDfprintf(f, "%" PRIuHSIZE "%s", dinfo[u].stride, (u < (ndims - 1) ? ", " : "]\n")); HDfprintf(f, "%s: %s: count=[", __func__, field); for (u = 0; u < ndims; u++) - HDfprintf(f, "%Hu%s", dinfo[u].count, (u < (ndims - 1) ? ", " : "]\n")); + HDfprintf(f, "%" PRIuHSIZE "%s", dinfo[u].count, (u < (ndims - 1) ? ", " : "]\n")); HDfprintf(f, "%s: %s: block=[", __func__, field); for (u = 0; u < ndims; u++) - HDfprintf(f, "%Hu%s", dinfo[u].block, (u < (ndims - 1) ? ", " : "]\n")); + HDfprintf(f, "%" PRIuHSIZE "%s", dinfo[u].block, (u < (ndims - 1) ? ", " : "]\n")); } /* end if */ else HDfprintf(f, "%s: %s==NULL\n", __func__, field); @@ -412,31 +413,31 @@ H5S__hyper_print_spans_dfs(FILE *f, const H5S_hyper_span_info_t *span_lst, unsig for (u = 0; u < depth; u++) HDfprintf(f, "\t"); - HDfprintf(f, "DIM[%u]: ref_count=%u, #elems=%u, head=%p, tail=%p, actual_tail=%p, matched=%t\n", depth, - span_lst->count, num_elems, span_lst->head, span_lst->tail, actual_tail, + HDfprintf(f, "DIM[%u]: ref_count=%u, #elems=%u, head=%p, tail=%p, actual_tail=%p, matched=%d\n", depth, + span_lst->count, num_elems, (void *)span_lst->head, (void *)span_lst->tail, (void *)actual_tail, (span_lst->tail == actual_tail)); for (u = 0; u < depth; u++) HDfprintf(f, "\t"); HDfprintf(f, "low_bounds=["); for (u = 0; u < dims - 1; u++) - HDfprintf(f, "%llu,", span_lst->low_bounds[u]); - HDfprintf(f, "%llu]\n", span_lst->low_bounds[dims - 1]); + HDfprintf(f, "%" PRIuHSIZE ",", span_lst->low_bounds[u]); + HDfprintf(f, "%" PRIuHSIZE "]\n", span_lst->low_bounds[dims - 1]); for (u = 0; u < depth; u++) HDfprintf(f, "\t"); HDfprintf(f, "high_bounds=["); for (u = 0; u < dims - 1; u++) - HDfprintf(f, "%llu,", span_lst->high_bounds[u]); - HDfprintf(f, "%llu]\n", span_lst->high_bounds[dims - 1]); + HDfprintf(f, "%" PRIuHSIZE ",", span_lst->high_bounds[u]); + HDfprintf(f, "%" PRIuHSIZE "]\n", span_lst->high_bounds[dims - 1]); cur_elem = span_lst->head; elem_idx = 0; while (cur_elem) { for (u = 0; u < depth; u++) HDfprintf(f, "\t"); - HDfprintf(f, "ELEM[%u]: ptr=%p, low=%Hu, high=%Hu, down=%p\n", elem_idx++, cur_elem, cur_elem->low, - cur_elem->high, cur_elem->down); + HDfprintf(f, "ELEM[%u]: ptr=%p, low=%" PRIuHSIZE ", high=%" PRIuHSIZE ", down=%p\n", elem_idx++, + (void *)cur_elem, cur_elem->low, cur_elem->high, (void *)cur_elem->down); if (cur_elem->down) H5S__hyper_print_spans_dfs(f, cur_elem->down, depth + 1, dims); cur_elem = cur_elem->next; @@ -473,7 +474,7 @@ H5S__hyper_print_space_dfs(FILE *f, const H5S_t *space) HDassert(hslab); HDfprintf(f, "=======================\n"); - HDfprintf(f, "SPACE: span_lst=%p, #dims=%u, offset_changed=%d\n", hslab->span_lst, dims, + HDfprintf(f, "SPACE: span_lst=%p, #dims=%u, offset_changed=%d\n", (void *)hslab->span_lst, dims, space->select.offset_changed); HDfprintf(f, " offset=["); @@ -484,25 +485,25 @@ H5S__hyper_print_space_dfs(FILE *f, const H5S_t *space) HDfprintf(f, " low_bounds=["); if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES) { for (u = 0; u < dims - 1; u++) - HDfprintf(f, "%llu,", space->select.sel_info.hslab->diminfo.low_bounds[u]); - HDfprintf(f, "%llu]\n", space->select.sel_info.hslab->diminfo.low_bounds[dims - 1]); + HDfprintf(f, "%" PRIuHSIZE ",", space->select.sel_info.hslab->diminfo.low_bounds[u]); + HDfprintf(f, "%" PRIuHSIZE "]\n", space->select.sel_info.hslab->diminfo.low_bounds[dims - 1]); } /* end if */ else { for (u = 0; u < dims - 1; u++) - HDfprintf(f, "%llu,", space->select.sel_info.hslab->span_lst->low_bounds[u]); - HDfprintf(f, "%llu]\n", space->select.sel_info.hslab->span_lst->low_bounds[dims - 1]); + HDfprintf(f, "%" PRIuHSIZE ",", space->select.sel_info.hslab->span_lst->low_bounds[u]); + HDfprintf(f, "%" PRIuHSIZE "]\n", space->select.sel_info.hslab->span_lst->low_bounds[dims - 1]); } /* end else */ HDfprintf(f, " high_bounds=["); if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES) { for (u = 0; u < dims - 1; u++) - HDfprintf(f, "%llu,", space->select.sel_info.hslab->diminfo.high_bounds[u]); - HDfprintf(f, "%llu]\n", space->select.sel_info.hslab->diminfo.high_bounds[dims - 1]); + HDfprintf(f, "%" PRIuHSIZE ",", space->select.sel_info.hslab->diminfo.high_bounds[u]); + HDfprintf(f, "%" PRIuHSIZE "]\n", space->select.sel_info.hslab->diminfo.high_bounds[dims - 1]); } /* end if */ else { for (u = 0; u < dims - 1; u++) - HDfprintf(f, "%llu,", space->select.sel_info.hslab->span_lst->high_bounds[u]); - HDfprintf(f, "%llu]\n", space->select.sel_info.hslab->span_lst->high_bounds[dims - 1]); + HDfprintf(f, "%" PRIuHSIZE ",", space->select.sel_info.hslab->span_lst->high_bounds[u]); + HDfprintf(f, "%" PRIuHSIZE "]\n", space->select.sel_info.hslab->span_lst->high_bounds[dims - 1]); } /* end else */ /* Print out diminfo, if it's valid */ @@ -559,7 +560,7 @@ H5S__hyper_get_op_gen(void) *------------------------------------------------------------------------- */ static herr_t -H5S__hyper_iter_init(const H5S_t *space, H5S_sel_iter_t *iter) +H5S__hyper_iter_init(H5S_t *space, H5S_sel_iter_t *iter) { hsize_t *slab_size; /* Pointer to the dataspace dimensions to use for calc. slab */ hsize_t acc; /* Accumulator for computing cumulative sizes */ @@ -586,7 +587,7 @@ H5S__hyper_iter_init(const H5S_t *space, H5S_sel_iter_t *iter) * to be impossible. */ if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_NO) - H5S__hyper_rebuild((H5S_t *)space); /* Casting away const OK -NAF */ + H5S__hyper_rebuild(space); /* Check for the special case of just one H5Sselect_hyperslab call made */ if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES) { @@ -2060,7 +2061,7 @@ H5S__hyper_iter_get_seq_list_opt(H5S_sel_iter_t *iter, size_t maxseq, size_t max /* Increment the offset and count for the other dimensions */ temp_dim = (int)fast_dim - 1; while (temp_dim >= 0) { - /* Move to the next row in the curent dimension */ + /* Move to the next row in the current dimension */ offset[temp_dim]++; tmp_block[temp_dim]++; @@ -2178,7 +2179,7 @@ H5S__hyper_iter_get_seq_list_opt(H5S_sel_iter_t *iter, size_t maxseq, size_t max /* Increment the offset and count for the other dimensions */ temp_dim = (int)fast_dim - 1; while (temp_dim >= 0) { - /* Move to the next row in the curent dimension */ + /* Move to the next row in the current dimension */ offset[temp_dim]++; tmp_block[temp_dim]++; @@ -2492,7 +2493,7 @@ H5S__hyper_iter_get_seq_list_single(H5S_sel_iter_t *iter, size_t maxseq, size_t /* Increment the offset and count for the other dimensions */ while (temp_dim >= 0) { - /* Move to the next row in the curent dimension */ + /* Move to the next row in the current dimension */ offset[temp_dim]++; tmp_block[temp_dim]++; @@ -3575,7 +3576,7 @@ H5S__hyper_get_enc_size_real(hsize_t max_size) REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S__hyper_get_version_enc_size(const H5S_t *space, hsize_t block_count, uint32_t *version, uint8_t *enc_size) +H5S__hyper_get_version_enc_size(H5S_t *space, hsize_t block_count, uint32_t *version, uint8_t *enc_size) { hsize_t bounds_start[H5S_MAX_RANK]; /* Starting coordinate of bounding box */ hsize_t bounds_end[H5S_MAX_RANK]; /* Opposite coordinate of bounding box */ @@ -3584,7 +3585,7 @@ H5S__hyper_get_version_enc_size(const H5S_t *space, hsize_t block_count, uint32_ H5F_libver_t low_bound; /* The 'low' bound of library format versions */ H5F_libver_t high_bound; /* The 'high' bound of library format versions */ htri_t is_regular; /* A regular hyperslab or not */ - uint32_t tmp_version; /* Local temporay version */ + uint32_t tmp_version; /* Local temporary version */ unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -3728,7 +3729,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ static hssize_t -H5S__hyper_serial_size(const H5S_t *space) +H5S__hyper_serial_size(H5S_t *space) { hsize_t block_count = 0; /* block counter for regular hyperslabs */ uint32_t version; /* Version number */ @@ -3928,7 +3929,7 @@ H5S__hyper_serialize_helper(const H5S_hyper_span_info_t *spans, hsize_t *start, Serialize the current selection into a user-provided buffer. USAGE herr_t H5S__hyper_serialize(space, p) - const H5S_t *space; IN: Dataspace with selection to serialize + H5S_t *space; IN: Dataspace with selection to serialize uint8_t **p; OUT: Pointer to buffer to put serialized selection. Will be advanced to end of serialized selection. @@ -3943,7 +3944,7 @@ H5S__hyper_serialize_helper(const H5S_hyper_span_info_t *spans, hsize_t *start, REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S__hyper_serialize(const H5S_t *space, uint8_t **p) +H5S__hyper_serialize(H5S_t *space, uint8_t **p) { const H5S_hyper_dim_t *diminfo; /* Alias for dataspace's diminfo information */ hsize_t tmp_count[H5S_MAX_RANK]; /* Temporary hyperslab counts */ @@ -3972,7 +3973,7 @@ H5S__hyper_serialize(const H5S_t *space, uint8_t **p) pp = (*p); HDassert(pp); - /* Set some convienence values */ + /* Set some convenience values */ ndims = space->extent.rank; diminfo = space->select.sel_info.hslab->diminfo.opt; @@ -4075,7 +4076,7 @@ H5S__hyper_serialize(const H5S_t *space, uint8_t **p) else { HDassert(version == H5S_HYPER_VERSION_1); - /* Set some convienence values */ + /* Set some convenience values */ fast_dim = ndims - 1; /* Encode number of hyperslabs */ @@ -4225,7 +4226,7 @@ H5S__hyper_deserialize(H5S_t **space, const uint8_t **p) { H5S_t *tmp_space = NULL; /* Pointer to actual dataspace to use, either *space or a newly allocated one */ - hsize_t dims[H5S_MAX_RANK]; /* Dimenion sizes */ + hsize_t dims[H5S_MAX_RANK]; /* Dimension sizes */ hsize_t start[H5S_MAX_RANK]; /* hyperslab start information */ hsize_t block[H5S_MAX_RANK]; /* hyperslab block information */ uint32_t version; /* Version number */ @@ -4618,7 +4619,7 @@ H5S__get_select_hyper_blocklist(H5S_t *space, hsize_t startblock, hsize_t numblo hbool_t done; /* Whether we are done with the iteration */ unsigned u; /* Counter */ - /* Set some convienence values */ + /* Set some convenience values */ ndims = space->extent.rank; fast_dim = ndims - 1; @@ -5112,7 +5113,7 @@ H5S__hyper_is_contiguous(const H5S_t *space) large_contiguous = TRUE; /* assume true and reset if the dimensions don't match */ small_contiguous = FALSE; /* assume false initially */ - /* Check for a "large contigous" block */ + /* Check for a "large contiguous" block */ for (u = 0; u < space->extent.rank; u++) { if (diminfo[u].count > 1) { large_contiguous = FALSE; @@ -5326,7 +5327,7 @@ done: Check if a hyperslab selection is "regular" USAGE htri_t H5S__hyper_is_regular(space) - const H5S_t *space; IN: Dataspace pointer to check + H5S_t *space; IN: Dataspace pointer to check RETURNS TRUE/FALSE/FAIL DESCRIPTION @@ -5339,7 +5340,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__hyper_is_regular(const H5S_t *space) +H5S__hyper_is_regular(H5S_t *space) { htri_t ret_value = FAIL; /* return value */ @@ -5352,7 +5353,7 @@ H5S__hyper_is_regular(const H5S_t *space) * to be impossible. */ if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_NO) - H5S__hyper_rebuild((H5S_t *)space); /* Casting away const OK -NAF */ + H5S__hyper_rebuild(space); /* Only simple check for regular hyperslabs for now... */ if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES) @@ -5576,8 +5577,8 @@ H5S__hyper_spans_shape_same(const H5S_hyper_span_info_t *span_info1, const H5S_h Check if a two hyperslab selections are the same shape USAGE htri_t H5S__hyper_shape_same(space1, space2) - const H5S_t *space1; IN: First dataspace to check - const H5S_t *space2; IN: Second dataspace to check + H5S_t *space1; IN: First dataspace to check + H5S_t *space2; IN: Second dataspace to check RETURNS TRUE / FALSE / FAIL DESCRIPTION @@ -5594,7 +5595,7 @@ H5S__hyper_spans_shape_same(const H5S_hyper_span_info_t *span_info1, const H5S_h REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__hyper_shape_same(const H5S_t *space1, const H5S_t *space2) +H5S__hyper_shape_same(H5S_t *space1, H5S_t *space2) { unsigned space1_rank; /* Number of dimensions of first dataspace */ unsigned space2_rank; /* Number of dimensions of second dataspace */ @@ -5617,9 +5618,9 @@ H5S__hyper_shape_same(const H5S_t *space1, const H5S_t *space2) /* Rebuild diminfo if it is invalid and has not been confirmed to be * impossible */ if (space1->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_NO) - H5S__hyper_rebuild((H5S_t *)space1); /* Casting away const OK -QAK */ + H5S__hyper_rebuild(space1); if (space2->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_NO) - H5S__hyper_rebuild((H5S_t *)space2); /* Casting away const OK -QAK */ + H5S__hyper_rebuild(space2); /* If both are regular hyperslabs, compare their diminfo values */ if (space1->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES && @@ -5664,11 +5665,11 @@ H5S__hyper_shape_same(const H5S_t *space1, const H5S_t *space2) /* Make certain that both selections have span trees */ if (NULL == space1->select.sel_info.hslab->span_lst) - if (H5S__hyper_generate_spans((H5S_t *)space1) < 0) /* Casting away const OK -QAK */ + if (H5S__hyper_generate_spans(space1) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, FAIL, "can't construct span tree for hyperslab selection") if (NULL == space2->select.sel_info.hslab->span_lst) - if (H5S__hyper_generate_spans((H5S_t *)space2) < 0) /* Casting away const OK -QAK */ + if (H5S__hyper_generate_spans(space2) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, FAIL, "can't construct span tree for hyperslab selection") @@ -6255,7 +6256,7 @@ done: Detect intersections of selection with block USAGE htri_t H5S__hyper_intersect_block(space, start, end) - const H5S_t *space; IN: Dataspace with selection to use + H5S_t *space; IN: Dataspace with selection to use const hsize_t *start; IN: Starting coordinate for block const hsize_t *end; IN: Ending coordinate for block RETURNS @@ -6270,7 +6271,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__hyper_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end) +H5S__hyper_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end) { htri_t ret_value = FAIL; /* Return value */ @@ -6286,7 +6287,7 @@ H5S__hyper_intersect_block(const H5S_t *space, const hsize_t *start, const hsize * to be impossible. */ if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_NO) - H5S__hyper_rebuild((H5S_t *)space); /* Casting away const OK -QAK */ + H5S__hyper_rebuild(space); /* Check for regular hyperslab intersection */ if (space->select.sel_info.hslab->diminfo_valid == H5S_DIMINFO_VALID_YES) { @@ -11516,7 +11517,7 @@ H5S__hyper_proj_int_iterate(const H5S_hyper_span_info_t *ss_span_info, udata->skip += (count - 1) * (udata->skip - old_skip); } /* end if */ else { - /* Third case: agorithm added skip and nelem (in that + /* Third case: algorithm added skip and nelem (in that * order). Add the same skip and nelem once for each item * remaining in count. */ hsize_t skip_add; @@ -11592,8 +11593,8 @@ also that proj_space can share some span trees with dst_space, so proj_space mus if dst_space must be preserved. GLOBAL VARIABLES COMMENTS, BUGS, ASSUMPTIONS EXAMPLES REVISION LOG --------------------------------------------------------------------------*/ herr_t -H5S__hyper_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, - const H5S_t *src_intersect_space, H5S_t *proj_space, hbool_t share_selection) +H5S__hyper_project_intersection(H5S_t *src_space, H5S_t *dst_space, H5S_t *src_intersect_space, + H5S_t *proj_space, hbool_t share_selection) { H5S_hyper_project_intersect_ud_t udata; /* User data for subroutines */ const H5S_hyper_span_info_t * ss_span_info; @@ -11622,7 +11623,7 @@ H5S__hyper_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, if (H5S_GET_SELECT_TYPE(src_space) == H5S_SEL_HYPERSLABS) { /* Make certain the selection has a span tree */ if (NULL == src_space->select.sel_info.hslab->span_lst) - if (H5S__hyper_generate_spans((H5S_t *)src_space) < 0) /* Casting away const OK -NAF */ + if (H5S__hyper_generate_spans(src_space) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, FAIL, "can't construct span tree for source hyperslab selection") @@ -11644,7 +11645,7 @@ H5S__hyper_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, if (H5S_GET_SELECT_TYPE(dst_space) == H5S_SEL_HYPERSLABS) { /* Make certain the selection has a span tree */ if (NULL == dst_space->select.sel_info.hslab->span_lst) - if (H5S__hyper_generate_spans((H5S_t *)dst_space) < 0) /* Casting away const OK -NAF */ + if (H5S__hyper_generate_spans(dst_space) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, FAIL, "can't construct span tree for dsetination hyperslab selection") @@ -11664,7 +11665,7 @@ H5S__hyper_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, /* Make certain the source intersect selection has a span tree */ if (NULL == src_intersect_space->select.sel_info.hslab->span_lst) - if (H5S__hyper_generate_spans((H5S_t *)src_intersect_space) < 0) /* Casting away const OK -NAF */ + if (H5S__hyper_generate_spans(src_intersect_space) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, FAIL, "can't construct span tree for source intersect hyperslab selection") @@ -12036,7 +12037,7 @@ H5S_hyper_get_clip_extent(const H5S_t *clip_space, const H5S_t *match_space, hbo hsize_t num_slices; /* Number of slices in unlimited dimension */ hsize_t ret_value = 0; /* Return value */ - FUNC_ENTER_NOAPI(0) + FUNC_ENTER_NOAPI_NOERR /* Check parameters */ HDassert(clip_space); @@ -12058,7 +12059,6 @@ H5S_hyper_get_clip_extent(const H5S_t *clip_space, const H5S_t *match_space, hbo /* Call "real" get_clip_extent function */ ret_value = H5S__hyper_get_clip_extent_real(clip_space, num_slices, incl_trail); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_hyper_get_clip_extent() */ @@ -12094,7 +12094,7 @@ H5S_hyper_get_clip_extent_match(const H5S_t *clip_space, const H5S_t *match_spac hsize_t num_slices; /* Number of slices in unlimited dimension */ hsize_t ret_value = 0; /* Return value */ - FUNC_ENTER_NOAPI(0) + FUNC_ENTER_NOAPI_NOERR /* Check parameters */ HDassert(clip_space); @@ -12140,7 +12140,6 @@ H5S_hyper_get_clip_extent_match(const H5S_t *clip_space, const H5S_t *match_spac /* Call "real" get_clip_extent function */ ret_value = H5S__hyper_get_clip_extent_real(clip_space, num_slices, incl_trail); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_hyper_get_clip_extent_match() */ @@ -12256,7 +12255,7 @@ H5S_hyper_get_first_inc_block(const H5S_t *space, hsize_t clip_size, hbool_t *pa H5S_hyper_dim_t *diminfo; /* Convenience pointer to diminfo in unlimited dimension */ hsize_t ret_value = 0; - FUNC_ENTER_NOAPI(0) + FUNC_ENTER_NOAPI_NOERR /* Check parameters */ HDassert(space); @@ -12286,7 +12285,6 @@ H5S_hyper_get_first_inc_block(const H5S_t *space, hsize_t clip_size, hbool_t *pa } /* end if */ } /* end else */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_hyper_get_first_inc_block */ diff --git a/src/H5Smpio.c b/src/H5Smpio.c index b626b77..aec5560 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -1145,7 +1145,7 @@ H5S__obtain_datatype(H5S_hyper_span_info_t *spans, const hsize_t *down, size_t e /* If this is the fastest changing dimension, it is the base case for derived datatype. */ span = spans->head; if (NULL == span->down) { - hbool_t large_block = FALSE; /* Wether the block length is larger than 32 bit integer */ + hbool_t large_block = FALSE; /* Whether the block length is larger than 32 bit integer */ outercount = 0; while (span) { diff --git a/src/H5Snone.c b/src/H5Snone.c index 82e513e..9292cd4 100644 --- a/src/H5Snone.c +++ b/src/H5Snone.c @@ -49,22 +49,22 @@ static herr_t H5S__none_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); static herr_t H5S__none_release(H5S_t *space); static htri_t H5S__none_is_valid(const H5S_t *space); -static hssize_t H5S__none_serial_size(const H5S_t *space); -static herr_t H5S__none_serialize(const H5S_t *space, uint8_t **p); +static hssize_t H5S__none_serial_size(H5S_t *space); +static herr_t H5S__none_serialize(H5S_t *space, uint8_t **p); static herr_t H5S__none_deserialize(H5S_t **space, const uint8_t **p); static herr_t H5S__none_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S__none_offset(const H5S_t *space, hsize_t *off); static int H5S__none_unlim_dim(const H5S_t *space); static htri_t H5S__none_is_contiguous(const H5S_t *space); static htri_t H5S__none_is_single(const H5S_t *space); -static htri_t H5S__none_is_regular(const H5S_t *space); -static htri_t H5S__none_shape_same(const H5S_t *space1, const H5S_t *space2); -static htri_t H5S__none_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end); +static htri_t H5S__none_is_regular(H5S_t *space); +static htri_t H5S__none_shape_same(H5S_t *space1, H5S_t *space2); +static htri_t H5S__none_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end); static herr_t H5S__none_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__none_adjust_s(H5S_t *space, const hssize_t *offset); static herr_t H5S__none_project_scalar(const H5S_t *space, hsize_t *offset); static herr_t H5S__none_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); -static herr_t H5S__none_iter_init(const H5S_t *space, H5S_sel_iter_t *iter); +static herr_t H5S__none_iter_init(H5S_t *space, H5S_sel_iter_t *iter); /* Selection iteration callbacks */ static herr_t H5S__none_iter_coords(const H5S_sel_iter_t *iter, hsize_t *coords); @@ -144,7 +144,7 @@ static const H5S_sel_iter_class_t H5S_sel_iter_none[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5S__none_iter_init(const H5S_t H5_ATTR_UNUSED *space, H5S_sel_iter_t *iter) +H5S__none_iter_init(H5S_t H5_ATTR_UNUSED *space, H5S_sel_iter_t *iter) { FUNC_ENTER_STATIC_NOERR @@ -513,7 +513,7 @@ H5S__none_is_valid(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static hssize_t -H5S__none_serial_size(const H5S_t H5_ATTR_UNUSED *space) +H5S__none_serial_size(H5S_t H5_ATTR_UNUSED *space) { FUNC_ENTER_STATIC_NOERR @@ -533,7 +533,7 @@ H5S__none_serial_size(const H5S_t H5_ATTR_UNUSED *space) Serialize the current selection into a user-provided buffer. USAGE herr_t H5S__none_serialize(space, p) - const H5S_t *space; IN: Dataspace with selection to serialize + H5S_t *space; IN: Dataspace with selection to serialize uint8_t **p; OUT: Pointer to buffer to put serialized selection. Will be advanced to end of serialized selection. @@ -548,7 +548,7 @@ H5S__none_serial_size(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S__none_serialize(const H5S_t *space, uint8_t **p) +H5S__none_serialize(H5S_t *space, uint8_t **p) { uint8_t *pp = (*p); /* Local pointer for decoding */ @@ -803,7 +803,7 @@ H5S__none_is_single(const H5S_t H5_ATTR_UNUSED *space) Check if a "none" selection is "regular" USAGE htri_t H5S__none_is_regular(space) - const H5S_t *space; IN: Dataspace pointer to check + H5S_t *space; IN: Dataspace pointer to check RETURNS TRUE/FALSE/FAIL DESCRIPTION @@ -816,7 +816,7 @@ H5S__none_is_single(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__none_is_regular(const H5S_t H5_ATTR_UNUSED *space) +H5S__none_is_regular(H5S_t H5_ATTR_UNUSED *space) { FUNC_ENTER_STATIC_NOERR @@ -833,8 +833,8 @@ H5S__none_is_regular(const H5S_t H5_ATTR_UNUSED *space) Check if a two "none" selections are the same shape USAGE htri_t H5S__none_shape_same(space1, space2) - const H5S_t *space1; IN: First dataspace to check - const H5S_t *space2; IN: Second dataspace to check + H5S_t *space1; IN: First dataspace to check + H5S_t *space2; IN: Second dataspace to check RETURNS TRUE / FALSE / FAIL DESCRIPTION @@ -846,7 +846,7 @@ H5S__none_is_regular(const H5S_t H5_ATTR_UNUSED *space) REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__none_shape_same(const H5S_t H5_ATTR_UNUSED *space1, const H5S_t H5_ATTR_UNUSED *space2) +H5S__none_shape_same(H5S_t H5_ATTR_UNUSED *space1, H5S_t H5_ATTR_UNUSED *space2) { FUNC_ENTER_STATIC_NOERR @@ -864,7 +864,7 @@ H5S__none_shape_same(const H5S_t H5_ATTR_UNUSED *space1, const H5S_t H5_ATTR_UNU Detect intersections of selection with block USAGE htri_t H5S__none_intersect_block(space, start, end) - const H5S_t *space; IN: Dataspace with selection to use + H5S_t *space; IN: Dataspace with selection to use const hsize_t *start; IN: Starting coordinate for block const hsize_t *end; IN: Ending coordinate for block RETURNS @@ -877,7 +877,7 @@ H5S__none_shape_same(const H5S_t H5_ATTR_UNUSED *space1, const H5S_t H5_ATTR_UNU REVISION LOG --------------------------------------------------------------------------*/ htri_t -H5S__none_intersect_block(const H5S_t H5_ATTR_UNUSED *space, const hsize_t H5_ATTR_UNUSED *start, +H5S__none_intersect_block(H5S_t H5_ATTR_UNUSED *space, const hsize_t H5_ATTR_UNUSED *start, const hsize_t H5_ATTR_UNUSED *end) { FUNC_ENTER_STATIC_NOERR diff --git a/src/H5Spkg.h b/src/H5Spkg.h index c89b616..afabf25 100644 --- a/src/H5Spkg.h +++ b/src/H5Spkg.h @@ -201,7 +201,7 @@ struct H5S_hyper_span_info_t { typedef enum { H5S_DIMINFO_VALID_IMPOSSIBLE, /* 0: diminfo is not valid and can never be valid with the current selection */ - H5S_DIMINFO_VALID_NO, /* 1: diminfo is not valid but may or may not be possible to constuct */ + H5S_DIMINFO_VALID_NO, /* 1: diminfo is not valid but may or may not be possible to construct */ H5S_DIMINFO_VALID_YES /* 2: diminfo is valid */ } H5S_diminfo_valid_t; @@ -242,9 +242,9 @@ typedef herr_t (*H5S_sel_release_func_t)(H5S_t *space); /* Method to determine if current selection is valid for dataspace */ typedef htri_t (*H5S_sel_is_valid_func_t)(const H5S_t *space); /* Method to determine number of bytes required to store current selection */ -typedef hssize_t (*H5S_sel_serial_size_func_t)(const H5S_t *space); +typedef hssize_t (*H5S_sel_serial_size_func_t)(H5S_t *space); /* Method to store current selection in "serialized" form (a byte sequence suitable for storing on disk) */ -typedef herr_t (*H5S_sel_serialize_func_t)(const H5S_t *space, uint8_t **p); +typedef herr_t (*H5S_sel_serialize_func_t)(H5S_t *space, uint8_t **p); /* Method to create selection from "serialized" form (a byte sequence suitable for storing on disk) */ typedef herr_t (*H5S_sel_deserialize_func_t)(H5S_t **space, const uint8_t **p); /* Method to determine smallest n-D bounding box containing the current selection */ @@ -260,12 +260,11 @@ typedef htri_t (*H5S_sel_is_contiguous_func_t)(const H5S_t *space); /* Method to determine if current selection is a single block */ typedef htri_t (*H5S_sel_is_single_func_t)(const H5S_t *space); /* Method to determine if current selection is "regular" */ -typedef htri_t (*H5S_sel_is_regular_func_t)(const H5S_t *space); +typedef htri_t (*H5S_sel_is_regular_func_t)(H5S_t *space); /* Method to determine if two dataspaces' selections are the same shape */ -typedef htri_t (*H5S_sel_shape_same_func_t)(const H5S_t *space1, const H5S_t *space2); +typedef htri_t (*H5S_sel_shape_same_func_t)(H5S_t *space1, H5S_t *space2); /* Method to determine if selection intersects a block */ -typedef htri_t (*H5S_sel_intersect_block_func_t)(const H5S_t *space, const hsize_t *start, - const hsize_t *end); +typedef htri_t (*H5S_sel_intersect_block_func_t)(H5S_t *space, const hsize_t *start, const hsize_t *end); /* Method to adjust a selection by an offset */ typedef herr_t (*H5S_sel_adjust_u_func_t)(H5S_t *space, const hsize_t *offset); /* Method to adjust a selection by an offset (signed) */ @@ -275,7 +274,7 @@ typedef herr_t (*H5S_sel_project_scalar)(const H5S_t *space, hsize_t *offset); /* Method to construct selection projection onto/into simple dataspace */ typedef herr_t (*H5S_sel_project_simple)(const H5S_t *space, H5S_t *new_space, hsize_t *offset); /* Method to initialize iterator for current selection */ -typedef herr_t (*H5S_sel_iter_init_func_t)(const H5S_t *space, H5S_sel_iter_t *sel_iter); +typedef herr_t (*H5S_sel_iter_init_func_t)(H5S_t *space, H5S_sel_iter_t *sel_iter); /* Selection class information */ typedef struct { @@ -405,9 +404,8 @@ H5_DLL herr_t H5S__extent_copy_real(H5S_extent_t *dst, const H5S_extent_t *src, H5_DLL uint64_t H5S__hyper_get_op_gen(void); H5_DLL void H5S__hyper_rebuild(H5S_t *space); H5_DLL herr_t H5S__modify_select(H5S_t *space1, H5S_seloper_t op, H5S_t *space2); -H5_DLL herr_t H5S__hyper_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, - const H5S_t *src_intersect_space, H5S_t *proj_space, - hbool_t share_space); +H5_DLL herr_t H5S__hyper_project_intersection(H5S_t *src_space, H5S_t *dst_space, H5S_t *src_intersect_space, + H5S_t *proj_space, hbool_t share_space); /* Operations on selection iterators */ H5_DLL herr_t H5S__sel_iter_close_cb(H5S_sel_iter_t *_sel_iter, void **request); diff --git a/src/H5Spoint.c b/src/H5Spoint.c index 5991116..240b722 100644 --- a/src/H5Spoint.c +++ b/src/H5Spoint.c @@ -59,22 +59,22 @@ static void H5S__free_pnt_list(H5S_pnt_list_t *pnt_lst); static herr_t H5S__point_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); static herr_t H5S__point_release(H5S_t *space); static htri_t H5S__point_is_valid(const H5S_t *space); -static hssize_t H5S__point_serial_size(const H5S_t *space); -static herr_t H5S__point_serialize(const H5S_t *space, uint8_t **p); +static hssize_t H5S__point_serial_size(H5S_t *space); +static herr_t H5S__point_serialize(H5S_t *space, uint8_t **p); static herr_t H5S__point_deserialize(H5S_t **space, const uint8_t **p); static herr_t H5S__point_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S__point_offset(const H5S_t *space, hsize_t *off); static int H5S__point_unlim_dim(const H5S_t *space); static htri_t H5S__point_is_contiguous(const H5S_t *space); static htri_t H5S__point_is_single(const H5S_t *space); -static htri_t H5S__point_is_regular(const H5S_t *space); -static htri_t H5S__point_shape_same(const H5S_t *space1, const H5S_t *space2); -static htri_t H5S__point_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end); +static htri_t H5S__point_is_regular(H5S_t *space); +static htri_t H5S__point_shape_same(H5S_t *space1, H5S_t *space2); +static htri_t H5S__point_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end); static herr_t H5S__point_adjust_u(H5S_t *space, const hsize_t *offset); static herr_t H5S__point_adjust_s(H5S_t *space, const hssize_t *offset); static herr_t H5S__point_project_scalar(const H5S_t *space, hsize_t *offset); static herr_t H5S__point_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); -static herr_t H5S__point_iter_init(const H5S_t *space, H5S_sel_iter_t *iter); +static herr_t H5S__point_iter_init(H5S_t *space, H5S_sel_iter_t *iter); static herr_t H5S__point_get_version_enc_size(const H5S_t *space, uint32_t *version, uint8_t *enc_size); /* Selection iteration callbacks */ @@ -170,7 +170,7 @@ H5FL_DEFINE_STATIC(H5S_pnt_list_t); *------------------------------------------------------------------------- */ static herr_t -H5S__point_iter_init(const H5S_t *space, H5S_sel_iter_t *iter) +H5S__point_iter_init(H5S_t *space, H5S_sel_iter_t *iter) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1159,7 +1159,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ static hssize_t -H5S__point_serial_size(const H5S_t *space) +H5S__point_serial_size(H5S_t *space) { uint32_t version; /* Version number */ uint8_t enc_size; /* Encoded size of point selection info */ @@ -1204,7 +1204,7 @@ done: Serialize the current selection into a user-provided buffer. USAGE herr_t H5S__point_serialize(space, p) - const H5S_t *space; IN: Dataspace with selection to serialize + H5S_t *space; IN: Dataspace with selection to serialize uint8_t **p; OUT: Pointer to buffer to put serialized selection. Will be advanced to end of serialized selection. @@ -1219,7 +1219,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S__point_serialize(const H5S_t *space, uint8_t **p) +H5S__point_serialize(H5S_t *space, uint8_t **p) { H5S_pnt_node_t *curr; /* Point information nodes */ uint8_t * pp; /* Local pointer for encoding */ @@ -1505,7 +1505,7 @@ done: dataspace they are located within. The list of points is formatted as follows: followed by the next coordinate, etc. until all the point information in the selection have been put into the user's buffer. - The points are returned in the order they will be interated through + The points are returned in the order they will be iterated through when a selection is read/written from/to disk. GLOBAL VARIABLES COMMENTS, BUGS, ASSUMPTIONS @@ -1579,7 +1579,7 @@ H5S__get_select_elem_pointlist(const H5S_t *space, hsize_t startpoint, hsize_t n dataspace they are located within. The list of points is formatted as follows: followed by the next coordinate, etc. until all the point information in the selection have been put into the user's buffer. - The points are returned in the order they will be interated through + The points are returned in the order they will be iterated through when a selection is read/written from/to disk. GLOBAL VARIABLES COMMENTS, BUGS, ASSUMPTIONS @@ -1844,7 +1844,7 @@ H5S__point_is_single(const H5S_t *space) Check if a point selection is "regular" USAGE htri_t H5S__point_is_regular(space) - const H5S_t *space; IN: Dataspace pointer to check + H5S_t *space; IN: Dataspace pointer to check RETURNS TRUE/FALSE/FAIL DESCRIPTION @@ -1859,7 +1859,7 @@ H5S__point_is_single(const H5S_t *space) REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__point_is_regular(const H5S_t *space) +H5S__point_is_regular(H5S_t *space) { htri_t ret_value = FAIL; /* Return value */ @@ -1884,8 +1884,8 @@ H5S__point_is_regular(const H5S_t *space) Check if a two "point" selections are the same shape USAGE htri_t H5S__point_shape_same(space1, space2) - const H5S_t *space1; IN: First dataspace to check - const H5S_t *space2; IN: Second dataspace to check + H5S_t *space1; IN: First dataspace to check + H5S_t *space2; IN: Second dataspace to check RETURNS TRUE / FALSE / FAIL DESCRIPTION @@ -1897,7 +1897,7 @@ H5S__point_is_regular(const H5S_t *space) REVISION LOG --------------------------------------------------------------------------*/ static htri_t -H5S__point_shape_same(const H5S_t *space1, const H5S_t *space2) +H5S__point_shape_same(H5S_t *space1, H5S_t *space2) { H5S_pnt_node_t *pnt1, *pnt2; /* Point information nodes */ hssize_t offset[H5S_MAX_RANK]; /* Offset between the selections */ @@ -1990,7 +1990,7 @@ done: Detect intersections of selection with block USAGE htri_t H5S__point_intersect_block(space, start, end) - const H5S_t *space; IN: Dataspace with selection to use + H5S_t *space; IN: Dataspace with selection to use const hsize_t *start; IN: Starting coordinate for block const hsize_t *end; IN: Ending coordinate for block RETURNS @@ -2003,7 +2003,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ htri_t -H5S__point_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end) +H5S__point_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end) { H5S_pnt_node_t *pnt; /* Point information node */ htri_t ret_value = FALSE; /* Return value */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 51a98a6..8a14563 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -232,36 +232,35 @@ H5_DLL herr_t H5S_extent_copy(H5S_t *dst, const H5S_t *src); /* Operations on selections */ H5_DLL herr_t H5S_select_deserialize(H5S_t **space, const uint8_t **p); H5_DLL H5S_sel_type H5S_get_select_type(const H5S_t *space); -H5_DLL herr_t H5S_select_iterate(void *buf, const H5T_t *type, const H5S_t *space, - const H5S_sel_iter_op_t *op, void *op_data); -H5_DLL herr_t H5S_select_fill(const void *fill, size_t fill_size, const H5S_t *space, void *buf); -H5_DLL htri_t H5S_select_valid(const H5S_t *space); -H5_DLL hsize_t H5S_get_select_npoints(const H5S_t *space); -H5_DLL herr_t H5S_get_select_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); -H5_DLL herr_t H5S_get_select_offset(const H5S_t *space, hsize_t *offset); -H5_DLL int H5S_get_select_unlim_dim(const H5S_t *space); -H5_DLL herr_t H5S_get_select_num_elem_non_unlim(const H5S_t *space, hsize_t *num_elem_non_unlim); -H5_DLL herr_t H5S_select_offset(H5S_t *space, const hssize_t *offset); -H5_DLL herr_t H5S_select_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); -H5_DLL htri_t H5S_select_shape_same(const H5S_t *space1, const H5S_t *space2); -H5_DLL htri_t H5S_select_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end); -H5_DLL herr_t H5S_select_construct_projection(const H5S_t *base_space, H5S_t **new_space_ptr, - unsigned new_space_rank, const void *buf, - void const **adj_buf_ptr, hsize_t element_size); -H5_DLL herr_t H5S_select_release(H5S_t *ds); -H5_DLL hssize_t H5S_select_serial_size(const H5S_t *space); -H5_DLL herr_t H5S_select_serialize(const H5S_t *space, uint8_t **p); -H5_DLL htri_t H5S_select_is_contiguous(const H5S_t *space); -H5_DLL htri_t H5S_select_is_single(const H5S_t *space); -H5_DLL htri_t H5S_select_is_regular(const H5S_t *space); -H5_DLL herr_t H5S_select_adjust_u(H5S_t *space, const hsize_t *offset); -H5_DLL herr_t H5S_select_adjust_s(H5S_t *space, const hssize_t *offset); -H5_DLL herr_t H5S_select_project_scalar(const H5S_t *space, hsize_t *offset); -H5_DLL herr_t H5S_select_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); -H5_DLL herr_t H5S_select_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, - const H5S_t *src_intersect_space, H5S_t **new_space_ptr, - hbool_t share_space); -H5_DLL herr_t H5S_select_subtract(H5S_t *space, H5S_t *subtract_space); +H5_DLL herr_t H5S_select_iterate(void *buf, const H5T_t *type, H5S_t *space, const H5S_sel_iter_op_t *op, + void *op_data); +H5_DLL herr_t H5S_select_fill(const void *fill, size_t fill_size, H5S_t *space, void *buf); +H5_DLL htri_t H5S_select_valid(const H5S_t *space); +H5_DLL hsize_t H5S_get_select_npoints(const H5S_t *space); +H5_DLL herr_t H5S_get_select_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); +H5_DLL herr_t H5S_get_select_offset(const H5S_t *space, hsize_t *offset); +H5_DLL int H5S_get_select_unlim_dim(const H5S_t *space); +H5_DLL herr_t H5S_get_select_num_elem_non_unlim(const H5S_t *space, hsize_t *num_elem_non_unlim); +H5_DLL herr_t H5S_select_offset(H5S_t *space, const hssize_t *offset); +H5_DLL herr_t H5S_select_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); +H5_DLL htri_t H5S_select_shape_same(H5S_t *space1, H5S_t *space2); +H5_DLL htri_t H5S_select_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end); +H5_DLL herr_t H5S_select_construct_projection(H5S_t *base_space, H5S_t **new_space_ptr, + unsigned new_space_rank, const void *buf, + void const **adj_buf_ptr, hsize_t element_size); +H5_DLL herr_t H5S_select_release(H5S_t *ds); +H5_DLL hssize_t H5S_select_serial_size(H5S_t *space); +H5_DLL herr_t H5S_select_serialize(H5S_t *space, uint8_t **p); +H5_DLL htri_t H5S_select_is_contiguous(const H5S_t *space); +H5_DLL htri_t H5S_select_is_single(const H5S_t *space); +H5_DLL htri_t H5S_select_is_regular(H5S_t *space); +H5_DLL herr_t H5S_select_adjust_u(H5S_t *space, const hsize_t *offset); +H5_DLL herr_t H5S_select_adjust_s(H5S_t *space, const hssize_t *offset); +H5_DLL herr_t H5S_select_project_scalar(const H5S_t *space, hsize_t *offset); +H5_DLL herr_t H5S_select_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); +H5_DLL herr_t H5S_select_project_intersection(H5S_t *src_space, H5S_t *dst_space, H5S_t *src_intersect_space, + H5S_t **new_space_ptr, hbool_t share_space); +H5_DLL herr_t H5S_select_subtract(H5S_t *space, H5S_t *subtract_space); /* Operations on all selections */ H5_DLL herr_t H5S_select_all(H5S_t *space, hbool_t rel_prev); @@ -290,8 +289,7 @@ H5_DLL H5S_t * H5S_hyper_get_unlim_block(const H5S_t *space, hsize_t block_index H5_DLL hsize_t H5S_hyper_get_first_inc_block(const H5S_t *space, hsize_t clip_size, hbool_t *partial); /* Operations on selection iterators */ -H5_DLL herr_t H5S_select_iter_init(H5S_sel_iter_t *iter, const H5S_t *space, size_t elmt_size, - unsigned flags); +H5_DLL herr_t H5S_select_iter_init(H5S_sel_iter_t *iter, H5S_t *space, size_t elmt_size, unsigned flags); H5_DLL herr_t H5S_select_iter_coords(const H5S_sel_iter_t *sel_iter, hsize_t *coords); H5_DLL hsize_t H5S_select_iter_nelmts(const H5S_sel_iter_t *sel_iter); H5_DLL herr_t H5S_select_iter_next(H5S_sel_iter_t *sel_iter, size_t nelem); diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 7a0ea3c..bcf12a0 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -304,7 +304,7 @@ done: *------------------------------------------------------------------------- */ hssize_t -H5S_select_serial_size(const H5S_t *space) +H5S_select_serial_size(H5S_t *space) { hssize_t ret_value = -1; /* Return value */ @@ -343,7 +343,7 @@ H5S_select_serial_size(const H5S_t *space) REVISION LOG --------------------------------------------------------------------------*/ herr_t -H5S_select_serialize(const H5S_t *space, uint8_t **p) +H5S_select_serialize(H5S_t *space, uint8_t **p) { herr_t ret_value = SUCCEED; /* Return value */ @@ -880,7 +880,7 @@ H5S_select_is_single(const H5S_t *space) REVISION LOG --------------------------------------------------------------------------*/ htri_t -H5S_select_is_regular(const H5S_t *space) +H5S_select_is_regular(H5S_t *space) { herr_t ret_value = FAIL; /* Return value */ @@ -1116,7 +1116,7 @@ H5S_select_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset) in the dataspace's selection. --------------------------------------------------------------------------*/ herr_t -H5S_select_iter_init(H5S_sel_iter_t *sel_iter, const H5S_t *space, size_t elmt_size, unsigned flags) +H5S_select_iter_init(H5S_sel_iter_t *sel_iter, H5S_t *space, size_t elmt_size, unsigned flags) { herr_t ret_value = FAIL; /* Return value */ @@ -1497,8 +1497,7 @@ H5S_select_iter_release(H5S_sel_iter_t *sel_iter) the selection is not modified. --------------------------------------------------------------------------*/ herr_t -H5S_select_iterate(void *buf, const H5T_t *type, const H5S_t *space, const H5S_sel_iter_op_t *op, - void *op_data) +H5S_select_iterate(void *buf, const H5T_t *type, H5S_t *space, const H5S_sel_iter_op_t *op, void *op_data) { H5S_sel_iter_t *iter = NULL; /* Selection iteration info */ hbool_t iter_init = FALSE; /* Selection iteration info has been initialized */ @@ -1739,7 +1738,7 @@ H5S_get_select_type(const H5S_t *space) REVISION LOG --------------------------------------------------------------------------*/ htri_t -H5S_select_shape_same(const H5S_t *space1, const H5S_t *space2) +H5S_select_shape_same(H5S_t *space1, H5S_t *space2) { H5S_sel_iter_t *iter_a = NULL; /* Selection a iteration info */ H5S_sel_iter_t *iter_b = NULL; /* Selection b iteration info */ @@ -1760,8 +1759,8 @@ H5S_select_shape_same(const H5S_t *space1, const H5S_t *space2) /* Check special cases if both dataspaces aren't scalar */ /* (If only one is, the number of selected points check is sufficient) */ if (space1->extent.rank > 0 && space2->extent.rank > 0) { - const H5S_t *space_a; /* Dataspace with larger rank */ - const H5S_t *space_b; /* Dataspace with smaller rank */ + H5S_t * space_a; /* Dataspace with larger rank */ + H5S_t * space_b; /* Dataspace with smaller rank */ unsigned space_a_rank; /* Number of dimensions of dataspace A */ unsigned space_b_rank; /* Number of dimensions of dataspace B */ int space_a_dim; /* Current dimension in dataspace A */ @@ -2063,7 +2062,7 @@ done: don't call it directly, use the appropriate macro defined in H5Sprivate.h. --------------------------------------------------------------------------*/ htri_t -H5S_select_intersect_block(const H5S_t *space, const hsize_t *start, const hsize_t *end) +H5S_select_intersect_block(H5S_t *space, const hsize_t *start, const hsize_t *end) { htri_t ret_value = TRUE; /* Return value */ @@ -2214,7 +2213,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ herr_t -H5S_select_construct_projection(const H5S_t *base_space, H5S_t **new_space_ptr, unsigned new_space_rank, +H5S_select_construct_projection(H5S_t *base_space, H5S_t **new_space_ptr, unsigned new_space_rank, const void *buf, void const **adj_buf_ptr, hsize_t element_size) { H5S_t * new_space = NULL; /* New dataspace constructed */ @@ -2247,7 +2246,7 @@ H5S_select_construct_projection(const H5S_t *base_space, H5S_t **new_space_ptr, if (new_space_rank == 0) { hssize_t npoints; /* Number of points selected */ - /* Retreve the number of elements selected */ + /* Retrieve the number of elements selected */ if ((npoints = (hssize_t)H5S_GET_SELECT_NPOINTS(base_space)) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "unable to get number of points selected") HDassert(npoints <= 1); @@ -2316,12 +2315,12 @@ H5S_select_construct_projection(const H5S_t *base_space, H5S_t **new_space_ptr, * true on selections of different rank iff: * * 1) the selection in the lower rank dataspace matches that - * in the dimensions with the fastest changing indicies in + * in the dimensions with the fastest changing indices in * the larger rank dataspace, and * * 2) the selection has thickness 1 in all ranks that appear * only in the higher rank dataspace (i.e. those with - * more slowly changing indicies). + * more slowly changing indices). */ if (new_space_rank > base_space_rank) { hsize_t tmp_dim_size = 1; /* Temporary dimension value, for filling arrays */ @@ -2452,7 +2451,7 @@ done: REVISION LOG --------------------------------------------------------------------------*/ herr_t -H5S_select_fill(const void *fill, size_t fill_size, const H5S_t *space, void *_buf) +H5S_select_fill(const void *fill, size_t fill_size, H5S_t *space, void *_buf) { H5S_sel_iter_t *iter = NULL; /* Selection iteration info */ hbool_t iter_init = FALSE; /* Selection iteration info has been initialized */ @@ -2568,9 +2567,8 @@ to share structures inside dst_space with proj_space REVISION LOG --------------------------------------------------------------------------*/ herr_t -H5S_select_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, - const H5S_t *src_intersect_space, H5S_t **new_space_ptr, - hbool_t share_selection) +H5S_select_project_intersection(H5S_t *src_space, H5S_t *dst_space, H5S_t *src_intersect_space, + H5S_t **new_space_ptr, hbool_t share_selection) { H5S_t * new_space = NULL; /* New dataspace constructed */ H5S_t * tmp_src_intersect_space = NULL; /* Temporary SIS converted from points->hyperslabs */ diff --git a/src/H5Stest.c b/src/H5Stest.c index a851549..6a31f3c 100644 --- a/src/H5Stest.c +++ b/src/H5Stest.c @@ -185,12 +185,13 @@ H5S__check_spans_tail_ptr(const H5S_hyper_span_info_t *span_lst) if (NULL != cur_elem->down) if ((ret_value = H5S__check_spans_tail_ptr(cur_elem->down)) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_INCONSISTENTSTATE, FAIL, - "the seletion has inconsistent tail pointers") + "the selection has inconsistent tail pointers") cur_elem = cur_elem->next; } /* end while */ if (actual_tail != span_lst->tail) - HGOTO_ERROR(H5E_DATASPACE, H5E_INCONSISTENTSTATE, FAIL, "the seletion has inconsistent tail pointers") + HGOTO_ERROR(H5E_DATASPACE, H5E_INCONSISTENTSTATE, FAIL, + "the selection has inconsistent tail pointers") done: FUNC_LEAVE_NOAPI(ret_value) @@ -231,7 +232,8 @@ H5S__check_points_tail_ptr(const H5S_pnt_list_t *pnt_lst) cur_elem = cur_elem->next; } /* end while */ if (actual_tail != pnt_lst->tail) - HGOTO_ERROR(H5E_DATASPACE, H5E_INCONSISTENTSTATE, FAIL, "the seletion has inconsistent tail pointers") + HGOTO_ERROR(H5E_DATASPACE, H5E_INCONSISTENTSTATE, FAIL, + "the selection has inconsistent tail pointers") done: FUNC_LEAVE_NOAPI(ret_value) @@ -314,7 +316,7 @@ H5S__check_internal_consistency(const H5S_t *space) if ((NULL != hslab) && (NULL != hslab->span_lst)) if (H5S__check_spans_tail_ptr(hslab->span_lst) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_INCONSISTENTSTATE, FAIL, - "the seletion has inconsistent tail pointers") + "the selection has inconsistent tail pointers") } /* end if */ else if (space->select.type->type == H5S_SEL_POINTS) { H5S_pnt_list_t *pnt_lst = space->select.sel_info.pnt_lst; @@ -322,7 +324,7 @@ H5S__check_internal_consistency(const H5S_t *space) if (NULL != pnt_lst) if (H5S__check_points_tail_ptr(pnt_lst) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_INCONSISTENTSTATE, FAIL, - "the seletion has inconsistent tail pointers") + "the selection has inconsistent tail pointers") } /* end else-if */ done: diff --git a/src/H5T.c b/src/H5T.c index 461e6b9..bbe6ce7 100644 --- a/src/H5T.c +++ b/src/H5T.c @@ -368,12 +368,9 @@ H5T_order_t H5T_native_order_g = H5T_ORDER_ERROR; /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* * Predefined data types. These are initialized at runtime in H5Tinit.c and - * by H5T__init_package() in this source file. + * by H5T_init() in this source file. * * If more of these are added, the new ones must be added to the list of * types to reset in H5T_term_package(). @@ -600,34 +597,6 @@ static const H5I_class_t H5I_DATATYPE_CLS[1] = {{ (H5I_free_t)H5T__close_cb /* Callback routine for closing objects of this class */ }}; -/* Flag indicating "top" of interface has been initialized */ -static hbool_t H5T_top_package_initialize_s = FALSE; - -/*------------------------------------------------------------------------- - * Function: H5T_init - * - * Purpose: Initialize the interface from some other package. - * - * Return: Success: non-negative - * Failure: negative - * - * Programmer: Robb Matzke - * Wednesday, December 16, 1998 - * - *------------------------------------------------------------------------- - */ -herr_t -H5T_init(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5T_init() */ - /*------------------------------------------------------------------------- * Function: H5T__init_inf * @@ -740,19 +709,17 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5T__init_inf() */ -/*-------------------------------------------------------------------------- -NAME - H5T__init_package -- Initialize interface-specific information -USAGE - herr__t H5T_init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ +/*------------------------------------------------------------------------- + * Function: H5T_init + * + * Purpose: Initialize the interface from some other layer. + * + * Return: Success: non-negative + * Failure: negative + *------------------------------------------------------------------------- + */ herr_t -H5T__init_package(void) +H5T_init(void) { H5T_t * native_schar = NULL; /* Datatype structure for native signed char */ H5T_t * native_uchar = NULL; /* Datatype structure for native unsigned char */ @@ -793,7 +760,7 @@ H5T__init_package(void) TRUE; /* Flag to indicate whether datatype was copied or allocated (for error cleanup) */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI(FAIL) /* Initialize the ID group for the file IDs */ if (H5I_register_type(H5I_DATATYPE_CLS) < 0) @@ -1433,7 +1400,7 @@ H5T__init_package(void) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to register conversion function(s)") /* Register datatype creation property class properties here. See similar - * code in H5D__init_package(), etc. for example. + * code in H5D_init(), etc. for example. */ /* Only register the default property list if it hasn't been created yet */ @@ -1446,9 +1413,6 @@ H5T__init_package(void) HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "can't insert property into class") } /* end if */ - /* Mark "top" of interface as initialized, too */ - H5T_top_package_initialize_s = TRUE; - done: /* General cleanup */ if (compound != NULL) @@ -1476,7 +1440,7 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5T__init_package() */ +} /* end H5T_init() */ /*------------------------------------------------------------------------- * Function: H5T__unlock_cb @@ -1532,183 +1496,177 @@ H5T_top_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5T_top_package_initialize_s) { - /* Unregister all conversion functions */ - if (H5T_g.path) { - int i, nprint = 0; - - for (i = 0; i < H5T_g.npaths; i++) { - H5T_path_t *path; - - path = H5T_g.path[i]; - HDassert(path); - if (path->conv.u.app_func) { - H5T__print_stats(path, &nprint /*in,out*/); - path->cdata.command = H5T_CONV_FREE; - if (path->conv.is_app) { - if ((path->conv.u.app_func)((hid_t)FAIL, (hid_t)FAIL, &(path->cdata), (size_t)0, - (size_t)0, (size_t)0, NULL, NULL, H5CX_get_dxpl()) < 0) { + /* Unregister all conversion functions */ + if (H5T_g.path) { + int i, nprint = 0; + + for (i = 0; i < H5T_g.npaths; i++) { + H5T_path_t *path; + + path = H5T_g.path[i]; + HDassert(path); + if (path->conv.u.app_func) { + H5T__print_stats(path, &nprint /*in,out*/); + path->cdata.command = H5T_CONV_FREE; + if (path->conv.is_app) { + if ((path->conv.u.app_func)((hid_t)FAIL, (hid_t)FAIL, &(path->cdata), (size_t)0, + (size_t)0, (size_t)0, NULL, NULL, H5CX_get_dxpl()) < 0) { #ifdef H5T_DEBUG - if (H5DEBUG(T)) { - HDfprintf(H5DEBUG(T), - "H5T: conversion function " - "0x%08lx failed to free private data for " - "%s (ignored)\n", - (unsigned long)(path->conv.u.app_func), path->name); - } /* end if */ + if (H5DEBUG(T)) { + HDfprintf(H5DEBUG(T), + "H5T: conversion function " + "0x%08lx failed to free private data for " + "%s (ignored)\n", + (unsigned long)(path->conv.u.app_func), path->name); + } /* end if */ #endif - H5E_clear_stack(NULL); /*ignore the error*/ - } /* end if */ - } /* end if */ - else { - if ((path->conv.u.lib_func)((hid_t)FAIL, (hid_t)FAIL, &(path->cdata), (size_t)0, - (size_t)0, (size_t)0, NULL, NULL) < 0) { + H5E_clear_stack(NULL); /*ignore the error*/ + } /* end if */ + } /* end if */ + else { + if ((path->conv.u.lib_func)((hid_t)FAIL, (hid_t)FAIL, &(path->cdata), (size_t)0, + (size_t)0, (size_t)0, NULL, NULL) < 0) { #ifdef H5T_DEBUG - if (H5DEBUG(T)) { - HDfprintf(H5DEBUG(T), - "H5T: conversion function " - "0x%08lx failed to free private data for " - "%s (ignored)\n", - (unsigned long)(path->conv.u.lib_func), path->name); - } /* end if */ + if (H5DEBUG(T)) { + HDfprintf(H5DEBUG(T), + "H5T: conversion function " + "0x%08lx failed to free private data for " + "%s (ignored)\n", + (unsigned long)(path->conv.u.lib_func), path->name); + } /* end if */ #endif - H5E_clear_stack(NULL); /*ignore the error*/ - } /* end if */ - } /* end else */ - } /* end if */ - - if (path->src) - (void)H5T_close_real(path->src); - if (path->dst) - (void)H5T_close_real(path->dst); - path = H5FL_FREE(H5T_path_t, path); - H5T_g.path[i] = NULL; - } /* end for */ - - /* Clear conversion tables */ - H5T_g.path = (H5T_path_t **)H5MM_xfree(H5T_g.path); - H5T_g.npaths = 0; - H5T_g.apaths = 0; - H5T_g.soft = (H5T_soft_t *)H5MM_xfree(H5T_g.soft); - H5T_g.nsoft = 0; - H5T_g.asoft = 0; + H5E_clear_stack(NULL); /*ignore the error*/ + } /* end if */ + } /* end else */ + } /* end if */ + + if (path->src) + (void)H5T_close_real(path->src); + if (path->dst) + (void)H5T_close_real(path->dst); + path = H5FL_FREE(H5T_path_t, path); + H5T_g.path[i] = NULL; + } /* end for */ - n++; - } /* end if */ + /* Clear conversion tables */ + H5T_g.path = (H5T_path_t **)H5MM_xfree(H5T_g.path); + H5T_g.npaths = 0; + H5T_g.apaths = 0; + H5T_g.soft = (H5T_soft_t *)H5MM_xfree(H5T_g.soft); + H5T_g.nsoft = 0; + H5T_g.asoft = 0; - /* Unlock all datatypes, then free them */ - /* note that we are ignoring the return value from H5I_iterate() */ - /* Also note that we are incrementing 'n' in the callback */ - H5I_iterate(H5I_DATATYPE, H5T__unlock_cb, &n, FALSE); - - /* Release all datatype IDs */ - if (H5I_nmembers(H5I_DATATYPE) > 0) { - (void)H5I_clear_type(H5I_DATATYPE, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - - /* Reset all the datatype IDs */ - if (H5T_IEEE_F32BE_g > 0) { - H5T_IEEE_F32BE_g = FAIL; - H5T_IEEE_F32LE_g = FAIL; - H5T_IEEE_F64BE_g = FAIL; - H5T_IEEE_F64LE_g = FAIL; - - H5T_STD_I8BE_g = FAIL; - H5T_STD_I8LE_g = FAIL; - H5T_STD_I16BE_g = FAIL; - H5T_STD_I16LE_g = FAIL; - H5T_STD_I32BE_g = FAIL; - H5T_STD_I32LE_g = FAIL; - H5T_STD_I64BE_g = FAIL; - H5T_STD_I64LE_g = FAIL; - H5T_STD_U8BE_g = FAIL; - H5T_STD_U8LE_g = FAIL; - H5T_STD_U16BE_g = FAIL; - H5T_STD_U16LE_g = FAIL; - H5T_STD_U32BE_g = FAIL; - H5T_STD_U32LE_g = FAIL; - H5T_STD_U64BE_g = FAIL; - H5T_STD_U64LE_g = FAIL; - H5T_STD_B8BE_g = FAIL; - H5T_STD_B8LE_g = FAIL; - H5T_STD_B16BE_g = FAIL; - H5T_STD_B16LE_g = FAIL; - H5T_STD_B32BE_g = FAIL; - H5T_STD_B32LE_g = FAIL; - H5T_STD_B64BE_g = FAIL; - H5T_STD_B64LE_g = FAIL; - H5T_STD_REF_OBJ_g = FAIL; - H5T_STD_REF_DSETREG_g = FAIL; - H5T_STD_REF_g = FAIL; - - H5T_UNIX_D32BE_g = FAIL; - H5T_UNIX_D32LE_g = FAIL; - H5T_UNIX_D64BE_g = FAIL; - H5T_UNIX_D64LE_g = FAIL; - - H5T_C_S1_g = FAIL; - - H5T_FORTRAN_S1_g = FAIL; - - H5T_NATIVE_SCHAR_g = FAIL; - H5T_NATIVE_UCHAR_g = FAIL; - H5T_NATIVE_SHORT_g = FAIL; - H5T_NATIVE_USHORT_g = FAIL; - H5T_NATIVE_INT_g = FAIL; - H5T_NATIVE_UINT_g = FAIL; - H5T_NATIVE_LONG_g = FAIL; - H5T_NATIVE_ULONG_g = FAIL; - H5T_NATIVE_LLONG_g = FAIL; - H5T_NATIVE_ULLONG_g = FAIL; - H5T_NATIVE_FLOAT_g = FAIL; - H5T_NATIVE_DOUBLE_g = FAIL; - H5T_NATIVE_LDOUBLE_g = FAIL; - H5T_NATIVE_B8_g = FAIL; - H5T_NATIVE_B16_g = FAIL; - H5T_NATIVE_B32_g = FAIL; - H5T_NATIVE_B64_g = FAIL; - H5T_NATIVE_OPAQUE_g = FAIL; - H5T_NATIVE_HADDR_g = FAIL; - H5T_NATIVE_HSIZE_g = FAIL; - H5T_NATIVE_HSSIZE_g = FAIL; - H5T_NATIVE_HERR_g = FAIL; - H5T_NATIVE_HBOOL_g = FAIL; - - H5T_NATIVE_INT8_g = FAIL; - H5T_NATIVE_UINT8_g = FAIL; - H5T_NATIVE_INT_LEAST8_g = FAIL; - H5T_NATIVE_UINT_LEAST8_g = FAIL; - H5T_NATIVE_INT_FAST8_g = FAIL; - H5T_NATIVE_UINT_FAST8_g = FAIL; - - H5T_NATIVE_INT16_g = FAIL; - H5T_NATIVE_UINT16_g = FAIL; - H5T_NATIVE_INT_LEAST16_g = FAIL; - H5T_NATIVE_UINT_LEAST16_g = FAIL; - H5T_NATIVE_INT_FAST16_g = FAIL; - H5T_NATIVE_UINT_FAST16_g = FAIL; - - H5T_NATIVE_INT32_g = FAIL; - H5T_NATIVE_UINT32_g = FAIL; - H5T_NATIVE_INT_LEAST32_g = FAIL; - H5T_NATIVE_UINT_LEAST32_g = FAIL; - H5T_NATIVE_INT_FAST32_g = FAIL; - H5T_NATIVE_UINT_FAST32_g = FAIL; - - H5T_NATIVE_INT64_g = FAIL; - H5T_NATIVE_UINT64_g = FAIL; - H5T_NATIVE_INT_LEAST64_g = FAIL; - H5T_NATIVE_UINT_LEAST64_g = FAIL; - H5T_NATIVE_INT_FAST64_g = FAIL; - H5T_NATIVE_UINT_FAST64_g = FAIL; - - n++; - } /* end if */ + n++; + } /* end if */ - /* Mark "top" of interface as closed */ - if (0 == n) - H5T_top_package_initialize_s = FALSE; + /* Unlock all datatypes, then free them */ + /* note that we are ignoring the return value from H5I_iterate() */ + /* Also note that we are incrementing 'n' in the callback */ + H5I_iterate(H5I_DATATYPE, H5T__unlock_cb, &n, FALSE); + + /* Release all datatype IDs */ + if (H5I_nmembers(H5I_DATATYPE) > 0) { + (void)H5I_clear_type(H5I_DATATYPE, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + + /* Reset all the datatype IDs */ + if (H5T_IEEE_F32BE_g > 0) { + H5T_IEEE_F32BE_g = FAIL; + H5T_IEEE_F32LE_g = FAIL; + H5T_IEEE_F64BE_g = FAIL; + H5T_IEEE_F64LE_g = FAIL; + + H5T_STD_I8BE_g = FAIL; + H5T_STD_I8LE_g = FAIL; + H5T_STD_I16BE_g = FAIL; + H5T_STD_I16LE_g = FAIL; + H5T_STD_I32BE_g = FAIL; + H5T_STD_I32LE_g = FAIL; + H5T_STD_I64BE_g = FAIL; + H5T_STD_I64LE_g = FAIL; + H5T_STD_U8BE_g = FAIL; + H5T_STD_U8LE_g = FAIL; + H5T_STD_U16BE_g = FAIL; + H5T_STD_U16LE_g = FAIL; + H5T_STD_U32BE_g = FAIL; + H5T_STD_U32LE_g = FAIL; + H5T_STD_U64BE_g = FAIL; + H5T_STD_U64LE_g = FAIL; + H5T_STD_B8BE_g = FAIL; + H5T_STD_B8LE_g = FAIL; + H5T_STD_B16BE_g = FAIL; + H5T_STD_B16LE_g = FAIL; + H5T_STD_B32BE_g = FAIL; + H5T_STD_B32LE_g = FAIL; + H5T_STD_B64BE_g = FAIL; + H5T_STD_B64LE_g = FAIL; + H5T_STD_REF_OBJ_g = FAIL; + H5T_STD_REF_DSETREG_g = FAIL; + H5T_STD_REF_g = FAIL; + + H5T_UNIX_D32BE_g = FAIL; + H5T_UNIX_D32LE_g = FAIL; + H5T_UNIX_D64BE_g = FAIL; + H5T_UNIX_D64LE_g = FAIL; + + H5T_C_S1_g = FAIL; + + H5T_FORTRAN_S1_g = FAIL; + + H5T_NATIVE_SCHAR_g = FAIL; + H5T_NATIVE_UCHAR_g = FAIL; + H5T_NATIVE_SHORT_g = FAIL; + H5T_NATIVE_USHORT_g = FAIL; + H5T_NATIVE_INT_g = FAIL; + H5T_NATIVE_UINT_g = FAIL; + H5T_NATIVE_LONG_g = FAIL; + H5T_NATIVE_ULONG_g = FAIL; + H5T_NATIVE_LLONG_g = FAIL; + H5T_NATIVE_ULLONG_g = FAIL; + H5T_NATIVE_FLOAT_g = FAIL; + H5T_NATIVE_DOUBLE_g = FAIL; + H5T_NATIVE_LDOUBLE_g = FAIL; + H5T_NATIVE_B8_g = FAIL; + H5T_NATIVE_B16_g = FAIL; + H5T_NATIVE_B32_g = FAIL; + H5T_NATIVE_B64_g = FAIL; + H5T_NATIVE_OPAQUE_g = FAIL; + H5T_NATIVE_HADDR_g = FAIL; + H5T_NATIVE_HSIZE_g = FAIL; + H5T_NATIVE_HSSIZE_g = FAIL; + H5T_NATIVE_HERR_g = FAIL; + H5T_NATIVE_HBOOL_g = FAIL; + + H5T_NATIVE_INT8_g = FAIL; + H5T_NATIVE_UINT8_g = FAIL; + H5T_NATIVE_INT_LEAST8_g = FAIL; + H5T_NATIVE_UINT_LEAST8_g = FAIL; + H5T_NATIVE_INT_FAST8_g = FAIL; + H5T_NATIVE_UINT_FAST8_g = FAIL; + + H5T_NATIVE_INT16_g = FAIL; + H5T_NATIVE_UINT16_g = FAIL; + H5T_NATIVE_INT_LEAST16_g = FAIL; + H5T_NATIVE_UINT_LEAST16_g = FAIL; + H5T_NATIVE_INT_FAST16_g = FAIL; + H5T_NATIVE_UINT_FAST16_g = FAIL; + + H5T_NATIVE_INT32_g = FAIL; + H5T_NATIVE_UINT32_g = FAIL; + H5T_NATIVE_INT_LEAST32_g = FAIL; + H5T_NATIVE_UINT_LEAST32_g = FAIL; + H5T_NATIVE_INT_FAST32_g = FAIL; + H5T_NATIVE_UINT_FAST32_g = FAIL; + + H5T_NATIVE_INT64_g = FAIL; + H5T_NATIVE_UINT64_g = FAIL; + H5T_NATIVE_INT_LEAST64_g = FAIL; + H5T_NATIVE_UINT_LEAST64_g = FAIL; + H5T_NATIVE_INT_FAST64_g = FAIL; + H5T_NATIVE_UINT_FAST64_g = FAIL; + + n++; } /* end if */ FUNC_LEAVE_NOAPI(n) @@ -1739,18 +1697,11 @@ H5T_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - /* Sanity check */ - HDassert(0 == H5I_nmembers(H5I_DATATYPE)); - HDassert(FALSE == H5T_top_package_initialize_s); - - /* Destroy the datatype object id group */ - n += (H5I_dec_type_ref(H5I_DATATYPE) > 0); + /* Sanity check */ + HDassert(0 == H5I_nmembers(H5I_DATATYPE)); - /* Mark interface as closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ + /* Destroy the datatype object id group */ + n += (H5I_dec_type_ref(H5I_DATATYPE) > 0); FUNC_LEAVE_NOAPI(n) } /* end H5T_term_package() */ @@ -2170,7 +2121,7 @@ H5T_get_class(const H5T_t *dt, htri_t internal) { H5T_class_t ret_value = H5T_NO_CLASS; /* Return value */ - FUNC_ENTER_NOAPI(H5T_NO_CLASS) + FUNC_ENTER_NOAPI_NOERR HDassert(dt); @@ -2185,7 +2136,6 @@ H5T_get_class(const H5T_t *dt, htri_t internal) ret_value = dt->shared->type; } -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5T_get_class() */ @@ -2243,7 +2193,7 @@ H5T_detect_class(const H5T_t *dt, H5T_class_t cls, hbool_t from_api) unsigned i; htri_t ret_value = FALSE; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(dt); HDassert(cls > H5T_NO_CLASS && cls < H5T_NCLASSES); @@ -4276,7 +4226,7 @@ done: * * Return: Success: non-negative * - * Failure: nagative + * Failure: negative * * Programmer: Robb Matzke * Tuesday, December 22, 1998 @@ -5626,14 +5576,13 @@ H5T_is_immutable(const H5T_t *dt) { htri_t ret_value = FALSE; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(dt); if (dt->shared->state == H5T_STATE_IMMUTABLE) ret_value = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } @@ -5651,7 +5600,7 @@ H5T_is_named(const H5T_t *dt) { htri_t ret_value = FALSE; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(dt); @@ -5660,7 +5609,6 @@ H5T_is_named(const H5T_t *dt) else ret_value = (H5T_STATE_OPEN == dt->shared->state || H5T_STATE_NAMED == dt->shared->state); -done: FUNC_LEAVE_NOAPI(ret_value) } @@ -5738,14 +5686,13 @@ H5T_get_ref_type(const H5T_t *dt) { H5R_type_t ret_value = H5R_BADTYPE; - FUNC_ENTER_NOAPI(H5R_BADTYPE) + FUNC_ENTER_NOAPI_NOERR HDassert(dt); if (dt->shared->type == H5T_REFERENCE) ret_value = dt->shared->u.atomic.u.r.rtype; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5T_get_ref_type() */ @@ -5768,7 +5715,7 @@ H5T_is_sensible(const H5T_t *dt) { htri_t ret_value = FAIL; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(dt); @@ -5806,7 +5753,6 @@ H5T_is_sensible(const H5T_t *dt) break; } /* end switch */ -done: FUNC_LEAVE_NOAPI(ret_value) } @@ -5999,7 +5945,7 @@ H5T_is_relocatable(const H5T_t *dt) { htri_t ret_value = FALSE; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ HDassert(dt); @@ -6008,7 +5954,6 @@ H5T_is_relocatable(const H5T_t *dt) if (H5T_detect_class(dt, H5T_VLEN, FALSE) || H5T_detect_class(dt, H5T_REFERENCE, FALSE)) ret_value = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5T_is_relocatable() */ @@ -6099,7 +6044,7 @@ H5T_is_vl_storage(const H5T_t *dt) { htri_t ret_value = FALSE; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ HDassert(dt); @@ -6112,7 +6057,6 @@ H5T_is_vl_storage(const H5T_t *dt) else ret_value = FALSE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5T_is_vl_storage() */ @@ -6266,7 +6210,7 @@ H5T_patch_file(H5T_t *dt, H5F_t *f) { herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ HDassert(dt); @@ -6277,7 +6221,6 @@ H5T_patch_file(H5T_t *dt, H5F_t *f) dt->sh_loc.file = f; } /* end if */ -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5T_patch_file() */ diff --git a/src/H5TS.c b/src/H5TS.c index 258c9df..20aa283 100644 --- a/src/H5TS.c +++ b/src/H5TS.c @@ -43,7 +43,7 @@ /* Local Typedefs */ /******************/ -/* Cancelability structure */ +/* Cancellability structure */ typedef struct H5TS_cancel_struct { int previous_state; unsigned int cancel_count; @@ -685,9 +685,9 @@ H5TSmutex_release(unsigned int *lock_count) * Creates a cancellation counter for a thread if it is the first time * the thread is entering the library. * - * if counter value is zero, then set cancelability type of the thread + * if counter value is zero, then set cancellability type of the thread * to PTHREAD_CANCEL_DISABLE as thread is entering the library and store - * the previous cancelability type into cancellation counter. + * the previous cancellability type into cancellation counter. * Increase the counter value by 1. * * PROGRAMMER: Chee Wai LEE @@ -758,8 +758,8 @@ done: * 0 on success and a non-zero error code on error. * * DESCRIPTION - * If counter value is one, then set cancelability type of the thread - * to the previous cancelability type stored in the cancellation counter. + * If counter value is one, then set cancellability type of the thread + * to the previous cancellability type stored in the cancellation counter. * (the thread is leaving the library). * * Decrement the counter value by 1. diff --git a/src/H5Tbit.c b/src/H5Tbit.c index 04e2a97..20f3855 100644 --- a/src/H5Tbit.c +++ b/src/H5Tbit.c @@ -228,7 +228,7 @@ done: * Purpose: Return a small bit sequence as a number. Bit vector starts * at OFFSET and is SIZE bits long. * - * Return: The bit sequence interpretted as an unsigned integer + * Return: The bit sequence interpreted as an unsigned integer * *------------------------------------------------------------------------- */ @@ -538,7 +538,7 @@ H5T__bit_inc(uint8_t *buf, size_t start, size_t size) /*------------------------------------------------------------------------- * Function: H5T__bit_dec * - * Purpose: Decrement part of a bit field by substracting 1. The bit + * Purpose: Decrement part of a bit field by subtracting 1. The bit * field starts with bit position START and is SIZE bits long. * * Return: The "borrow-in" value. It's TRUE if underflows, FALSE @@ -564,9 +564,9 @@ H5T__bit_dec(uint8_t *buf, size_t start, size_t size) if ((size + start - 1) / 8 > idx) { /* The bit sequence doesn't end in the same byte as starts */ - /* Example: a sequence like 11000100 and start = 3. We substract 00001000 from + /* Example: a sequence like 11000100 and start = 3. We subtract 00001000 from * it and get 10111100. If a sequence is 00000111, we do right shift for START - * bits and get 00000000. So we need to borrow from higher byte when we substract + * bits and get 00000000. So we need to borrow from higher byte when we subtract * 00001000. */ if (!(buf[idx] >> pos)) @@ -597,7 +597,7 @@ H5T__bit_dec(uint8_t *buf, size_t start, size_t size) else { /* The bit sequence ends in the same byte as starts */ - /* Example: a sequence like 11000100 and pos=3, size=3. We substract 00001000 + /* Example: a sequence like 11000100 and pos=3, size=3. We subtract 00001000 * and get 10111100. A bit is borrowed from 6th bit(buf[idx]>>6=00000010, tmp>>6=00000011, * not equal). We need to put this bit back by increment 1000000. */ diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index d079e71..a3a1aa0 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -1127,7 +1127,7 @@ H5T_open(const H5G_loc_t *loc) done: if (ret_value == NULL) { if (dt) { - if (shared_fo == NULL) { /* Need to free shared fo */ + if (shared_fo == NULL) { /* Need to free shared file object */ if (dt->shared->owned_vol_obj && H5VL_free_object(dt->shared->owned_vol_obj) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, NULL, "unable to close owned VOL object") dt->shared = H5FL_FREE(H5T_shared_t, dt->shared); diff --git a/src/H5Tconv.c b/src/H5Tconv.c index c06c895..5efff10 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -2753,7 +2753,7 @@ H5T__conv_enum_init(H5T_t *src, H5T_t *dst, H5T_cdata_t *cdata) HDassert(domain[1] >= domain[0]); length = (unsigned)(domain[1] - domain[0]) + 1; if (src->shared->u.enumer.nmembs < 2 || - (double)length / src->shared->u.enumer.nmembs < (double)(1.2f)) { + (double)length / src->shared->u.enumer.nmembs < (double)(1.2F)) { priv->base = domain[0]; priv->length = length; if (NULL == (map = (int *)H5MM_malloc(length * sizeof(int)))) @@ -3205,14 +3205,14 @@ H5T__conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, si HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy src type for conversion") /* References need to know about the src file */ if (tsrc_cpy->shared->type == H5T_REFERENCE) - if (H5T_set_loc(tsrc_cpy, src->shared->u.vlen.file, H5T_LOC_MEMORY) < 0) + if (H5T_set_loc(tsrc_cpy, src->shared->u.vlen.file, src->shared->u.vlen.loc) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, FAIL, "can't set datatype location"); if (NULL == (tdst_cpy = H5T_copy(dst->shared->parent, H5T_COPY_ALL))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy dst type for conversion") /* References need to know about the dst file */ if (tdst_cpy->shared->type == H5T_REFERENCE) - if (H5T_set_loc(tdst_cpy, dst->shared->u.vlen.file, H5T_LOC_MEMORY) < 0) + if (H5T_set_loc(tdst_cpy, dst->shared->u.vlen.file, dst->shared->u.vlen.loc) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, FAIL, "can't set datatype location"); if (((tsrc_id = H5I_register(H5I_DATATYPE, tsrc_cpy, FALSE)) < 0) || @@ -4867,7 +4867,7 @@ H5T__conv_s_s(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, siz size_t olap; /*num overlapping elements */ size_t nchars = 0; /*number of characters copied */ uint8_t *s, *sp, *d, *dp; /*src and dst traversal pointers*/ - uint8_t *dbuf = NULL; /*temp buf for overlap convers. */ + uint8_t *dbuf = NULL; /*temp buf for overlap converts. */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -7145,7 +7145,7 @@ H5T__conv_schar_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7205,7 +7205,7 @@ H5T__conv_uchar_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7265,7 +7265,7 @@ H5T__conv_short_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7325,7 +7325,7 @@ H5T__conv_ushort_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7385,7 +7385,7 @@ H5T__conv_int_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelm * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7445,7 +7445,7 @@ H5T__conv_uint_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nel * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7505,7 +7505,7 @@ H5T__conv_long_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nel * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7565,7 +7565,7 @@ H5T__conv_ulong_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7625,7 +7625,7 @@ H5T__conv_llong_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7687,7 +7687,7 @@ H5T__conv_ullong_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7797,7 +7797,7 @@ H5T__conv_double_uchar(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7819,7 +7819,7 @@ H5T__conv_ldouble_schar(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7929,7 +7929,7 @@ H5T__conv_double_ushort(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -7951,7 +7951,7 @@ H5T__conv_ldouble_short(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -8061,7 +8061,7 @@ H5T__conv_double_uint(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nel * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -8083,7 +8083,7 @@ H5T__conv_ldouble_int(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nel * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -8193,7 +8193,7 @@ H5T__conv_double_ulong(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -8215,7 +8215,7 @@ H5T__conv_ldouble_long(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -8325,7 +8325,7 @@ H5T__conv_double_ullong(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ @@ -8349,7 +8349,7 @@ H5T__conv_ldouble_llong(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * Return: Non-negative on success/Negative on failure * * Programmer: Raymond Lu - * Tuesday, Febuary 1, 2005 + * Tuesday, February 1, 2005 * *------------------------------------------------------------------------- */ diff --git a/src/H5Tfields.c b/src/H5Tfields.c index baeae6d..68bc0a3 100644 --- a/src/H5Tfields.c +++ b/src/H5Tfields.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Module Info: This module contains commond functionality for fields in + * Module Info: This module contains command functionality for fields in * enumerated & compound datatypes in the H5T interface. */ diff --git a/src/H5Tnative.c b/src/H5Tnative.c index c9a2907..e6fab51 100644 --- a/src/H5Tnative.c +++ b/src/H5Tnative.c @@ -43,7 +43,7 @@ static herr_t H5T__cmp_offset(size_t *comp_size, size_t *offset, size_t elem_siz * * Purpose: High-level API to return the native type of a datatype. * The native type is chosen by matching the size and class of - * querried datatype from the following native premitive + * queried datatype from the following native primitive * datatypes: * H5T_NATIVE_CHAR H5T_NATIVE_UCHAR * H5T_NATIVE_SHORT H5T_NATIVE_USHORT @@ -56,7 +56,7 @@ static herr_t H5T__cmp_offset(size_t *comp_size, size_t *offset, size_t elem_siz * H5T_NATIVE_LDOUBLE * * Compound, array, enum, and VL types all choose among these - * types for theire members. Time, Bifield, Opaque, Reference + * types for their members. Time, Bitfield, Opaque, Reference * types are only copy out. * * Return: Success: Returns the native data type if successful. @@ -696,7 +696,7 @@ H5_GCC_DIAG_OFF("duplicated-branches") /*------------------------------------------------------------------------- * Function: H5T__get_native_float * - * Purpose: Returns the native floatt type of a datatype. + * Purpose: Returns the native float type of a datatype. * * Return: Success: Returns the native data type if successful. * diff --git a/src/H5Topaque.c b/src/H5Topaque.c index f097fca..c5c2788 100644 --- a/src/H5Topaque.c +++ b/src/H5Topaque.c @@ -70,7 +70,7 @@ done: /*------------------------------------------------------------------------- * Function: H5Tget_tag * - * Purpose: Get tha tag associated with an opaque datatype. + * Purpose: Get the tag associated with an opaque datatype. * * Return: A pointer to an allocated string. The caller should free * the string. NULL is returned for errors. diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index 19593dd..4062cbe 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -123,7 +123,7 @@ #endif /* Define an internal macro for converting unsigned long long to long double. SGI compilers give - * some incorect conversion. 64-bit Solaris does different rounding. Windows Visual Studio 6 does + * some incorrect conversion. 64-bit Solaris does different rounding. Windows Visual Studio 6 does * not support unsigned long long. For FreeBSD(sleipnir), the last 2 bytes of mantissa are lost when * compiler tries to do the conversion. For Cygwin, compiler doesn't do rounding correctly. * Mac OS 10.4 gives some incorrect result. */ diff --git a/src/H5Tpublic.h b/src/H5Tpublic.h index 911efdf..3df7ca2 100644 --- a/src/H5Tpublic.h +++ b/src/H5Tpublic.h @@ -148,7 +148,7 @@ typedef enum H5T_pad_t { H5T_PAD_ONE = 1, /**< always set to one */ H5T_PAD_BACKGROUND = 2, /**< set to background value */ - H5T_NPAD = 3 /**< sentinal: THIS MUST BE LAST */ + H5T_NPAD = 3 /**< sentinel: THIS MUST BE LAST */ } H5T_pad_t; //! @@ -1161,7 +1161,7 @@ H5_DLL herr_t H5Tlock(hid_t type_id); * the link(s) by which the new committed datatype is accessed and * the creation of any intermediate groups that may be missing. * - * Once commited, this datatype may be used to define the datatype + * Once committed, this datatype may be used to define the datatype * of any other dataset or attribute in the file. * * This function will not accept a datatype that cannot actually hold @@ -1171,7 +1171,7 @@ H5_DLL herr_t H5Tlock(hid_t type_id); * Committed datatypes are sometimes referred to as named datatypes. * * \version 1.8.7 Function modified in this release to reject datatypes that - * will not accomodate actual data, such as a compound datatype + * will not accommodate actual data, such as a compound datatype * with no fields or an enumerated datatype with no members. * * \since 1.8.0 @@ -1266,7 +1266,7 @@ H5_DLL hid_t H5Topen_async(const char *app_file, const char *app_func, unsigned * fields and enumerated datatypes with no members. * * \version 1.8.7 Function modified in this release to reject datatypes that - * will not accomodate actual data, such as a compound datatype + * will not accommodate actual data, such as a compound datatype * with no fields or an enumerated datatype with no members. * * \since 1.2.0 diff --git a/src/H5VLcallback.c b/src/H5VLcallback.c index fcab19f..0c5c73d 100644 --- a/src/H5VLcallback.c +++ b/src/H5VLcallback.c @@ -4967,7 +4967,7 @@ done: /*------------------------------------------------------------------------- * Function: H5VL__link_copy * - * Purpose: Copys a link from src to dst. + * Purpose: Copies a link from src to dst. * * Return: Success: Non-negative * Failure: Negative @@ -4998,7 +4998,7 @@ done: /*------------------------------------------------------------------------- * Function: H5VL_link_copy * - * Purpose: Copys a link from src to dst. + * Purpose: Copies a link from src to dst. * * Return: Success: Non-negative * Failure: Negative @@ -6408,7 +6408,7 @@ done: /*------------------------------------------------------------------------- * Function: H5VL__request_wait * - * Purpose: Waits on an asychronous request through the VOL + * Purpose: Waits on an asynchronous request through the VOL * * Return: Success: Non-negative * Failure: Negative @@ -6442,7 +6442,7 @@ done: /*------------------------------------------------------------------------- * Function: H5VL_request_wait * - * Purpose: Waits on an asychronous request through the VOL + * Purpose: Waits on an asynchronous request through the VOL * * Return: Success: Non-negative * Failure: Negative @@ -6452,18 +6452,28 @@ done: herr_t H5VL_request_wait(const H5VL_object_t *vol_obj, uint64_t timeout, H5VL_request_status_t *status) { - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t vol_wrapper_set = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ HDassert(vol_obj); + /* Set wrapper info in API context */ + if (H5VL_set_vol_wrapper(vol_obj) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set VOL wrapper info") + vol_wrapper_set = TRUE; + /* Call the corresponding internal VOL routine */ if (H5VL__request_wait(vol_obj->data, vol_obj->connector->cls, timeout, status) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "request wait failed") done: + /* Reset object wrapping info in API context */ + if (vol_wrapper_set && H5VL_reset_vol_wrapper() < 0) + HDONE_ERROR(H5E_VOL, H5E_CANTRESET, FAIL, "can't reset VOL wrapper info") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_request_wait() */ @@ -6546,18 +6556,28 @@ done: herr_t H5VL_request_notify(const H5VL_object_t *vol_obj, H5VL_request_notify_t cb, void *ctx) { - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t vol_wrapper_set = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity check */ HDassert(vol_obj); + /* Set wrapper info in API context */ + if (H5VL_set_vol_wrapper(vol_obj) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set VOL wrapper info") + vol_wrapper_set = TRUE; + /* Call the corresponding internal VOL routine */ if (H5VL__request_notify(vol_obj->data, vol_obj->connector->cls, cb, ctx) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "request notify failed") done: + /* Reset object wrapping info in API context */ + if (vol_wrapper_set && H5VL_reset_vol_wrapper() < 0) + HDONE_ERROR(H5E_VOL, H5E_CANTRESET, FAIL, "can't reset VOL wrapper info") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_request_notify() */ @@ -6639,18 +6659,28 @@ done: herr_t H5VL_request_cancel(const H5VL_object_t *vol_obj, H5VL_request_status_t *status) { - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t vol_wrapper_set = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity check */ HDassert(vol_obj); + /* Set wrapper info in API context */ + if (H5VL_set_vol_wrapper(vol_obj) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set VOL wrapper info") + vol_wrapper_set = TRUE; + /* Call the corresponding internal VOL routine */ if (H5VL__request_cancel(vol_obj->data, vol_obj->connector->cls, status) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "request cancel failed") done: + /* Reset object wrapping info in API context */ + if (vol_wrapper_set && H5VL_reset_vol_wrapper() < 0) + HDONE_ERROR(H5E_VOL, H5E_CANTRESET, FAIL, "can't reset VOL wrapper info") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_request_cancel() */ @@ -6732,19 +6762,29 @@ done: herr_t H5VL_request_specific(const H5VL_object_t *vol_obj, H5VL_request_specific_args_t *args) { - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t vol_wrapper_set = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity check */ HDassert(vol_obj); + /* Set wrapper info in API context */ + if (H5VL_set_vol_wrapper(vol_obj) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set VOL wrapper info") + vol_wrapper_set = TRUE; + /* Call the corresponding internal VOL routine */ if (H5VL__request_specific(vol_obj->data, vol_obj->connector->cls, args) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute asynchronous request specific callback") done: + /* Reset object wrapping info in API context */ + if (vol_wrapper_set && H5VL_reset_vol_wrapper() < 0) + HDONE_ERROR(H5E_VOL, H5E_CANTRESET, FAIL, "can't reset VOL wrapper info") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_request_specific() */ @@ -6827,19 +6867,29 @@ done: herr_t H5VL_request_optional(const H5VL_object_t *vol_obj, H5VL_optional_args_t *args) { - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t vol_wrapper_set = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity check */ HDassert(vol_obj); + /* Set wrapper info in API context */ + if (H5VL_set_vol_wrapper(vol_obj) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set VOL wrapper info") + vol_wrapper_set = TRUE; + /* Call the corresponding internal VOL routine */ if (H5VL__request_optional(vol_obj->data, vol_obj->connector->cls, args) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTOPERATE, FAIL, "unable to execute asynchronous request optional callback") done: + /* Reset object wrapping info in API context */ + if (vol_wrapper_set && H5VL_reset_vol_wrapper() < 0) + HDONE_ERROR(H5E_VOL, H5E_CANTRESET, FAIL, "can't reset VOL wrapper info") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_request_optional() */ @@ -6958,18 +7008,28 @@ done: herr_t H5VL_request_free(const H5VL_object_t *vol_obj) { - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t vol_wrapper_set = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity check */ HDassert(vol_obj); + /* Set wrapper info in API context */ + if (H5VL_set_vol_wrapper(vol_obj) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTSET, FAIL, "can't set VOL wrapper info") + vol_wrapper_set = TRUE; + /* Call the corresponding VOL callback */ if (H5VL__request_free(vol_obj->data, vol_obj->connector->cls) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTRELEASE, FAIL, "request free failed") done: + /* Reset object wrapping info in API context */ + if (vol_wrapper_set && H5VL_reset_vol_wrapper() < 0) + HDONE_ERROR(H5E_VOL, H5E_CANTRESET, FAIL, "can't reset VOL wrapper info") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_request_free() */ diff --git a/src/H5VLint.c b/src/H5VLint.c index 70c8112..391bd91 100644 --- a/src/H5VLint.c +++ b/src/H5VLint.c @@ -32,6 +32,7 @@ #include "H5CXprivate.h" /* API Contexts */ #include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ +#include "H5ESprivate.h" /* Event sets */ #include "H5Fprivate.h" /* Files */ #include "H5FLprivate.h" /* Free lists */ #include "H5Gprivate.h" /* Groups */ @@ -69,11 +70,7 @@ typedef struct H5VL_wrap_ctx_t { */ typedef struct { /* IN */ - H5VL_get_connector_kind_t kind; /* Which kind of connector search to make */ - union { - const char * name; /* The name of the VOL connector to check */ - H5VL_class_value_t value; /* The value of the VOL connector to check */ - } u; + H5PL_vol_key_t key; /* OUT */ hid_t found_id; /* The connector ID, if we found a match */ @@ -98,9 +95,6 @@ static herr_t H5VL__free_vol_wrapper(H5VL_wrap_ctx_t *vol_wrap_ctx); /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -153,7 +147,9 @@ H5VL_init_phase1(void) FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ + /* Initialize the ID group for the VL IDs */ + if (H5I_register_type(H5I_VOL_CLS) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize H5VL interface") done: FUNC_LEAVE_NOAPI(ret_value) @@ -175,23 +171,38 @@ done: herr_t H5VL_init_phase2(void) { + size_t i; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) + /* clang-format off */ + struct { + herr_t (*func)(void); + const char *descr; + } initializer[] = { + {H5T_init, "datatype"} + , {H5O_init, "object header"} + , {H5D_init, "dataset"} + , {H5F_init, "file"} + , {H5G_init, "group"} + , {H5A_init, "attribute"} + , {H5M_init, "map"} + , {H5CX_init, "context"} + , {H5ES_init, "event set"} + , {H5Z_init, "transform"} + , {H5R_init, "reference"} + }; + /* Initialize all packages for VOL-managed objects */ - if (H5T_init() < 0) - HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize datatype interface") - if (H5D_init() < 0) - HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize dataset interface") - if (H5F_init() < 0) - HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize file interface") - if (H5G_init() < 0) - HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize group interface") - if (H5A_init() < 0) - HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize attribute interface") - if (H5M_init() < 0) - HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize map interface") + for (i = 0; i < NELMTS(initializer); i++) { + if (initializer[i].func() < 0) { + HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, + "unable to initialize %s interface", initializer[i].descr) + } + } + + /* clang-format on */ /* Sanity check default VOL connector */ HDassert(H5VL_def_conn_s.connector_id == (-1)); @@ -206,32 +217,6 @@ done: } /* end H5VL_init_phase2() */ /*------------------------------------------------------------------------- - * Function: H5VL__init_package - * - * Purpose: Initialize interface-specific information - * - * Return: Success: Non-negative - * - * Failure: Negative - * - *------------------------------------------------------------------------- - */ -herr_t -H5VL__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Initialize the ID group for the VL IDs */ - if (H5I_register_type(H5I_VOL_CLS) < 0) - HGOTO_ERROR(H5E_VOL, H5E_CANTINIT, FAIL, "unable to initialize H5VL interface") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5VL__init_package() */ - -/*------------------------------------------------------------------------- * Function: H5VL_term_package * * Purpose: Terminate various H5VL objects @@ -249,37 +234,31 @@ H5VL_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { - if (H5VL_def_conn_s.connector_id > 0) { - /* Release the default VOL connector */ - (void)H5VL_conn_free(&H5VL_def_conn_s); - H5VL_def_conn_s.connector_id = -1; - H5VL_def_conn_s.connector_info = NULL; + if (H5VL_def_conn_s.connector_id > 0) { + /* Release the default VOL connector */ + (void)H5VL_conn_free(&H5VL_def_conn_s); + H5VL_def_conn_s.connector_id = -1; + H5VL_def_conn_s.connector_info = NULL; + n++; + } /* end if */ + else { + if (H5I_nmembers(H5I_VOL) > 0) { + /* Unregister all VOL connectors */ + (void)H5I_clear_type(H5I_VOL, TRUE, FALSE); n++; } /* end if */ else { - if (H5I_nmembers(H5I_VOL) > 0) { - /* Unregister all VOL connectors */ - (void)H5I_clear_type(H5I_VOL, TRUE, FALSE); + if (H5VL__num_opt_operation() > 0) { + /* Unregister all dynamically registered optional operations */ + (void)H5VL__term_opt_operation(); n++; } /* end if */ else { - if (H5VL__num_opt_operation() > 0) { - /* Unregister all dynamically registered optional operations */ - (void)H5VL__term_opt_operation(); - n++; - } /* end if */ - else { - /* Destroy the VOL connector ID group */ - n += (H5I_dec_type_ref(H5I_VOL) > 0); - - /* Mark interface as closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end else */ - } /* end else */ - } /* end if */ + /* Destroy the VOL connector ID group */ + n += (H5I_dec_type_ref(H5I_VOL) > 0); + } /* end else */ + } /* end else */ + } /* end else */ FUNC_LEAVE_NOAPI(n) } /* end H5VL_term_package() */ @@ -341,15 +320,15 @@ H5VL__get_connector_cb(void *obj, hid_t id, void *_op_data) FUNC_ENTER_STATIC_NOERR - if (H5VL_GET_CONNECTOR_BY_NAME == op_data->kind) { - if (0 == HDstrcmp(cls->name, op_data->u.name)) { + if (H5VL_GET_CONNECTOR_BY_NAME == op_data->key.kind) { + if (0 == HDstrcmp(cls->name, op_data->key.u.name)) { op_data->found_id = id; ret_value = H5_ITER_STOP; } /* end if */ } /* end if */ else { - HDassert(H5VL_GET_CONNECTOR_BY_VALUE == op_data->kind); - if (cls->value == op_data->u.value) { + HDassert(H5VL_GET_CONNECTOR_BY_VALUE == op_data->key.kind); + if (cls->value == op_data->key.u.value) { op_data->found_id = id; ret_value = H5_ITER_STOP; } /* end if */ @@ -398,7 +377,7 @@ H5VL__set_def_conn(void) } /* end if */ /* Check for environment variable set */ - env_var = HDgetenv("HDF5_VOL_CONNECTOR"); + env_var = HDgetenv(HDF5_VOL_CONNECTOR); /* Only parse the string if it's set */ if (env_var && *env_var) { @@ -976,7 +955,7 @@ H5VL_conn_inc_rc(H5VL_t *connector) { int64_t ret_value = -1; - FUNC_ENTER_NOAPI(-1) + FUNC_ENTER_NOAPI_NOERR /* Check arguments */ HDassert(connector); @@ -986,7 +965,6 @@ H5VL_conn_inc_rc(H5VL_t *connector) ret_value = connector->nrefs; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_conn_inc_rc() */ @@ -1299,9 +1277,9 @@ H5VL__register_connector_by_class(const H5VL_class_t *cls, hbool_t app_ref, hid_ "callback is provided") /* Set up op data for iteration */ - op_data.kind = H5VL_GET_CONNECTOR_BY_NAME; - op_data.u.name = cls->name; - op_data.found_id = H5I_INVALID_HID; + op_data.key.kind = H5VL_GET_CONNECTOR_BY_NAME; + op_data.key.u.name = cls->name; + op_data.found_id = H5I_INVALID_HID; /* Check if connector is already registered */ if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, TRUE) < 0) @@ -1350,15 +1328,15 @@ H5VL__register_connector_by_name(const char *name, hbool_t app_ref, hid_t vipl_i FUNC_ENTER_PACKAGE /* Set up op data for iteration */ - op_data.kind = H5VL_GET_CONNECTOR_BY_NAME; - op_data.u.name = name; - op_data.found_id = H5I_INVALID_HID; + op_data.key.kind = H5VL_GET_CONNECTOR_BY_NAME; + op_data.key.u.name = name; + op_data.found_id = H5I_INVALID_HID; /* Check if connector is already registered */ if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, app_ref) < 0) HGOTO_ERROR(H5E_VOL, H5E_BADITER, H5I_INVALID_HID, "can't iterate over VOL ids") - /* If connector alread registered, increment ref count on ID and return ID */ + /* If connector already registered, increment ref count on ID and return ID */ if (op_data.found_id != H5I_INVALID_HID) { if (H5I_inc_ref(op_data.found_id, app_ref) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTINC, H5I_INVALID_HID, @@ -1410,15 +1388,15 @@ H5VL__register_connector_by_value(H5VL_class_value_t value, hbool_t app_ref, hid FUNC_ENTER_PACKAGE /* Set up op data for iteration */ - op_data.kind = H5VL_GET_CONNECTOR_BY_VALUE; - op_data.u.value = value; - op_data.found_id = H5I_INVALID_HID; + op_data.key.kind = H5VL_GET_CONNECTOR_BY_VALUE; + op_data.key.u.value = value; + op_data.found_id = H5I_INVALID_HID; /* Check if connector is already registered */ - if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, TRUE) < 0) + if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, app_ref) < 0) HGOTO_ERROR(H5E_VOL, H5E_BADITER, H5I_INVALID_HID, "can't iterate over VOL ids") - /* If connector alread registered, increment ref count on ID and return ID */ + /* If connector already registered, increment ref count on ID and return ID */ if (op_data.found_id != H5I_INVALID_HID) { if (H5I_inc_ref(op_data.found_id, app_ref) < 0) HGOTO_ERROR(H5E_VOL, H5E_CANTINC, H5I_INVALID_HID, @@ -1449,8 +1427,9 @@ done: * * Purpose: Checks if a connector with a particular name is registered. * - * Return: Success: 0 - * Failure: -1 + * Return: >0 if a VOL connector with that name has been registered + * 0 if a VOL connector with that name has NOT been registered + * <0 on errors * * Programmer: Dana Robinson * June 17, 2017 @@ -1466,9 +1445,9 @@ H5VL__is_connector_registered_by_name(const char *name) FUNC_ENTER_PACKAGE /* Set up op data for iteration */ - op_data.kind = H5VL_GET_CONNECTOR_BY_NAME; - op_data.u.name = name; - op_data.found_id = H5I_INVALID_HID; + op_data.key.kind = H5VL_GET_CONNECTOR_BY_NAME; + op_data.key.u.name = name; + op_data.found_id = H5I_INVALID_HID; /* Find connector with name */ if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, TRUE) < 0) @@ -1488,8 +1467,9 @@ done: * Purpose: Checks if a connector with a particular value (ID) is * registered. * - * Return: Success: 0 - * Failure: -1 + * Return: >0 if a VOL connector with that value has been registered + * 0 if a VOL connector with that value hasn't been registered + * <0 on errors * *------------------------------------------------------------------------- */ @@ -1502,9 +1482,9 @@ H5VL__is_connector_registered_by_value(H5VL_class_value_t value) FUNC_ENTER_PACKAGE /* Set up op data for iteration */ - op_data.kind = H5VL_GET_CONNECTOR_BY_VALUE; - op_data.u.value = value; - op_data.found_id = H5I_INVALID_HID; + op_data.key.kind = H5VL_GET_CONNECTOR_BY_VALUE; + op_data.key.u.value = value; + op_data.found_id = H5I_INVALID_HID; /* Find connector with value */ if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, TRUE) < 0) @@ -1635,9 +1615,9 @@ H5VL__peek_connector_id_by_name(const char *name) FUNC_ENTER_PACKAGE /* Set up op data for iteration */ - op_data.kind = H5VL_GET_CONNECTOR_BY_NAME; - op_data.u.name = name; - op_data.found_id = H5I_INVALID_HID; + op_data.key.kind = H5VL_GET_CONNECTOR_BY_NAME; + op_data.key.u.name = name; + op_data.found_id = H5I_INVALID_HID; /* Find connector with name */ if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, TRUE) < 0) @@ -1671,9 +1651,9 @@ H5VL__peek_connector_id_by_value(H5VL_class_value_t value) FUNC_ENTER_PACKAGE /* Set up op data for iteration */ - op_data.kind = H5VL_GET_CONNECTOR_BY_VALUE; - op_data.u.value = value; - op_data.found_id = H5I_INVALID_HID; + op_data.key.kind = H5VL_GET_CONNECTOR_BY_VALUE; + op_data.key.u.value = value; + op_data.found_id = H5I_INVALID_HID; /* Find connector with value */ if (H5I_iterate(H5I_VOL, H5VL__get_connector_cb, &op_data, TRUE) < 0) @@ -1757,7 +1737,7 @@ H5VL__get_connector_name(hid_t id, char *name /*out*/, size_t size) len = HDstrlen(cls->name); if (name) { - HDstrncpy(name, cls->name, MIN(len + 1, size)); + HDstrncpy(name, cls->name, size); if (len >= size) name[size - 1] = '\0'; } /* end if */ @@ -2006,7 +1986,7 @@ H5VL_cmp_connector_cls(int *cmp_value, const H5VL_class_t *cls1, const H5VL_clas { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity checks */ HDassert(cls1); @@ -2534,7 +2514,7 @@ H5VL_check_plugin_load(const H5VL_class_t *cls, const H5PL_key_t *key, hbool_t * { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity checks */ HDassert(cls); @@ -2560,7 +2540,6 @@ H5VL_check_plugin_load(const H5VL_class_t *cls, const H5PL_key_t *key, hbool_t * if (*success && cls->version != H5VL_VERSION) *success = FALSE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5VL_check_plugin_load() */ diff --git a/src/H5VLnative.c b/src/H5VLnative.c index 18b5b9c..2a86c26 100644 --- a/src/H5VLnative.c +++ b/src/H5VLnative.c @@ -224,7 +224,7 @@ H5VL__native_term(void) * Purpose: Query the connector class. * * Note: This routine is in this file so that it can return the address - * of the staticly declared class struct. + * of the statically declared class struct. * * Returns: SUCCEED (Can't fail) * @@ -251,7 +251,7 @@ H5VL__native_introspect_get_conn_cls(void H5_ATTR_UNUSED *obj, H5VL_get_conn_lvl * Purpose: Query the capability flags for this connector. * * Note: This routine is in this file so that it can return the field - * from the staticly declared class struct. + * from the statically declared class struct. * * Returns: SUCCEED (Can't fail) * @@ -324,7 +324,7 @@ done: herr_t H5VL__native_get_file_addr_len(void *obj, H5I_type_t obj_type, size_t *addr_len) { - H5F_t *file = NULL; /* File stuct pointer */ + H5F_t *file = NULL; /* File struct pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) diff --git a/src/H5VLnative_introspect.c b/src/H5VLnative_introspect.c index fa11bea..0cc6ee4 100644 --- a/src/H5VLnative_introspect.c +++ b/src/H5VLnative_introspect.c @@ -53,7 +53,7 @@ /*******************/ /* Note: H5VL__native_introspect_get_conn_cls and H5VL__native_introspect_get_cap_flags - * are in src/H5VLnative.c so that they can work with the staticly declared + * are in src/H5VLnative.c so that they can work with the statically declared * class struct. */ diff --git a/src/H5VLnative_token.c b/src/H5VLnative_token.c index bed0164..b5bd7b8 100644 --- a/src/H5VLnative_token.c +++ b/src/H5VLnative_token.c @@ -112,7 +112,7 @@ H5VL__native_token_to_str(void *obj, H5I_type_t obj_type, const H5O_token_t *tok if (NULL == (*token_str = H5MM_malloc(addr_ndigits + 1))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer for token string") - HDsnprintf(*token_str, addr_ndigits + 1, H5_PRINTF_HADDR_FMT, addr); + HDsnprintf(*token_str, addr_ndigits + 1, "%" PRIuHADDR, addr); done: FUNC_LEAVE_NOAPI(ret_value) @@ -139,7 +139,7 @@ H5VL__native_str_to_token(void *obj, H5I_type_t obj_type, const char *token_str, /* Check parameters */ HDassert(token_str); - HDsscanf(token_str, H5_PRINTF_HADDR_FMT, &addr); + HDsscanf(token_str, "%" PRIuHADDR, &addr); if (H5VL_native_addr_to_token(obj, obj_type, addr, token) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "can't convert address to object token") diff --git a/src/H5VLpassthru.c b/src/H5VLpassthru.c index 681531e..6eda875 100644 --- a/src/H5VLpassthru.c +++ b/src/H5VLpassthru.c @@ -45,7 +45,7 @@ /* Macros */ /**********/ -/* Whether to display log messge when callback is invoked */ +/* Whether to display log message when callback is invoked */ /* (Uncomment to enable) */ /* #define ENABLE_PASSTHRU_LOGGING */ @@ -641,16 +641,13 @@ H5VL_pass_through_info_to_str(const void *_info, char **str) under_vol_str_len = strlen(under_vol_string); /* Allocate space for our info */ - *str = (char *)H5allocate_memory(32 + under_vol_str_len, (hbool_t)0); + size_t strSize = 32 + under_vol_str_len; + *str = (char *)H5allocate_memory(strSize, (hbool_t)0); assert(*str); - /* Encode our info - * Normally we'd use snprintf() here for a little extra safety, but that - * call had problems on Windows until recently. So, to be as platform-independent - * as we can, we're using sprintf() instead. - */ - sprintf(*str, "under_vol=%u;under_info={%s}", (unsigned)under_value, - (under_vol_string ? under_vol_string : "")); + /* Encode our info */ + snprintf(*str, strSize, "under_vol=%u;under_info={%s}", (unsigned)under_value, + (under_vol_string ? under_vol_string : "")); return 0; } /* end H5VL_pass_through_info_to_str() */ diff --git a/src/H5VM.c b/src/H5VM.c index 4d65d75..209ac25 100644 --- a/src/H5VM.c +++ b/src/H5VM.c @@ -1244,7 +1244,7 @@ H5VM_chunk_index_scaled(unsigned ndims, const hsize_t *coord, const uint32_t *ch * Function: H5VM_opvv * * Purpose: Perform an operation on a source & destination sequences - * of offset/length pairs. Each set of sequnces has an array + * of offset/length pairs. Each set of sequences has an array * of lengths, an array of offsets, the maximum number of * sequences and the current sequence to start at in the sequence. * diff --git a/src/H5WB.c b/src/H5WB.c index df7627f..84592bd 100644 --- a/src/H5WB.c +++ b/src/H5WB.c @@ -18,7 +18,7 @@ * Quincey Koziol * * Purpose: Implements the "wrapped buffer" code for wrapping - * an existing [staticly sized] buffer, in order to + * an existing [statically sized] buffer, in order to * avoid lots of memory allocation calls. * *------------------------------------------------------------------------- diff --git a/src/H5Z.c b/src/H5Z.c index ea92a01..763eac2 100644 --- a/src/H5Z.c +++ b/src/H5Z.c @@ -56,9 +56,6 @@ typedef enum { H5Z_PRELUDE_SET_LOCAL /* Call "set local" callback */ } H5Z_prelude_type_t; -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /* Local variables */ static size_t H5Z_table_alloc_g = 0; static size_t H5Z_table_used_g = 0; @@ -74,19 +71,23 @@ static int H5Z__check_unregister_group_cb(void *obj_ptr, hid_t obj_id, void *key static int H5Z__flush_file_cb(void *obj_ptr, hid_t obj_id, void *key); /*------------------------------------------------------------------------- - * Function: H5Z__init_package + * Function: H5Z_init * - * Purpose: Initializes the data filter layer. + * Purpose: Initialize the interface from some other layer. * - * Return: Non-negative on success/Negative on failure + * Return: Success: non-negative + * Failure: negative *------------------------------------------------------------------------- */ herr_t -H5Z__init_package(void) +H5Z_init(void) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_NOAPI(FAIL) + + if (H5_TERM_GLOBAL) + HGOTO_DONE(SUCCEED) /* Internal filters */ if (H5Z_register(H5Z_SHUFFLE) < 0) @@ -111,7 +112,7 @@ H5Z__init_package(void) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5Z__init_package() */ +} /*------------------------------------------------------------------------- * Function: H5Z_term_package @@ -128,76 +129,70 @@ H5Z_term_package(void) FUNC_ENTER_NOAPI_NOINIT_NOERR - if (H5_PKG_INIT_VAR) { #ifdef H5Z_DEBUG - char comment[16], bandwidth[32]; - int dir, nprint = 0; - size_t i; - - if (H5DEBUG(Z)) { - for (i = 0; i < H5Z_table_used_g; i++) { - for (dir = 0; dir < 2; dir++) { - struct { - char *user; - char *system; - char *elapsed; - } timestrs = {H5_timer_get_time_string(H5Z_stat_table_g[i].stats[dir].times.user), - H5_timer_get_time_string(H5Z_stat_table_g[i].stats[dir].times.system), - H5_timer_get_time_string(H5Z_stat_table_g[i].stats[dir].times.elapsed)}; - if (0 == H5Z_stat_table_g[i].stats[dir].total) - goto next; - - if (0 == nprint++) { - /* Print column headers */ - HDfprintf(H5DEBUG(Z), "H5Z: filter statistics " - "accumulated over life of library:\n"); - HDfprintf(H5DEBUG(Z), " %-16s %10s %10s %8s %8s %8s %10s\n", "Filter", "Total", - "Errors", "User", "System", "Elapsed", "Bandwidth"); - HDfprintf(H5DEBUG(Z), " %-16s %10s %10s %8s %8s %8s %10s\n", "------", "-----", - "------", "----", "------", "-------", "---------"); - } /* end if */ + char comment[16], bandwidth[32]; + int dir, nprint = 0; + size_t i; - /* Truncate the comment to fit in the field */ - HDstrncpy(comment, H5Z_table_g[i].name, sizeof comment); - comment[sizeof(comment) - 1] = '\0'; - - /* - * Format bandwidth to have four significant digits and - * units of `B/s', `kB/s', `MB/s', `GB/s', or `TB/s' or - * the word `Inf' if the elapsed time is zero. - */ - H5_bandwidth(bandwidth, (double)(H5Z_stat_table_g[i].stats[dir].total), - H5Z_stat_table_g[i].stats[dir].times.elapsed); - - /* Print the statistics */ - HDfprintf(H5DEBUG(Z), " %s%-15s %10" PRIdHSIZE " %10" PRIdHSIZE " %8s %8s %8s %10s\n", - (dir ? "<" : ">"), comment, H5Z_stat_table_g[i].stats[dir].total, - H5Z_stat_table_g[i].stats[dir].errors, timestrs.user, timestrs.system, - timestrs.elapsed, bandwidth); + if (H5DEBUG(Z)) { + for (i = 0; i < H5Z_table_used_g; i++) { + for (dir = 0; dir < 2; dir++) { + struct { + char *user; + char *system; + char *elapsed; + } timestrs = {H5_timer_get_time_string(H5Z_stat_table_g[i].stats[dir].times.user), + H5_timer_get_time_string(H5Z_stat_table_g[i].stats[dir].times.system), + H5_timer_get_time_string(H5Z_stat_table_g[i].stats[dir].times.elapsed)}; + if (0 == H5Z_stat_table_g[i].stats[dir].total) + goto next; + + if (0 == nprint++) { + /* Print column headers */ + HDfprintf(H5DEBUG(Z), "H5Z: filter statistics " + "accumulated over life of library:\n"); + HDfprintf(H5DEBUG(Z), " %-16s %10s %10s %8s %8s %8s %10s\n", "Filter", "Total", + "Errors", "User", "System", "Elapsed", "Bandwidth"); + HDfprintf(H5DEBUG(Z), " %-16s %10s %10s %8s %8s %8s %10s\n", "------", "-----", + "------", "----", "------", "-------", "---------"); + } /* end if */ + + /* Truncate the comment to fit in the field */ + HDstrncpy(comment, H5Z_table_g[i].name, sizeof comment); + comment[sizeof(comment) - 1] = '\0'; + + /* + * Format bandwidth to have four significant digits and + * units of `B/s', `kB/s', `MB/s', `GB/s', or `TB/s' or + * the word `Inf' if the elapsed time is zero. + */ + H5_bandwidth(bandwidth, (double)(H5Z_stat_table_g[i].stats[dir].total), + H5Z_stat_table_g[i].stats[dir].times.elapsed); + + /* Print the statistics */ + HDfprintf(H5DEBUG(Z), " %s%-15s %10" PRIdHSIZE " %10" PRIdHSIZE " %8s %8s %8s %10s\n", + (dir ? "<" : ">"), comment, H5Z_stat_table_g[i].stats[dir].total, + H5Z_stat_table_g[i].stats[dir].errors, timestrs.user, timestrs.system, + timestrs.elapsed, bandwidth); next: - HDfree(timestrs.user); - HDfree(timestrs.system); - HDfree(timestrs.elapsed); - } /* end for */ - } /* end for */ - } /* end if */ -#endif /* H5Z_DEBUG */ - - /* Free the table of filters */ - if (H5Z_table_g) { - H5Z_table_g = (H5Z_class2_t *)H5MM_xfree(H5Z_table_g); + HDfree(timestrs.user); + HDfree(timestrs.system); + HDfree(timestrs.elapsed); + } /* end for */ + } /* end for */ + } /* end if */ +#endif /* H5Z_DEBUG */ + + /* Free the table of filters */ + if (H5Z_table_g) { + H5Z_table_g = (H5Z_class2_t *)H5MM_xfree(H5Z_table_g); #ifdef H5Z_DEBUG - H5Z_stat_table_g = (H5Z_stats_t *)H5MM_xfree(H5Z_stat_table_g); + H5Z_stat_table_g = (H5Z_stats_t *)H5MM_xfree(H5Z_stat_table_g); #endif /* H5Z_DEBUG */ - H5Z_table_used_g = H5Z_table_alloc_g = 0; + H5Z_table_used_g = H5Z_table_alloc_g = 0; - n++; - } /* end if */ - - /* Mark interface as closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; + n++; } /* end if */ FUNC_LEAVE_NOAPI(n) @@ -599,14 +594,9 @@ H5Z__flush_file_cb(void *obj_ptr, hid_t H5_ATTR_UNUSED obj_id, void H5_ATTR_PARA /* Do a global flush if the file is opened for write */ if (H5F_ACC_RDWR & H5F_INTENT(f)) { -/* When parallel HDF5 is defined, check for collective metadata reads on this - * file and set the flag for metadata I/O in the API context. -QAK, 2018/02/14 - */ #ifdef H5_HAVE_PARALLEL /* Check if MPIO driver is used */ if (H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { - H5P_coll_md_read_flag_t coll_md_read; /* Do all metadata reads collectively */ - /* Sanity check for collectively calling H5Zunregister, if requested */ /* (Sanity check assumes that a barrier on one file's comm * is sufficient (i.e. that there aren't different comms for @@ -626,13 +616,8 @@ H5Z__flush_file_cb(void *obj_ptr, hid_t H5_ATTR_UNUSED obj_id, void H5_ATTR_PARA /* Set the "sanity checked" flag */ object->sanity_checked = TRUE; } /* end if */ - - /* Check whether to use the collective metadata read DXPL */ - coll_md_read = H5F_COLL_MD_READ(f); - if (H5P_USER_TRUE == coll_md_read) - H5CX_set_coll_metadata_read(TRUE); - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ /* Call the flush routine for mounted file hierarchies */ if (H5F_flush_mounts((H5F_t *)obj_ptr) < 0) @@ -1506,7 +1491,7 @@ done: /*------------------------------------------------------------------------- * Function: H5Z_filter_in_pline * - * Purpose: Check wheter a filter is in the filter pipeline using the + * Purpose: Check whether a filter is in the filter pipeline using the * filter ID. This function is very similar to H5Z_filter_info * * Return: TRUE - found filter @@ -1520,7 +1505,7 @@ H5Z_filter_in_pline(const H5O_pline_t *pline, H5Z_filter_t filter) size_t idx; /* Index of filter in pipeline */ htri_t ret_value = TRUE; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR HDassert(pline); HDassert(filter >= 0 && filter <= H5Z_FILTER_MAX); @@ -1534,7 +1519,6 @@ H5Z_filter_in_pline(const H5O_pline_t *pline, H5Z_filter_t filter) if (idx >= pline->nused) ret_value = FALSE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5Z_filter_in_pline() */ @@ -1554,7 +1538,7 @@ H5Z_all_filters_avail(const H5O_pline_t *pline) size_t i, j; /* Local index variable */ htri_t ret_value = TRUE; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Check args */ HDassert(pline); diff --git a/src/H5Znbit.c b/src/H5Znbit.c index b696085..905d417 100644 --- a/src/H5Znbit.c +++ b/src/H5Znbit.c @@ -975,7 +975,7 @@ H5Z__filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], s /* input; decompress */ if (flags & H5Z_FLAG_REVERSE) { - size_out = d_nelmts * cd_values[4]; /* cd_values[4] stores datatype size */ + size_out = d_nelmts * (size_t)cd_values[4]; /* cd_values[4] stores datatype size */ /* allocate memory space for decompressed buffer */ if (NULL == (outbuf = (unsigned char *)H5MM_malloc(size_out))) @@ -1015,7 +1015,7 @@ done: * assume one byte has 8 bit * assume padding bit is 0 * assume size of unsigned char is one byte - * assume one data item of certain datatype is stored continously in bytes + * assume one data item of certain datatype is stored continuously in bytes * atomic datatype is treated on byte basis */ @@ -1170,7 +1170,8 @@ H5Z__nbit_decompress_one_array(unsigned char *data, size_t data_offset, unsigned n = total_size / p.size; for (i = 0; i < n; i++) - H5Z__nbit_decompress_one_atomic(data, data_offset + i * p.size, buffer, j, buf_len, &p); + H5Z__nbit_decompress_one_atomic(data, data_offset + i * (size_t)p.size, buffer, j, buf_len, + &p); break; case H5Z_NBIT_ARRAY: @@ -1178,8 +1179,8 @@ H5Z__nbit_decompress_one_array(unsigned char *data, size_t data_offset, unsigned n = total_size / base_size; /* number of base_type elements inside the array datatype */ begin_index = *parms_index; for (i = 0; i < n; i++) { - if (H5Z__nbit_decompress_one_array(data, data_offset + i * base_size, buffer, j, buf_len, - parms, parms_index) < 0) + if (H5Z__nbit_decompress_one_array(data, data_offset + i * (size_t)base_size, buffer, j, + buf_len, parms, parms_index) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress array") *parms_index = begin_index; } @@ -1190,8 +1191,8 @@ H5Z__nbit_decompress_one_array(unsigned char *data, size_t data_offset, unsigned n = total_size / base_size; /* number of base_type elements inside the array datatype */ begin_index = *parms_index; for (i = 0; i < n; i++) { - if (H5Z__nbit_decompress_one_compound(data, data_offset + i * base_size, buffer, j, buf_len, - parms, parms_index) < 0) + if (H5Z__nbit_decompress_one_compound(data, data_offset + i * (size_t)base_size, buffer, j, + buf_len, parms, parms_index) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress compound") *parms_index = begin_index; } @@ -1291,7 +1292,7 @@ H5Z__nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buff FUNC_ENTER_STATIC /* may not have to initialize to zeros */ - HDmemset(data, 0, d_nelmts * parms[4]); + HDmemset(data, 0, d_nelmts * (size_t)parms[4]); /* initialization before the loop */ j = 0; @@ -1309,7 +1310,7 @@ H5Z__nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buff HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "invalid datatype precision/offset") for (i = 0; i < d_nelmts; i++) - H5Z__nbit_decompress_one_atomic(data, i * p.size, buffer, &j, &buf_len, &p); + H5Z__nbit_decompress_one_atomic(data, i * (size_t)p.size, buffer, &j, &buf_len, &p); break; case H5Z_NBIT_ARRAY: @@ -1468,7 +1469,7 @@ H5Z__nbit_compress_one_array(unsigned char *data, size_t data_offset, unsigned c p.offset = parms[(*parms_index)++]; n = total_size / p.size; for (i = 0; i < n; i++) - H5Z__nbit_compress_one_atomic(data, data_offset + i * p.size, buffer, j, buf_len, &p); + H5Z__nbit_compress_one_atomic(data, data_offset + i * (size_t)p.size, buffer, j, buf_len, &p); break; case H5Z_NBIT_ARRAY: @@ -1476,8 +1477,8 @@ H5Z__nbit_compress_one_array(unsigned char *data, size_t data_offset, unsigned c n = total_size / base_size; /* number of base_type elements inside the array datatype */ begin_index = *parms_index; for (i = 0; i < n; i++) { - H5Z__nbit_compress_one_array(data, data_offset + i * base_size, buffer, j, buf_len, parms, - parms_index); + H5Z__nbit_compress_one_array(data, data_offset + i * (size_t)base_size, buffer, j, buf_len, + parms, parms_index); *parms_index = begin_index; } break; @@ -1487,8 +1488,8 @@ H5Z__nbit_compress_one_array(unsigned char *data, size_t data_offset, unsigned c n = total_size / base_size; /* number of base_type elements inside the array datatype */ begin_index = *parms_index; for (i = 0; i < n; i++) { - H5Z__nbit_compress_one_compound(data, data_offset + i * base_size, buffer, j, buf_len, parms, - parms_index); + H5Z__nbit_compress_one_compound(data, data_offset + i * (size_t)base_size, buffer, j, buf_len, + parms, parms_index); *parms_index = begin_index; } break; @@ -1574,7 +1575,7 @@ H5Z__nbit_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer p.offset = parms[7]; for (i = 0; i < d_nelmts; i++) - H5Z__nbit_compress_one_atomic(data, i * p.size, buffer, &new_size, &buf_len, &p); + H5Z__nbit_compress_one_atomic(data, i * (size_t)p.size, buffer, &new_size, &buf_len, &p); break; case H5Z_NBIT_ARRAY: diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index 48a29cd..46c1a10 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -659,7 +659,7 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ buf[i] = (type)(buf[i] + (type)(minval)); \ } while (0) -/* Retrive minimum value of floating-point type */ +/* Retrieve minimum value of floating-point type */ #define H5Z_scaleoffset_get_min(type, minval, min) \ { \ if (sizeof(type) <= sizeof(long long)) \ @@ -1205,7 +1205,7 @@ H5Z__filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_valu /* prepare parameters to pass to compress/decompress functions */ p.size = cd_values[H5Z_SCALEOFFSET_PARM_SIZE]; - p.mem_order = H5T_native_order_g; + p.mem_order = (unsigned)H5T_native_order_g; /* input; decompress */ if (flags & H5Z_FLAG_REVERSE) { @@ -1240,7 +1240,7 @@ H5Z__filter_scaleoffset(unsigned flags, size_t cd_nelmts, const unsigned cd_valu p.minbits = minbits; /* calculate size of output buffer after decompression */ - size_out = d_nelmts * p.size; + size_out = d_nelmts * (size_t)p.size; /* allocate memory space for decompressed buffer */ if (NULL == (outbuf = (unsigned char *)H5MM_malloc(size_out))) @@ -1388,7 +1388,7 @@ done: * assume one byte has 8 bit * assume padding bit is 0 * assume size of unsigned char is one byte - * assume one data item of certain datatype is stored continously in bytes + * assume one data item of certain datatype is stored continuously in bytes * atomic datatype is treated on byte basis */ @@ -1403,7 +1403,7 @@ H5Z__scaleoffset_convert(void *buf, unsigned d_nelmts, unsigned dtype_size) unsigned char *buffer, temp; buffer = (unsigned char *)buf; - for (i = 0; i < d_nelmts * dtype_size; i += dtype_size) + for (i = 0; i < d_nelmts * (size_t)dtype_size; i += dtype_size) for (j = 0; j < dtype_size / 2; j++) { /* swap pair of bytes */ temp = buffer[i + j]; @@ -1681,7 +1681,7 @@ H5Z__scaleoffset_decompress(unsigned char *data, unsigned d_nelmts, unsigned cha unsigned buf_len; /* must initialize to zeros */ - for (i = 0; i < d_nelmts * p.size; i++) + for (i = 0; i < d_nelmts * (size_t)p.size; i++) data[i] = 0; /* initialization before the loop */ diff --git a/src/H5Ztrans.c b/src/H5Ztrans.c index ea532cf..092b289 100644 --- a/src/H5Ztrans.c +++ b/src/H5Ztrans.c @@ -1352,7 +1352,7 @@ H5Z__op_is_numbs2(H5Z_node *_tree) * Function: H5Z__xform_reduce_tree * * Purpose: Simplifies parse tree passed in by performing any obvious - * and trivial arithemtic calculations. + * and trivial arithmetic calculations. * * Return: None. * @@ -1692,7 +1692,7 @@ H5Z_xform_noop(const H5Z_data_xform_t *data_xform_prop) if (data_xform_prop) { ret_value = FALSE; - /* Check for trivial data tranformation: expression = "x" */ + /* Check for trivial data transformation: expression = "x" */ if ((HDstrlen(data_xform_prop->xform_exp) == 1) && data_xform_prop->dat_val_pointers && (data_xform_prop->dat_val_pointers->num_ptrs == 1)) { ret_value = TRUE; @@ -1723,7 +1723,7 @@ H5Z_xform_extract_xform_str(const H5Z_data_xform_t *data_xform_prop) /* There should be no way that this can be NULL since the function * that calls this one checks to make sure it isn't before - * pasing them */ + * passing them */ HDassert(data_xform_prop); FUNC_LEAVE_NOAPI(data_xform_prop->xform_exp) diff --git a/src/H5detect.c b/src/H5detect.c index 022cb55..e72d7a0 100644 --- a/src/H5detect.c +++ b/src/H5detect.c @@ -772,7 +772,7 @@ done:\n\ fprintf(rawoutstream, "/* sigprocmask() support: no */\n"); #endif - /* Print the statics of signal handlers called for debugging */ + /* Print the statistics of signal handlers called for debugging */ fprintf(rawoutstream, "\n" "/******************************/\n" "/* signal handlers statistics */\n" @@ -1475,7 +1475,7 @@ verify_signal_handlers(int signum, void (*handler)(int)) } else { if (val == signum) { - /* return from signum handler. Record a sucess. */ + /* return from signum handler. Record a success. */ nsuccesses++; } else { diff --git a/src/H5mpi.c b/src/H5mpi.c index 4a8aa44..15fb785 100644 --- a/src/H5mpi.c +++ b/src/H5mpi.c @@ -38,7 +38,7 @@ static hsize_t bigio_count_g = H5_MAX_MPI_COUNT; /*------------------------------------------------------------------------- * Function: H5_mpi_set_bigio_count * - * Purpose: Allow us to programatically change the switch point + * Purpose: Allow us to programmatically change the switch point * when we utilize derived datatypes. This is of * particular interest for allowing nightly testing * @@ -549,4 +549,237 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5_mpio_create_large_type() */ +/*------------------------------------------------------------------------- + * Function: H5_mpio_gatherv_alloc + * + * Purpose: A wrapper around MPI_(All)gatherv that performs allocation + * of the receive buffer on the caller's behalf. This + * routine's parameters are as follows: + * + * `send_buf` - The buffer that data will be sent from for + * the calling MPI rank. Analogous to + * MPI_(All)gatherv's `sendbuf` parameter. + * + * `send_count` - The number of `send_type` elements in the + * send buffer. Analogous to MPI_(All)gatherv's + * `sendcount` parameter. + * + * `send_type` - The MPI Datatype of the elements in the send + * buffer. Analogous to MPI_(All)gatherv's + * `sendtype` parameter. + * + * `recv_counts` - An array containing the number of elements + * to be received from each MPI rank. + * Analogous to MPI_(All)gatherv's `recvcount` + * parameter. + * + * `displacements` - An array containing the displacements + * in the receive buffer where data from + * each MPI rank should be placed. Analogous + * to MPI_(All)gatherv's `displs` parameter. + * + * `recv_type` - The MPI Datatype of the elements in the + * receive buffer. Analogous to + * MPI_(All)gatherv's `recvtype` parameter. + * + * `allgather` - Specifies whether the gather operation to be + * performed should be MPI_Allgatherv (TRUE) or + * MPI_Gatherv (FALSE). + * + * `root` - For MPI_Gatherv operations, specifies the rank + * that will receive the data sent by other ranks. + * Analogous to MPI_Gatherv's `root` parameter. For + * MPI_Allgatherv operations, this parameter is + * ignored. + * + * `comm` - Specifies the MPI Communicator for the operation. + * Analogous to MPI_(All)gatherv's `comm` parameter. + * + * `mpi_rank` - Specifies the calling rank's rank value, as + * obtained by calling MPI_Comm_rank on the + * MPI Communicator `comm`. + * + * `mpi_size` - Specifies the MPI Communicator size, as + * obtained by calling MPI_Comm_size on the + * MPI Communicator `comm`. + * + * `out_buf` - Resulting buffer that is allocated and + * returned to the caller after data has been + * gathered into it. Returned only to the rank + * specified by `root` for MPI_Gatherv + * operations, or to all ranks for + * MPI_Allgatherv operations. + * + * `out_buf_num_entries` - The number of elements in the + * resulting buffer, in terms of + * the MPI Datatype provided for + * `recv_type`. + * + * Notes: This routine is collective across `comm`. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpio_gatherv_alloc(void *send_buf, int send_count, MPI_Datatype send_type, const int recv_counts[], + const int displacements[], MPI_Datatype recv_type, hbool_t allgather, int root, + MPI_Comm comm, int mpi_rank, int mpi_size, void **out_buf, size_t *out_buf_num_entries) +{ + size_t recv_buf_num_entries = 0; + void * recv_buf = NULL; +#if MPI_VERSION >= 3 + MPI_Count type_lb; + MPI_Count type_extent; +#else + MPI_Aint type_lb; + MPI_Aint type_extent; +#endif + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(send_buf || send_count == 0); + if (allgather || (mpi_rank == root)) + HDassert(out_buf && out_buf_num_entries); + + /* Retrieve the extent of the MPI Datatype being used */ +#if MPI_VERSION >= 3 + if (MPI_SUCCESS != (mpi_code = MPI_Type_get_extent_x(recv_type, &type_lb, &type_extent))) +#else + if (MPI_SUCCESS != (mpi_code = MPI_Type_get_extent(recv_type, &type_lb, &type_extent))) +#endif + HMPI_GOTO_ERROR(FAIL, "MPI_Type_get_extent(_x) failed", mpi_code) + + if (type_extent < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "MPI recv_type had a negative extent") + + /* + * Calculate the total size of the buffer being + * returned and allocate it + */ + if (allgather || (mpi_rank == root)) { + size_t i; + size_t buf_size; + + for (i = 0, recv_buf_num_entries = 0; i < (size_t)mpi_size; i++) + recv_buf_num_entries += (size_t)recv_counts[i]; + buf_size = recv_buf_num_entries * (size_t)type_extent; + + /* If our buffer size is 0, there's nothing to do */ + if (buf_size == 0) + HGOTO_DONE(SUCCEED) + + if (NULL == (recv_buf = H5MM_malloc(buf_size))) + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate receive buffer") + } + + /* Perform gather operation */ + if (allgather) { + if (MPI_SUCCESS != (mpi_code = MPI_Allgatherv(send_buf, send_count, send_type, recv_buf, recv_counts, + displacements, recv_type, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code) + } + else { + if (MPI_SUCCESS != (mpi_code = MPI_Gatherv(send_buf, send_count, send_type, recv_buf, recv_counts, + displacements, recv_type, root, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Gatherv failed", mpi_code) + } + + if (allgather || (mpi_rank == root)) { + *out_buf = recv_buf; + *out_buf_num_entries = recv_buf_num_entries; + } + +done: + if (ret_value < 0) { + if (recv_buf) + H5MM_free(recv_buf); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpio_gatherv_alloc() */ + +/*------------------------------------------------------------------------- + * Function: H5_mpio_gatherv_alloc_simple + * + * Purpose: A slightly simplified interface to H5_mpio_gatherv_alloc + * which calculates the receive counts and receive buffer + * displacements for the caller. + * + * Notes: This routine is collective across `comm`. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpio_gatherv_alloc_simple(void *send_buf, int send_count, MPI_Datatype send_type, MPI_Datatype recv_type, + hbool_t allgather, int root, MPI_Comm comm, int mpi_rank, int mpi_size, + void **out_buf, size_t *out_buf_num_entries) +{ + int * recv_counts_disps_array = NULL; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(send_buf || send_count == 0); + if (allgather || (mpi_rank == root)) + HDassert(out_buf && out_buf_num_entries); + + /* + * Allocate array to store the receive counts of each rank, as well as + * the displacements into the final array where each rank will place + * their data. The first half of the array contains the receive counts + * (in rank order), while the latter half contains the displacements + * (also in rank order). + */ + if (allgather || (mpi_rank == root)) { + if (NULL == + (recv_counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*recv_counts_disps_array)))) + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + + /* Collect each rank's send count to interested ranks */ + if (allgather) { + if (MPI_SUCCESS != + (mpi_code = MPI_Allgather(&send_count, 1, MPI_INT, recv_counts_disps_array, 1, MPI_INT, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + } + else { + if (MPI_SUCCESS != + (mpi_code = MPI_Gather(&send_count, 1, MPI_INT, recv_counts_disps_array, 1, MPI_INT, root, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Gather failed", mpi_code) + } + + /* Set the displacements into the receive buffer for the gather operation */ + if (allgather || (mpi_rank == root)) { + size_t i; + int * displacements_ptr; + + displacements_ptr = &recv_counts_disps_array[mpi_size]; + + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + recv_counts_disps_array[i - 1]; + } + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(send_buf, send_count, send_type, recv_counts_disps_array, + &recv_counts_disps_array[mpi_size], recv_type, allgather, root, comm, mpi_rank, + mpi_size, out_buf, out_buf_num_entries) < 0) + HGOTO_ERROR(H5E_LIB, H5E_CANTGATHER, FAIL, "can't gather data") + +done: + if (recv_counts_disps_array) + H5MM_free(recv_counts_disps_array); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpio_gatherv_alloc_simple() */ + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5private.h b/src/H5private.h index 58a4100..da20a1b 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -44,10 +44,12 @@ #include #endif #ifdef H5_HAVE_UNISTD_H +#include +#endif #ifdef H5_HAVE_PWD_H #include #endif -#include +#ifdef H5_HAVE_WAITPID #include #endif @@ -106,8 +108,14 @@ */ #include "H5queue.h" -/* Define the default VFD for this platform. - * Since the removal of the Windows VFD, this is sec2 for all platforms. +/* Define the default VFD for this platform. Since the removal of the + * Windows VFD, this is sec2 for all platforms. + * + * Note well: if you change the default, then be sure to change + * H5_default_vfd_init() to call that default's initializer. Also, + * make sure that the initializer for each *non*-default VFD calls + * H5_init_library(); also, make sure that the initializer for default + * VFD does *not* call H5_init_library(). */ #define H5_DEFAULT_VFD H5FD_SEC2 @@ -171,7 +179,7 @@ */ #define BEGIN_MPE_LOG \ if (H5_MPEinit_g) { \ - sprintf(p_event_start, "start %s", __func__); \ + snprintf(p_event_start, sizeof(p_event_start), "start %s", __func__); \ if (eventa(__func__) == -1 && eventb(__func__) == -1) { \ const char *p_color = "red"; \ eventa(__func__) = MPE_Log_get_event_number(); \ @@ -386,6 +394,25 @@ #define HSSIZET_MAX ((hssize_t)LLONG_MAX) #define HSSIZET_MIN (~(HSSIZET_MAX)) +#ifdef H5_HAVE_PARALLEL + +/* Define a type for safely sending size_t values with MPI */ +#if SIZE_MAX == UCHAR_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_CHAR +#elif SIZE_MAX == USHRT_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_SHORT +#elif SIZE_MAX == UINT_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED +#elif SIZE_MAX == ULONG_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_LONG +#elif SIZE_MAX == ULLONG_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_LONG_LONG +#else +#error "no suitable MPI type for size_t" +#endif + +#endif /* H5_HAVE_PARALLEL */ + /* * Types and max sizes for POSIX I/O. * OS X (Darwin) is odd since the max I/O size does not match the types. @@ -507,6 +534,9 @@ #define H5_GCC_CLANG_DIAG_ON(x) #endif +/* Function pointer typedef for qsort */ +typedef int (*H5_sort_func_cb_t)(const void *, const void *); + /* Typedefs and functions for timing certain parts of the library. */ /* A set of elapsed/user/system times emitted as a time point by the @@ -575,7 +605,7 @@ typedef off_t h5_stat_size_t; #define HDoff_t off_t #endif -#/* Redefine all the POSIX and C functions. We should never see an +/* Redefine all the POSIX and C functions. We should never see an * undecorated POSIX or C function (or any other non-HDF5 function) * in the source. */ @@ -2006,7 +2036,7 @@ extern H5_api_t H5_g; #define H5_API_LOCK #define H5_API_UNLOCK -/* disable cancelability (sequential version) */ +/* disable cancellability (sequential version) */ #define H5_API_UNSET_CANCEL #define H5_API_SET_CANCEL @@ -2036,33 +2066,6 @@ extern hbool_t H5_libterm_g; /* Is the library being shutdown? */ extern hbool_t H5_MPEinit_g; /* Has the MPE Library been initialized? */ #endif -/* Macros for referencing package initialization symbols */ -#define H5_PACKAGE_INIT_VAR(x) H5_GLUE(x, _init_g) -#define H5_PACKAGE_INIT_FUNC(x) H5_GLUE(x, __init_package) - -/* Macros for defining package initialization routines */ -#ifdef H5_MY_PKG -#define H5_PKG_INIT_VAR H5_PACKAGE_INIT_VAR(H5_MY_PKG) -#define H5_PKG_INIT_FUNC H5_PACKAGE_INIT_FUNC(H5_MY_PKG) -#define H5_PACKAGE_YES_INIT(err) \ - /* Initialize this interface or bust */ \ - if (!H5_PKG_INIT_VAR && !H5_TERM_GLOBAL) { \ - H5_PKG_INIT_VAR = TRUE; \ - if (H5_PKG_INIT_FUNC() < 0) { \ - H5_PKG_INIT_VAR = FALSE; \ - HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, err, "interface initialization failed") \ - } \ - } -#define H5_PACKAGE_NO_INIT(err) \ - /* Initialize this interface or bust */ \ - if (!H5_PKG_INIT_VAR && !H5_TERM_GLOBAL) \ - H5_PKG_INIT_VAR = TRUE; -#define H5_PACKAGE_INIT(pkg_init, err) H5_GLUE3(H5_PACKAGE_, pkg_init, _INIT)(err) -#else /* H5_MY_PKG */ -#define H5_PKG_INIT_VAR (TRUE) -#define H5_PACKAGE_INIT(pkg_init, err) -#endif /* H5_MY_PKG */ - /* Forward declaration of H5CXpush() / H5CXpop() */ /* (Including H5CXprivate.h creates bad circular dependencies - QAK, 3/18/2018) */ H5_DLL herr_t H5CX_push(void); @@ -2116,13 +2119,9 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define FUNC_ENTER_API_INIT(err) \ /* Initialize the library */ \ if (!H5_INIT_GLOBAL && !H5_TERM_GLOBAL) { \ - H5_INIT_GLOBAL = TRUE; \ if (H5_init_library() < 0) \ HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, err, "library initialization failed") \ - } /* end if */ \ - \ - /* Initialize the package, if appropriate */ \ - H5_PACKAGE_INIT(H5_MY_PKG_INIT, err) + } #define FUNC_ENTER_API_PUSH(err) \ /* Push the name of this function on the function stack */ \ @@ -2131,6 +2130,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); /* Push the API context */ \ if (H5CX_push() < 0) \ HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, err, "can't set API context") \ + else \ + api_ctx_pushed = TRUE; \ \ BEGIN_MPE_LOG @@ -2170,6 +2171,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define FUNC_ENTER_API(err) \ { \ { \ + hbool_t api_ctx_pushed = FALSE; \ + \ FUNC_ENTER_API_COMMON \ FUNC_ENTER_API_INIT(err); \ FUNC_ENTER_API_PUSH(err); \ @@ -2182,6 +2185,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define FUNC_ENTER_API_NO_EOT(err) \ { \ { \ + hbool_t api_ctx_pushed = FALSE; \ + \ FUNC_ENTER_API_COMMON \ FUNC_ENTER_API_INIT(err); \ FUNC_ENTER_API_PUSH(err); \ @@ -2196,6 +2201,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define FUNC_ENTER_API_NOCLEAR(err) \ { \ { \ + hbool_t api_ctx_pushed = FALSE; \ + \ FUNC_ENTER_API_COMMON \ FUNC_ENTER_API_INIT(err); \ FUNC_ENTER_API_PUSH(err); \ @@ -2270,27 +2277,19 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); FUNC_ENTER_COMMON_NOERR(H5_IS_API(__func__)); \ { -/* Note: this macro only works when there's _no_ interface initialization routine for the module */ -#define FUNC_ENTER_NOAPI_INIT(err) \ - /* Initialize the package, if appropriate */ \ - H5_PACKAGE_INIT(H5_MY_PKG_INIT, err) \ - \ - /* Push the name of this function on the function stack */ \ - H5_PUSH_FUNC - /* Use this macro for all "normal" non-API functions */ #define FUNC_ENTER_NOAPI(err) \ { \ FUNC_ENTER_COMMON(!H5_IS_API(__func__)); \ - FUNC_ENTER_NOAPI_INIT(err) \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + H5_PUSH_FUNC \ + { /* Use this macro for all non-API functions, which propagate errors, but don't issue them */ #define FUNC_ENTER_NOAPI_NOERR \ { \ FUNC_ENTER_COMMON_NOERR(!H5_IS_API(__func__)); \ - FUNC_ENTER_NOAPI_INIT(-) \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + H5_PUSH_FUNC \ + { /* * Use this macro for non-API functions which fall into these categories: @@ -2304,7 +2303,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); { \ FUNC_ENTER_COMMON(!H5_IS_API(__func__)); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* * Use this macro for non-API functions which fall into these categories: @@ -2319,7 +2318,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); { \ FUNC_ENTER_COMMON_NOERR(!H5_IS_API(__func__)); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* * Use this macro for non-API functions which fall into these categories: @@ -2331,9 +2330,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); { \ FUNC_ENTER_COMMON(!H5_IS_API(__func__)); \ \ - /* Initialize the package, if appropriate */ \ - H5_PACKAGE_INIT(H5_MY_PKG_INIT, err) \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* * Use this macro for non-API functions which fall into these categories: @@ -2347,7 +2344,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define FUNC_ENTER_NOAPI_NOERR_NOFS \ { \ FUNC_ENTER_COMMON_NOERR(!H5_IS_API(__func__)); \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* * Use this macro for non-API functions that shouldn't perform _any_ initialization @@ -2369,8 +2366,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); \ FUNC_ENTER_COMMON(!H5_IS_API(__func__)); \ H5AC_tag(tag, &prev_tag); \ - FUNC_ENTER_NOAPI_INIT(err) \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + H5_PUSH_FUNC \ + { #define FUNC_ENTER_NOAPI_NOINIT_TAG(tag) \ { \ @@ -2379,21 +2376,21 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); FUNC_ENTER_COMMON(!H5_IS_API(__func__)); \ H5AC_tag(tag, &prev_tag); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* Use this macro for all "normal" package-level functions */ #define FUNC_ENTER_PACKAGE \ { \ FUNC_ENTER_COMMON(H5_IS_PKG(__func__)); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* Use this macro for package-level functions which propgate errors, but don't issue them */ #define FUNC_ENTER_PACKAGE_NOERR \ { \ FUNC_ENTER_COMMON_NOERR(H5_IS_PKG(__func__)); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* Use the following macro as replacement for the FUNC_ENTER_PACKAGE * macro when the function needs to set up a metadata tag. */ @@ -2404,28 +2401,28 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); FUNC_ENTER_COMMON(H5_IS_PKG(__func__)); \ H5AC_tag(tag, &prev_tag); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* Use this macro for all "normal" staticly-scoped functions */ #define FUNC_ENTER_STATIC \ { \ FUNC_ENTER_COMMON(H5_IS_PKG(__func__)); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* Use this macro for staticly-scoped functions which propgate errors, but don't issue them */ #define FUNC_ENTER_STATIC_NOERR \ { \ FUNC_ENTER_COMMON_NOERR(H5_IS_PKG(__func__)); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* Use this macro for staticly-scoped functions which propgate errors, but don't issue them */ /* And that shouldn't push their name on the function stack */ #define FUNC_ENTER_STATIC_NOERR_NOFS \ { \ FUNC_ENTER_COMMON_NOERR(H5_IS_PKG(__func__)); \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /* * Use this macro for non-API functions that shouldn't perform _any_ initialization @@ -2447,7 +2444,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); FUNC_ENTER_COMMON(H5_IS_PKG(__func__)); \ H5AC_tag(tag, &prev_tag); \ H5_PUSH_FUNC \ - if (H5_PKG_INIT_VAR || !H5_TERM_GLOBAL) { + { /*------------------------------------------------------------------------- * Purpose: Register function exit for code profiling. This should be @@ -2463,15 +2460,18 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); H5_API_SET_CANCEL #define FUNC_LEAVE_API_COMMON(ret_value) \ - ; \ - } /*end scope from end of FUNC_ENTER*/ \ FINISH_MPE_LOG \ H5TRACE_RETURN(ret_value); #define FUNC_LEAVE_API(ret_value) \ + ; \ + } /*end scope from end of FUNC_ENTER*/ \ VFD_SWMR_LEAVE(ret_value); \ FUNC_LEAVE_API_COMMON(ret_value); \ - (void)H5CX_pop(TRUE); \ + if (api_ctx_pushed) { \ + (void)H5CX_pop(TRUE); \ + api_ctx_pushed = FALSE; \ + } \ H5_POP_FUNC \ if (err_occurred) \ (void)H5E_dump_api_stack(TRUE); \ @@ -2482,8 +2482,13 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); /* Use this macro when VFD SWMR EOT is not used on leaving an API function */ #define FUNC_LEAVE_API_NO_EOT(ret_value) \ + ; \ + } /*end scope from end of FUNC_ENTER*/ \ FUNC_LEAVE_API_COMMON(ret_value); \ - (void)H5CX_pop(TRUE); \ + if (api_ctx_pushed) { \ + (void)H5CX_pop(TRUE); \ + api_ctx_pushed = FALSE; \ + } \ H5_POP_FUNC \ if (err_occurred) \ (void)H5E_dump_api_stack(TRUE); \ @@ -2494,6 +2499,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); /* Use this macro to match the FUNC_ENTER_API_NOINIT macro */ #define FUNC_LEAVE_API_NOINIT(ret_value) \ + ; \ + } /*end scope from end of FUNC_ENTER*/ \ FUNC_LEAVE_API_COMMON(ret_value); \ H5_POP_FUNC \ if (err_occurred) \ @@ -2506,6 +2513,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); /* Use this macro to match the FUNC_ENTER_API_NOINIT_NOERR_NOFS macro */ #define FUNC_LEAVE_API_NOFS(ret_value) \ + ; \ + } /*end scope from end of FUNC_ENTER*/ \ FUNC_LEAVE_API_COMMON(ret_value); \ FUNC_LEAVE_API_THREADSAFE \ return (ret_value); \ @@ -2582,27 +2591,6 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); return (ret_value); \ } /*end scope from beginning of FUNC_ENTER*/ -/* Macros to declare package initialization function, if a package initialization routine is defined */ -#ifdef H5_PKG_SINGLE_SOURCE -#define H5_PKG_DECLARE_YES_FUNC(pkg) static herr_t H5_PACKAGE_INIT_FUNC(pkg)(void); -#else -#define H5_PKG_DECLARE_YES_FUNC(pkg) extern herr_t H5_PACKAGE_INIT_FUNC(pkg)(void); -#endif -#define H5_PKG_DECLARE_NO_FUNC(pkg) - -/* Declare package initialization symbols (if in a package) */ -#ifdef H5_PKG_SINGLE_SOURCE -#define H5_PKG_DECLARE_VAR(pkg) static hbool_t H5_PACKAGE_INIT_VAR(pkg); -#else -#define H5_PKG_DECLARE_VAR(pkg) extern hbool_t H5_PACKAGE_INIT_VAR(pkg); -#endif -#define H5_PKG_DECLARE_FUNC(pkg_init, pkg) H5_GLUE3(H5_PKG_DECLARE_, pkg_init, _FUNC)(pkg) - -#ifdef H5_MY_PKG -H5_PKG_DECLARE_VAR(H5_MY_PKG) -H5_PKG_DECLARE_FUNC(H5_MY_PKG_INIT, H5_MY_PKG) -#endif - /* Macro to begin/end tagging (when FUNC_ENTER_*TAG macros are insufficient). * Make sure to use HGOTO_ERROR_TAG and HGOTO_DONE_TAG between these macros! */ #define H5_BEGIN_TAG(tag) \ @@ -2723,7 +2711,8 @@ struct h5_long_options { */ }; -H5_DLL int H5_get_option(int argc, const char **argv, const char *opt, const struct h5_long_options *l_opt); +H5_DLL int H5_get_option(int argc, const char *const *argv, const char *opt, + const struct h5_long_options *l_opt); #ifdef H5_HAVE_PARALLEL /* Generic MPI functions */ @@ -2737,6 +2726,14 @@ H5_DLL herr_t H5_mpi_comm_cmp(MPI_Comm comm1, MPI_Comm comm2, int *result); H5_DLL herr_t H5_mpi_info_cmp(MPI_Info info1, MPI_Info info2, int *result); H5_DLL herr_t H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datatype old_type, MPI_Datatype *new_type); +H5_DLL herr_t H5_mpio_gatherv_alloc(void *send_buf, int send_count, MPI_Datatype send_type, + const int recv_counts[], const int displacements[], + MPI_Datatype recv_type, hbool_t allgather, int root, MPI_Comm comm, + int mpi_rank, int mpi_size, void **out_buf, size_t *out_buf_num_entries); +H5_DLL herr_t H5_mpio_gatherv_alloc_simple(void *send_buf, int send_count, MPI_Datatype send_type, + MPI_Datatype recv_type, hbool_t allgather, int root, MPI_Comm comm, + int mpi_rank, int mpi_size, void **out_buf, + size_t *out_buf_num_entries); #endif /* H5_HAVE_PARALLEL */ /* Functions for debugging */ diff --git a/src/H5public.h b/src/H5public.h index 163deed..3f9848a 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -46,7 +46,7 @@ #include #include -/* Unlike most sys/ headers, which are POSIX-only, sys/types.h is avaible +/* Unlike most sys/ headers, which are POSIX-only, sys/types.h is available * on Windows, though it doesn't necessarily contain all the POSIX types * we need for HDF5 (e.g. ssize_t). */ @@ -83,7 +83,7 @@ /** * For tweaks, bug-fixes, or development */ -#define H5_VERS_RELEASE 1 +#define H5_VERS_RELEASE 2 /** * For pre-releases like \c snap0. Empty string for official releases. */ @@ -91,7 +91,7 @@ /** * Full version string */ -#define H5_VERS_INFO "HDF5 library version: 1.13.1-1" +#define H5_VERS_INFO "HDF5 library version: 1.13.2-1" #define H5check() H5check_version(H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE) @@ -168,6 +168,58 @@ (((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR == Min) && (H5_VERS_RELEASE <= Rel)) || \ ((H5_VERS_MAJOR == Maj) && (H5_VERS_MINOR < Min)) || (H5_VERS_MAJOR < Maj)) +/* Macros for various environment variables that HDF5 interprets */ +/** + * Used to specify the name of an HDF5 Virtual File Driver to use as + * the default file driver for file access. Setting this environment + * variable overrides the default file driver for File Access Property + * Lists. + */ +#define HDF5_DRIVER "HDF5_DRIVER" +/** + * Used to specify a configuration string for the HDF5 Virtual File + * Driver being used for file access. + */ +#define HDF5_DRIVER_CONFIG "HDF5_DRIVER_CONFIG" +/** + * Used to specify the name of an HDF5 Virtual Object Layer Connector + * to use as the default VOL connector for file access. Setting this + * environment variable overrides the default VOL connector for File + * Access Property Lists. + */ +#define HDF5_VOL_CONNECTOR "HDF5_VOL_CONNECTOR" +/** + * Used to specify a delimiter-separated (currently, ';' for Windows + * and ':' for other systems) list of paths that HDF5 should search + * when loading plugins. + */ +#define HDF5_PLUGIN_PATH "HDF5_PLUGIN_PATH" +/** + * Used to control the loading of HDF5 plugins at runtime. If this + * environment variable is set to the special string "::" (defined + * in H5PLpublic.h as H5PL_NO_PLUGIN), then dynamic loading of any + * HDF5 plugins will be disabled. No other values are valid for this + * environment variable. + */ +#define HDF5_PLUGIN_PRELOAD "HDF5_PLUGIN_PRELOAD" +/** + * Used to control whether HDF5 uses file locking when creating or + * opening a file. Valid values for this environment variable are + * as follows: + * + * "TRUE" or "1" - Request that file locks should be used + * "FALSE" or "0" - Request that file locks should NOT be used + * "BEST_EFFORT" - Request that file locks should be used and + * that any locking errors caused by file + * locking being disabled on the system + * should be ignored + */ +#define HDF5_USE_FILE_LOCKING "HDF5_USE_FILE_LOCKING" +/** + * Used to instruct HDF5 not to cleanup files created during testing. + */ +#define HDF5_NOCLEANUP "HDF5_NOCLEANUP" + /** * Status return values. Failed integer functions in HDF5 result almost * always in a negative value (unsigned failing functions sometimes return @@ -237,6 +289,11 @@ typedef long long ssize_t; * \internal Defined as a (minimum) 64-bit integer type. */ typedef uint64_t hsize_t; + +#ifdef H5_HAVE_PARALLEL +#define HSIZE_AS_MPI_TYPE MPI_UINT64_T +#endif + /** * The size of file objects. Used when negative values are needed to indicate errors. * @@ -271,7 +328,7 @@ typedef uint64_t haddr_t; #define HADDR_MAX (HADDR_UNDEF - 1) #ifdef H5_HAVE_PARALLEL -#define HADDR_AS_MPI_TYPE MPI_LONG_LONG_INT +#define HADDR_AS_MPI_TYPE MPI_UINT64_T #endif //! diff --git a/src/H5retry_private.h b/src/H5retry_private.h index 5ff63d3..057c402 100644 --- a/src/H5retry_private.h +++ b/src/H5retry_private.h @@ -81,7 +81,7 @@ h5_retry_init(struct h5_retry_t *r, unsigned int maxtries, uint64_t minival, uin return h5_retry_decrement(r); } -/* If any tries remain, sleep for the mininum interval, or twice the +/* If any tries remain, sleep for the minimum interval, or twice the * previous sleep time, and return true. If no tries remain, return false. */ static inline bool diff --git a/src/H5system.c b/src/H5system.c index e671ea9..b280c84 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -1096,7 +1096,7 @@ const char *H5_optarg; /* Flag argument (or value) */ *------------------------------------------------------------------------- */ int -H5_get_option(int argc, const char **argv, const char *opts, const struct h5_long_options *l_opts) +H5_get_option(int argc, const char *const *argv, const char *opts, const struct h5_long_options *l_opts) { static int sp = 1; /* character index in current token */ int optchar = '?'; /* option character passed back to user */ diff --git a/src/H5timer.c b/src/H5timer.c index b2cc5f0..b5dba97 100644 --- a/src/H5timer.c +++ b/src/H5timer.c @@ -193,17 +193,26 @@ H5_now_usec(void) struct timespec ts; HDclock_gettime(CLOCK_MONOTONIC, &ts); - now = (uint64_t)(ts.tv_sec * (1000 * 1000)) + (uint64_t)(ts.tv_nsec / 1000); + + /* Cast all values in this expression to uint64_t to ensure that all intermediate + * calculations are done in 64 bit, to prevent overflow */ + now = ((uint64_t)ts.tv_sec * ((uint64_t)1000 * (uint64_t)1000)) + + ((uint64_t)ts.tv_nsec / (uint64_t)1000); } #elif defined(H5_HAVE_GETTIMEOFDAY) { struct timeval now_tv; HDgettimeofday(&now_tv, NULL); - now = (uint64_t)(now_tv.tv_sec * (1000 * 1000)) + (uint64_t)now_tv.tv_usec; + + /* Cast all values in this expression to uint64_t to ensure that all intermediate + * calculations are done in 64 bit, to prevent overflow */ + now = ((uint64_t)now_tv.tv_sec * ((uint64_t)1000 * (uint64_t)1000)) + (uint64_t)now_tv.tv_usec; } #else /* H5_HAVE_GETTIMEOFDAY */ - now = (uint64_t)(HDtime(NULL) * (1000 * 1000)); + /* Cast all values in this expression to uint64_t to ensure that all intermediate calculations + * are done in 64 bit, to prevent overflow */ + now = ((uint64_t)HDtime(NULL) * ((uint64_t)1000 * (uint64_t)1000)); #endif /* H5_HAVE_GETTIMEOFDAY */ return (now); diff --git a/src/H5trace.c b/src/H5trace.c index 8daa17d..8defbc1 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -1046,6 +1046,68 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap) } /* end block */ break; + case 'V': /* H5FD_class_value_t */ + { + H5FD_class_value_t class_val = + (H5FD_class_value_t)HDva_arg(ap, H5FD_class_value_t); + + switch (class_val) { + case H5_VFD_INVALID: + H5RS_acat(rs, "H5_VFD_INVALID"); + break; + case H5_VFD_SEC2: + H5RS_acat(rs, "H5_VFD_SEC2"); + break; + case H5_VFD_CORE: + H5RS_acat(rs, "H5_VFD_CORE"); + break; + case H5_VFD_LOG: + H5RS_acat(rs, "H5_VFD_LOG"); + break; + case H5_VFD_FAMILY: + H5RS_acat(rs, "H5_VFD_FAMILY"); + break; + case H5_VFD_MULTI: + H5RS_acat(rs, "H5_VFD_MULTI"); + break; + case H5_VFD_STDIO: + H5RS_acat(rs, "H5_VFD_STDIO"); + break; +#ifdef H5_HAVE_PARALLEL + case H5_VFD_MPIO: + H5RS_acat(rs, "H5_VFD_MPIO"); + break; +#endif +#ifdef H5_HAVE_DIRECT + case H5_VFD_DIRECT: + H5RS_acat(rs, "H5_VFD_DIRECT"); + break; +#endif +#ifdef H5_HAVE_MIRROR_VFD + case H5_VFD_MIRROR: + H5RS_acat(rs, "H5_VFD_MIRROR"); + break; +#endif +#ifdef H5_HAVE_LIBHDFS + case H5_VFD_HDFS: + H5RS_acat(rs, "H5_VFD_HDFS"); + break; +#endif +#ifdef H5_HAVE_ROS3_VFD + case H5_VFD_ROS3: + H5RS_acat(rs, "H5_VFD_ROS3"); + break; +#endif + case H5_VFD_SWMR: + H5RS_acat(rs, "H5_VFD_SWMR"); + break; + default: + H5RS_asprintf_cat(rs, "%ld", (long)class_val); + break; + } + } /* end block */ + break; + default: H5RS_asprintf_cat(rs, "BADTYPE(D%c)", type[1]); goto error; @@ -1189,7 +1251,7 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap) { H5FD_class_t cls = HDva_arg(ap, H5FD_class_t); - H5RS_asprintf_cat(rs, "{'%s', " H5_PRINTF_HADDR_FMT ", ", cls.name, cls.maxaddr); + H5RS_asprintf_cat(rs, "{'%s', %" PRIuHADDR ", ", cls.name, cls.maxaddr); H5_trace_args_close_degree(rs, cls.fc_degree); H5RS_acat(rs, ", ...}"); } /* end block */ @@ -3964,7 +4026,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) H5_timer_get_times(function_timer, &function_times); H5_timer_get_times(running_timer, &running_times); - HDsprintf(tmp, "%.6f", (function_times.elapsed - running_times.elapsed)); + HDsnprintf(tmp, sizeof(tmp), "%.6f", (function_times.elapsed - running_times.elapsed)); H5RS_asprintf_cat(rs, " %*s ", (int)HDstrlen(tmp), ""); } for (i = 0; i < current_depth; i++) diff --git a/src/Makefile.am b/src/Makefile.am index ae058f4..3baf833 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -63,7 +63,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c \ - H5FDmulti.c H5FDsec2.c H5FDspace.c \ + H5FDmulti.c H5FDperform.c H5FDsec2.c H5FDspace.c \ H5FDsplitter.c H5FDstdio.c H5FDtest.c \ H5FDvfd_swmr.c H5FDvfd_swmr_instr.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 61f11a9..22034c8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -89,7 +89,7 @@ if (HDF5_ENABLE_FORMATTERS) endif () ################################################################################# -# If filter and vol plugin tests can be tested +# If filter, vfd and vol plugin tests can be tested ################################################################################# if (BUILD_SHARED_LIBS) # make plugins dir @@ -173,6 +173,50 @@ if (BUILD_SHARED_LIBS) endforeach () ################################################################################# + # make vfd plugins dir + ################################################################################# + file (MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/null_vfd_plugin_dir") + + #----------------------------------------------------------------------------- + # Define VFD Plugin Test Sources + #----------------------------------------------------------------------------- + set (VFD_PLUGIN_LIBS + null_vfd_plugin + ) + + foreach (vfd_lib ${VFD_PLUGIN_LIBS}) + set (HDF5_VFD_PLUGIN_LIB_CORENAME "${vfd_lib}") + set (HDF5_VFD_PLUGIN_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_VFD_PLUGIN_LIB_CORENAME}") + set (HDF5_VFD_PLUGIN_LIB_TARGET ${HDF5_VFD_PLUGIN_LIB_CORENAME}) + + add_library (${HDF5_VFD_PLUGIN_LIB_TARGET} SHARED ${HDF5_TEST_SOURCE_DIR}/${vfd_lib}.c) + target_include_directories (${HDF5_VFD_PLUGIN_LIB_TARGET} PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + TARGET_C_PROPERTIES (${HDF5_VFD_PLUGIN_LIB_TARGET} SHARED) + target_link_libraries (${HDF5_VFD_PLUGIN_LIB_TARGET} PUBLIC ${HDF5_TEST_LIBSH_TARGET}) + H5_SET_LIB_OPTIONS (${HDF5_VFD_PLUGIN_LIB_TARGET} ${HDF5_VFD_PLUGIN_LIB_NAME} SHARED "LIB") + set_target_properties (${HDF5_VFD_PLUGIN_LIB_TARGET} PROPERTIES FOLDER libraries/TEST_PLUGIN) + + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TEST_${HDF5_VFD_PLUGIN_LIB_TARGET}_FORMAT ${HDF5_VFD_PLUGIN_LIB_TARGET}) + endif () + + #----------------------------------------------------------------------------- + # Copy VFD plugin to a plugins folder + #----------------------------------------------------------------------------- + add_custom_command ( + TARGET ${HDF5_VFD_PLUGIN_LIB_TARGET} + POST_BUILD + COMMAND ${CMAKE_COMMAND} + ARGS -E copy_if_different + "$" + "${CMAKE_BINARY_DIR}/null_vfd_plugin_dir/$" + ) + endforeach () + + ################################################################################# # make vol plugins dir ################################################################################# file (MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/null_vol_plugin_dir") @@ -334,7 +378,7 @@ set (H5_TESTS ros3 s3comms hdfs - mirror_vfd + #mirror_vfd # multiple source ntypes dangle dtransform @@ -356,6 +400,9 @@ set (H5_TESTS event_set vfd_swmr ) +if (HDF5_BUILD_UTILS) + set (H5_TESTS ${H5_TESTS} mirror_vfd) +endif () macro (ADD_H5_EXE file) add_executable (${file} ${HDF5_TEST_SOURCE_DIR}/${file}.c) @@ -530,30 +577,25 @@ if (HDF5_ENABLE_FORMATTERS) clang_format (HDF5_TEST_thread_id_FORMAT thread_id) endif () -#-- Adding test for mirror_vfd -add_executable (mirror_vfd ${mirror_vfd_SOURCES}) -target_include_directories (mirror_vfd PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") -if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (mirror_vfd STATIC) - target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIB_TARGET}) -else () - TARGET_C_PROPERTIES (mirror_vfd SHARED) - target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIBSH_TARGET}) -endif () -set_target_properties (mirror_vfd PROPERTIES FOLDER test) - -#----------------------------------------------------------------------------- -# Add Target to clang-format -#----------------------------------------------------------------------------- -if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TEST_mirror_vfd_FORMAT mirror_vfd) -endif () +if (HDF5_BUILD_UTILS) # requires mirror server + #-- Adding test for mirror_vfd + add_executable (mirror_vfd ${mirror_vfd_SOURCES}) + target_include_directories (mirror_vfd PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (mirror_vfd STATIC) + target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIB_TARGET}) + else () + TARGET_C_PROPERTIES (mirror_vfd SHARED) + target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIBSH_TARGET}) + endif () + set_target_properties (mirror_vfd PROPERTIES FOLDER test) -#----------------------------------------------------------------------------- -# Add Target to clang-format -#----------------------------------------------------------------------------- -if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TEST_event_set_FORMAT event_set) + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TEST_mirror_vfd_FORMAT mirror_vfd) + endif () endif () ############################################################################## @@ -595,7 +637,6 @@ endif () set (H5_SWMR_TESTS swmr_addrem_writer - swmr_check_compat_vfd swmr_generator swmr_reader swmr_remove_reader @@ -702,6 +743,19 @@ if (BUILD_SHARED_LIBS) clang_format (HDF5_TEST_filter_plugin_FORMAT filter_plugin) endif () + add_executable (vfd_plugin ${HDF5_TEST_SOURCE_DIR}/vfd_plugin.c) + target_include_directories (vfd_plugin PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + TARGET_C_PROPERTIES (vfd_plugin SHARED) + target_link_libraries (vfd_plugin PRIVATE ${HDF5_TEST_LIBSH_TARGET}) + set_target_properties (vfd_plugin PROPERTIES FOLDER test) + + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TEST_vfd_plugin_FORMAT vfd_plugin) + endif () + add_executable (vol_plugin ${HDF5_TEST_SOURCE_DIR}/vol_plugin.c) target_include_directories (vol_plugin PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") TARGET_C_PROPERTIES (vol_plugin SHARED) @@ -739,24 +793,26 @@ if (HDF5_ENABLE_FORMATTERS) clang_format (HDF5_TEST_use_append_chunk_FORMAT use_append_chunk) endif () -set (use_append_chunk_mirror_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_chunk_mirror.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) -add_executable (use_append_chunk_mirror ${use_append_chunk_mirror_SOURCES}) -target_compile_options(use_append_chunk_mirror PRIVATE "${HDF5_CMAKE_C_FLAGS}") -target_include_directories (use_append_chunk_mirror PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") -if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (use_append_chunk_mirror STATIC) - target_link_libraries (use_append_chunk_mirror PRIVATE ${HDF5_TEST_LIB_TARGET}) -else () - TARGET_C_PROPERTIES (use_append_chunk_mirror SHARED) - target_link_libraries (use_append_chunk_mirror PRIVATE ${HDF5_TEST_LIBSH_TARGET}) -endif () -set_target_properties (use_append_chunk_mirror PROPERTIES FOLDER test) +if (HDF5_BUILD_UTILS) # requires mirror server + set (use_append_chunk_mirror_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_chunk_mirror.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) + add_executable (use_append_chunk_mirror ${use_append_chunk_mirror_SOURCES}) + target_compile_options(use_append_chunk_mirror PRIVATE "${HDF5_CMAKE_C_FLAGS}") + target_include_directories (use_append_chunk_mirror PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (use_append_chunk_mirror STATIC) + target_link_libraries (use_append_chunk_mirror PRIVATE ${HDF5_TEST_LIB_TARGET}) + else () + TARGET_C_PROPERTIES (use_append_chunk_mirror SHARED) + target_link_libraries (use_append_chunk_mirror PRIVATE ${HDF5_TEST_LIBSH_TARGET}) + endif () + set_target_properties (use_append_chunk_mirror PROPERTIES FOLDER test) -#----------------------------------------------------------------------------- -# Add Target to clang-format -#----------------------------------------------------------------------------- -if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TEST_use_append_chunk_mirror_FORMAT use_append_chunk_mirror) + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TEST_use_append_chunk_mirror_FORMAT use_append_chunk_mirror) + endif () endif () set (use_append_mchunks_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_mchunks.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index bb5356d..4e828f8 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -127,10 +127,10 @@ set (HDF5_REFERENCE_TEST_FILES btree_idx_1_8.h5 corrupt_stab_msg.h5 deflate.h5 - family_v16_00000.h5 - family_v16_00001.h5 - family_v16_00002.h5 - family_v16_00003.h5 + family_v16-000000.h5 + family_v16-000001.h5 + family_v16-000002.h5 + family_v16-000003.h5 file_image_core_test.h5 filespace_1_6.h5 filespace_1_8.h5 @@ -331,7 +331,7 @@ set (test_CLEANFILES sec2_file.h5 direct_file.h5 family_file000*.h5 - new_family_v16_000*.h5 + new_family_v16-000*.h5 multi_file-*.h5 core_file filter_plugin.h5 @@ -649,10 +649,11 @@ set_tests_properties (H5TEST-tcheck_version-minor PROPERTIES WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST WILL_FAIL "true" ) -# release + 1 should pass +# release + 1 should pass on non-develop branches add_test (NAME H5TEST-tcheck_version-release COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ "-tr") set_tests_properties (H5TEST-tcheck_version-release PROPERTIES WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST + WILL_FAIL "true" ) ############################################################################## @@ -879,8 +880,6 @@ if (ENABLE_EXTENDED_TESTS) # testswmr.sh: swmr* # testvdsswmr.sh: vds_swmr* -# add_test (NAME H5Test-swmr_check_compat_vfd COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) - #-- Adding test for flushrefresh file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/flushrefresh_test") if (H5_PERL_FOUND) diff --git a/test/CMakeVFDTests.cmake b/test/CMakeVFDTests.cmake index 892ccf3..e08e4d2 100644 --- a/test/CMakeVFDTests.cmake +++ b/test/CMakeVFDTests.cmake @@ -16,22 +16,10 @@ ############################################################################## ############################################################################## # included from CMakeTests.cmake +H5_CREATE_VFD_DIR() -set (VFD_LIST - sec2 - stdio - core - core_paged - split - multi - family -) -if (H5_HAVE_DIRECT) - set (VFD_LIST ${VFD_LIST} direct) -endif () - +# create more test folders for each VFD foreach (vfdtest ${VFD_LIST}) - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdtest}") file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdtest}/testfiles") file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdtest}/testfiles/plist_files") endforeach () @@ -208,3 +196,22 @@ add_custom_target(HDF5_VFDTEST_LIB_files ALL COMMENT "Copying files needed by HD foreach (h5_vfd ${VFD_LIST}) ADD_VFD_TEST (${h5_vfd} 0) endforeach () + + ############################################################################## + ### V F D P L U G I N T E S T S + ############################################################################## + if (BUILD_SHARED_LIBS) + if (WIN32) + set (CMAKE_SEP "\;") + set (BIN_REL_PATH "../../") + else () + set (CMAKE_SEP ":") + set (BIN_REL_PATH "../") + endif () + + add_test (NAME H5PLUGIN-vfd_plugin COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) + set_tests_properties (H5PLUGIN-vfd_plugin PROPERTIES + ENVIRONMENT "HDF5_PLUGIN_PATH=${CMAKE_BINARY_DIR}/null_vfd_plugin_dir;srcdir=${HDF5_TEST_BINARY_DIR}" + WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR} + ) + endif () diff --git a/test/Makefile.am b/test/Makefile.am index f7ed4c6..2a22ac4 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -37,8 +37,7 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_builddir)/src # test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh testexternal_env.sh \ testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh \ - testvfdswmr.sh \ - test_mirror.sh + testvfdswmr.sh SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ external_env$(EXEEXT) filenotclosed$(EXEEXT) del_many_dense_attrs$(EXEEXT) \ flushrefresh$(EXEEXT) use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) use_disable_mdc_flushes$(EXEEXT) \ @@ -57,13 +56,16 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ vfd_swmr_gperf_reader$(EXEEXT) vfd_swmr_gperf_writer$(EXEEXT) \ vfd_swmr_gfail_reader$(EXEEXT) vfd_swmr_gfail_writer$(EXEEXT) \ vfd_swmr_indep_wr_p0$(EXEEXT) vfd_swmr_indep_wr_p1$(EXEEXT) \ - vds_env$(EXEEXT) \ - vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT) + vds_env$(EXEEXT) vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT) if HAVE_SHARED_CONDITIONAL TEST_SCRIPT += test_filter_plugin.sh test_vol_plugin.sh SCRIPT_DEPEND += filter_plugin$(EXEEXT) vol_plugin$(EXEEXT) endif +if MIRROR_VFD_CONDITIONAL + TEST_SCRIPT += test_mirror.sh +endif + check_SCRIPTS = $(TEST_SCRIPT) # These are our main targets. They should be listed in the order to be @@ -117,12 +119,14 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \ vfd_swmr_gperf_reader vfd_swmr_gperf_writer \ vfd_swmr_gfail_reader vfd_swmr_gfail_writer \ vfd_swmr_indep_wr_p0 vfd_swmr_indep_wr_p1 \ - vfd_swmr_check_compat \ vfd_swmr_dsetchks_reader vfd_swmr_dsetchks_writer \ - swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer \ - mirror_vfd + vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer if HAVE_SHARED_CONDITIONAL - check_PROGRAMS+= filter_plugin vol_plugin + check_PROGRAMS+= filter_plugin vfd_plugin vol_plugin +endif + +if MIRROR_VFD_CONDITIONAL + check_PROGRAMS+= mirror_vfd endif # These programs generate test files for the tests. They don't need to be @@ -145,7 +149,7 @@ if HAVE_SHARED_CONDITIONAL # The libh5test library provides common support code for the tests. # The filter_plugin* libraries are for use in filter_plugin.c. # Build them as shared libraries if that option was enabled in configure. - noinst_LTLIBRARIES=libh5test.la libfilter_plugin1_dsets.la libfilter_plugin2_dsets.la libfilter_plugin3_dsets.la libfilter_plugin4_groups.la libnull_vol_connector.la + noinst_LTLIBRARIES=libh5test.la libfilter_plugin1_dsets.la libfilter_plugin2_dsets.la libfilter_plugin3_dsets.la libfilter_plugin4_groups.la libnull_vfd_plugin.la libnull_vol_connector.la libfilter_plugin1_dsets_la_SOURCES=filter_plugin1_dsets.c libfilter_plugin2_dsets_la_SOURCES=filter_plugin2_dsets.c libfilter_plugin3_dsets_la_SOURCES=filter_plugin3_dsets.c @@ -157,6 +161,12 @@ if HAVE_SHARED_CONDITIONAL libfilter_plugin3_dsets_la_LIBADD=$(LIBHDF5) libfilter_plugin4_groups_la_LIBADD=$(LIBHDF5) + # VFD plugin test libraries + # + # null_vfd_plugin is used for testing basic VFD plugin functionality. + libnull_vfd_plugin_la_SOURCES=null_vfd_plugin.c + libnull_vfd_plugin_la_LDFLAGS=$(AM_LDFLAGS) -avoid-version -module -shared -export-dynamic -rpath /nowhere + # VOL plugin test libraries # # null_vol_connector is used for testing basic VOL plugin functionality. @@ -238,7 +248,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse enum1.h5 titerate.h5 ttsafe.h5 tarray1.h5 tgenprop.h5 \ tmisc[0-9]*.h5 set_extent[1-5].h5 ext[12].bin \ getname.h5 getname[1-3].h5 sec2_file.h5 direct_file.h5 \ - family_file000[0-3][0-9].h5 new_family_v16_000[0-3][0-9].h5 \ + family_file000[0-3][0-9].h5 new_family_v16-000[0-3][0-9].h5 \ multi_file-[rs].h5 core_file filter_plugin.h5 \ new_move_[ab].h5 ntypes.h5 dangle.h5 error_test.h5 err_compat.h5 \ dtransform.h5 test_filters.h5 get_file_name.h5 tstint[1-2].h5 \ diff --git a/test/SWMR_POSIX_Order_UG.txt b/test/SWMR_POSIX_Order_UG.txt index 2771af1..a4190bc 100644 --- a/test/SWMR_POSIX_Order_UG.txt +++ b/test/SWMR_POSIX_Order_UG.txt @@ -3,7 +3,7 @@ POSIX Write Order Test Instructions Purpose ------- -This documents shows the requirments, implementaion design and instructions +This documents shows the requirements, implementation design and instructions of building and running the POSIX Write Order test. The name of the test is twriteorder and it resides in the test/ directory. @@ -44,7 +44,7 @@ and they are ready for the reader processes to access. If the system, in which the writer and reader processes run, the readers will always get all chain-linked blocks correctly. If the order of write -is not maintained, some reader processes may found unexpect block data. +is not maintained, some reader processes may found unexpected block data. Building the Tests ------------------ diff --git a/test/SWMR_UseCase_UG.txt b/test/SWMR_UseCase_UG.txt index 1e3d1e6..677fdc4 100644 --- a/test/SWMR_UseCase_UG.txt +++ b/test/SWMR_UseCase_UG.txt @@ -2,7 +2,7 @@ User Guide for SWMR Use Case Programs 2. Purpose: - This is a User Guide of the SWMR Use Case programs. It descibes the use + This is a User Guide of the SWMR Use Case programs. It describes the use case program and explain how to run them. 2.1. Author and Dates: @@ -87,7 +87,7 @@ It opens the HDF5 data file without the SWMR access mode (0 means off). This likely will result in error. This option is provided for - users to see the effect of the neede SWMR access mode for concurrent + users to see the effect of the needed SWMR access mode for concurrent access. 3.4. Test Shell Script: @@ -176,7 +176,7 @@ It opens the HDF5 data file without the SWMR access mode (0 means off). This likely will result in error. This option is provided for - users to see the effect of the neede SWMR access mode for concurrent + users to see the effect of the needed SWMR access mode for concurrent access. 4.4. Test Shell Script: diff --git a/test/ShellTests.cmake b/test/ShellTests.cmake index 9614152..e7de735 100644 --- a/test/ShellTests.cmake +++ b/test/ShellTests.cmake @@ -15,12 +15,35 @@ ### T E S T I N G S H E L L S C R I P T S ### ############################################################################## -if (UNIX) +find_program (PWSH NAMES pwsh powershell) +if (PWSH) + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/usecases_test") + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/swmr_test") + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/vds_swmr_test") + set (srcdir ${HDF5_TEST_SOURCE_DIR}) + set (H5_UTILS_TEST_BUILDDIR ${CMAKE_TEST_OUTPUT_DIRECTORY}) + set (H5_TEST_BUILDDIR ${HDF5_TEST_BINARY_DIR}/H5TEST) + configure_file(${HDF5_TEST_SOURCE_DIR}/testswmr.pwsh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/testswmr.ps1 @ONLY) + # test commented out as currently the programs are not allowing another access to the data file + #add_test (H5SHELL-testswmr ${PWSH} ${HDF5_TEST_BINARY_DIR}/H5TEST/testswmr.ps1) + #set_tests_properties (H5SHELL-testswmr PROPERTIES + # ENVIRONMENT "PATH=$ENV{PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" + # WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST + #) + configure_file(${HDF5_TEST_SOURCE_DIR}/testvdsswmr.pwsh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/testvdsswmr.ps1 @ONLY) + # test commented out as currently the programs are not allowing another access to the data file + #add_test (H5SHELL-testvdsswmr ${PWSH} ${HDF5_TEST_BINARY_DIR}/H5TEST/testvdsswmr.ps1) + #set_tests_properties (H5SHELL-testvdsswmr PROPERTIES + # ENVIRONMENT "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" + # WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST + #) +elseif (UNIX) find_program (SH_PROGRAM bash) if (SH_PROGRAM) set (srcdir ${HDF5_TEST_SOURCE_DIR}) - set (bindir ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + set (H5_UTILS_TEST_BUILDDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + set (H5_TEST_BUILDDIR ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) ############################################################################## # configure scripts to test dir ############################################################################## @@ -34,22 +57,13 @@ if (UNIX) ############################################################################## # copy test programs to test dir ############################################################################## - #shell script creates dir - #file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/swmr_test") add_custom_command ( - TARGET swmr_check_compat_vfd + TARGET accum_swmr_reader POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different "${HDF5_SOURCE_DIR}/bin/output_filter.sh" "${HDF5_TEST_BINARY_DIR}/H5TEST/bin/output_filter.sh" ) - #shell script creates dir - #file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/usecases_test") - - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/swmr_test") - - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/vds_swmr_test") - ############################################################################## ############################################################################## ### A D D I T I O N A L T E S T S ### @@ -105,6 +119,5 @@ if (UNIX) ENVIRONMENT "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST ) - endif () endif () diff --git a/test/accum.c b/test/accum.c index c3346bc..080edac 100644 --- a/test/accum.c +++ b/test/accum.c @@ -1348,7 +1348,7 @@ test_accum_adjust(H5F_t *f) /* ==> Accumulator will try to resize, but see that it's getting too big */ /* ==> Size of new block is less than than half maximum size of accumulator */ /* ==> New block being appended to accumulator */ - /* ==> We can slide the dirty region down, to accomodate the request */ + /* ==> We can slide the dirty region down, to accommodate the request */ /* ==> Max Buffer Size - (dirty offset + adjust size) >= 2 * size) */ /* ==> Need to adjust location of accumulator while appending */ /* ==> Accumulator will need to be reallocated */ @@ -1394,7 +1394,7 @@ test_accum_adjust(H5F_t *f) /* ==> Accumulator will try to resize, but see that it's getting too big */ /* ==> Size of new block is less than than half maximum size of accumulator */ /* ==> New block being appended to accumulator */ - /* ==> We can slide the dirty region down, to accomodate the request */ + /* ==> We can slide the dirty region down, to accommodate the request */ /* ==> Max Buffer Size - (dirty offset + adjust size) < 2 * size) */ /* ==> Need to adjust location of accumulator while appending */ if (accum_write((1024 * 1024) - 5, 10, wbuf) < 0) @@ -2121,7 +2121,7 @@ test_swmr_write_big(hbool_t newest_format) /* Skip this test if SWMR I/O is not supported for the VFD specified * by the environment variable. */ - driver = HDgetenv("HDF5_DRIVER"); + driver = HDgetenv(HDF5_DRIVER); if (!H5FD__supports_swmr_test(driver)) { SKIPPED(); HDputs(" Test skipped due to VFD not supporting SWMR I/O."); diff --git a/test/accum_swmr_reader.c b/test/accum_swmr_reader.c index ce0e830..76f45bf 100644 --- a/test/accum_swmr_reader.c +++ b/test/accum_swmr_reader.c @@ -56,7 +56,7 @@ main(void) /* Skip this test if SWMR I/O is not supported for the VFD specified * by the environment variable. */ - driver = HDgetenv("HDF5_DRIVER"); + driver = HDgetenv(HDF5_DRIVER); if (!H5FD__supports_swmr_test(driver)) return EXIT_SUCCESS; diff --git a/test/app_ref.c b/test/app_ref.c index 09cfb72..c4ac60c 100644 --- a/test/app_ref.c +++ b/test/app_ref.c @@ -69,8 +69,10 @@ Abrt_Handler(int H5_ATTR_UNUSED sig) { int i, n; + const char *string = " ID reference count: "; for (i = 0; i < T_NUMCLASSES; i++) { - HDfprintf(stderr, "%s ID reference count: %n", IDNAME[i], &n); + HDfprintf(stderr, "%s%s", IDNAME[i], string); + n = (int)(strlen(IDNAME[i]) + strlen(string)); HDfprintf(stderr, "%*d\n", (n < ERR_WIDTH) ? (ERR_WIDTH - n) : 0, rc[i]); } } @@ -79,11 +81,12 @@ Abrt_Handler(int H5_ATTR_UNUSED sig) int main(void) { - hid_t ids[T_NUMCLASSES]; - hid_t fapl; /* File Access Property List */ - int ninc; - int i; - char filename[1024]; + const char *env_h5_drvr; /* File Driver value from environment */ + hid_t ids[T_NUMCLASSES]; + hid_t fapl; /* File Access Property List */ + int ninc; + int i; + char filename[1024]; h5_reset(); h5_fixname(FILENAME[0], H5P_DEFAULT, filename, sizeof filename); @@ -92,6 +95,19 @@ main(void) TESTING("library shutdown with reference count > 1"); + /* Get the VFD to use */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* Don't run this test with the multi/split VFD. A bug in library shutdown + * ordering causes problems with the multi VFD when IDs are left dangling. + */ + if (!HDstrcmp(env_h5_drvr, "multi") || !HDstrcmp(env_h5_drvr, "split")) { + HDputs("\n -- SKIPPED for incompatible VFD --"); + return 0; + } + /* Create the file */ if ((ids[T_FILE] = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR diff --git a/test/atomic_writer.c b/test/atomic_writer.c index 5a6310a..33252f1 100644 --- a/test/atomic_writer.c +++ b/test/atomic_writer.c @@ -112,7 +112,7 @@ int main(int argc, char *argv[]) { int fd = -1; /* file descriptor */ - ssize_t bytes_wrote; /* the nubmer of bytes written */ + ssize_t bytes_wrote; /* the number of bytes written */ unsigned int *buf = NULL; /* buffer to hold written data */ unsigned int n, u, i; /* local index variable */ int temp; /* temporary variable */ diff --git a/test/big.c b/test/big.c index 6f8ce67..43fef56 100644 --- a/test/big.c +++ b/test/big.c @@ -43,7 +43,7 @@ * that disk space is allocated only when the contents are actually written. * E.g., If one creates a new file, seeks forward 10 million bytes, writes * 1 bytes and closes the file, then a sparse file, will show file size of - * 10 million bytes but actaully uses only couple disk blocks, much smaller + * 10 million bytes but actually uses only couple disk blocks, much smaller * than the formal file size.) * * One more consideration is that we want to distinguish an HDF5 library @@ -70,7 +70,7 @@ #endif /* Define Small, Large, Extra Large, Huge File which - * corrspond to less than 2GB, 2GB, 4GB, and tens of GB file size. + * correspond to less than 2GB, 2GB, 4GB, and tens of GB file size. * NO_FILE stands for "no file" to be tested. */ typedef enum fsizes_t { SFILE, LFILE, XLFILE, HUGEFILE, NO_FILE } fsizes_t; @@ -607,9 +607,9 @@ usage(void) "\t-fsize\tChange family size default to where is\n" "\t\ta positive float point number. Default value is %" PRIuHSIZE ".\n" "Examples:\n" - "\tbig -fsize 2.1e9 \t# test with file size just under 2GB\n" - "\tbig -fsize 2.2e9 \t# test with file size just above 2GB\n" - "\tBe sure the file system can support the file size requested\n", + "\t big -fsize 2.1e9 \t# test with file size just under 2GB\n" + "\t big -fsize 2.2e9 \t# test with file size just above 2GB\n" + "\t Be sure the file system can support the file size requested\n", (hsize_t)FAMILY_SIZE); } diff --git a/test/btree2.c b/test/btree2.c index aca0c77..9cbb211 100644 --- a/test/btree2.c +++ b/test/btree2.c @@ -700,7 +700,7 @@ test_insert_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (H5B2_insert(bt2, &record) < 0) FAIL_STACK_ERROR - /* Attempt to find non-existant record in B-tree with 1 record */ + /* Attempt to find non-existent record in B-tree with 1 record */ /* (Should not be found, but not fail) */ idx = 41; found = FALSE; @@ -717,7 +717,7 @@ test_insert_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (found) TEST_ERROR - /* Attempt to find existant record in B-tree with 1 record */ + /* Attempt to find existent record in B-tree with 1 record */ idx = 42; found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) @@ -732,7 +732,7 @@ test_insert_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (!found) TEST_ERROR - /* Attempt to index non-existant record in B-tree with 1 record */ + /* Attempt to index non-existent record in B-tree with 1 record */ idx = 0; H5E_BEGIN_TRY { @@ -780,7 +780,7 @@ test_insert_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (H5B2_insert(bt2, &record) < 0) FAIL_STACK_ERROR - /* Attempt to find non-existant record in level-0 B-tree with several records */ + /* Attempt to find non-existent record in level-0 B-tree with several records */ /* (Should not be found, but not fail) */ idx = 41; found = FALSE; @@ -789,7 +789,7 @@ test_insert_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (found) TEST_ERROR - /* Attempt to find existant record in level-0 B-tree with several record */ + /* Attempt to find existent record in level-0 B-tree with several record */ idx = 56; found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) @@ -797,7 +797,7 @@ test_insert_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (!found) TEST_ERROR - /* Attempt to index non-existant record in B-tree with several records */ + /* Attempt to index non-existent record in B-tree with several records */ idx = 0; H5E_BEGIN_TRY { @@ -958,7 +958,7 @@ test_insert_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (idx != (INSERT_SPLIT_ROOT_NREC + 2)) TEST_ERROR - /* Attempt to find non-existant record in level-1 B-tree */ + /* Attempt to find non-existent record in level-1 B-tree */ /* (Should not be found, but not fail) */ idx = INSERT_SPLIT_ROOT_NREC + 10; found = FALSE; @@ -967,7 +967,7 @@ test_insert_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (found) TEST_ERROR - /* Attempt to find existant record in root of level-1 B-tree */ + /* Attempt to find existent record in root of level-1 B-tree */ idx = 33; found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) @@ -975,7 +975,7 @@ test_insert_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (!found) TEST_ERROR - /* Attempt to find existant record in leaf of level-1 B-tree */ + /* Attempt to find existent record in leaf of level-1 B-tree */ idx = 56; found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) @@ -983,7 +983,7 @@ test_insert_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (!found) TEST_ERROR - /* Attempt to index non-existant record in level-1 B-tree */ + /* Attempt to index non-existent record in level-1 B-tree */ idx = 0; H5E_BEGIN_TRY { @@ -1704,7 +1704,7 @@ test_insert_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (idx != ((INSERT_SPLIT_ROOT_NREC * 29) + 5)) TEST_ERROR - /* Attempt to find non-existant record in level-2 B-tree */ + /* Attempt to find non-existent record in level-2 B-tree */ /* (Should not be found, but not fail) */ idx = INSERT_SPLIT_ROOT_NREC * 30; found = FALSE; @@ -1713,7 +1713,7 @@ test_insert_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (found) TEST_ERROR - /* Attempt to find existant record in root of level-2 B-tree */ + /* Attempt to find existent record in root of level-2 B-tree */ idx = 948; found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) @@ -1726,7 +1726,7 @@ test_insert_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)2) < 0) TEST_ERROR - /* Attempt to find existant record in internal node of level-2 B-tree */ + /* Attempt to find existent record in internal node of level-2 B-tree */ idx = 505; found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) @@ -1739,7 +1739,7 @@ test_insert_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)1) < 0) TEST_ERROR - /* Attempt to find existant record in leaf of level-2 B-tree */ + /* Attempt to find existent record in leaf of level-2 B-tree */ idx = 555; found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) @@ -1752,7 +1752,7 @@ test_insert_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)0) < 0) TEST_ERROR - /* Attempt to index non-existant record in level-2 B-tree */ + /* Attempt to index non-existent record in level-2 B-tree */ idx = 0; H5E_BEGIN_TRY { @@ -3106,7 +3106,7 @@ HDfprintf(stderr,"curr_time=%lu\n",(unsigned long)curr_time); if (idx != INSERT_MANY) TEST_ERROR - /* Attempt to find non-existant record in level-4 B-tree */ + /* Attempt to find non-existent record in level-4 B-tree */ /* (Should not be found, but not fail) */ idx = INSERT_MANY * 2; found = FALSE; @@ -3120,7 +3120,7 @@ HDfprintf(stderr,"curr_time=%lu\n",(unsigned long)curr_time); /* Pick random record */ idx = (hsize_t)(HDrandom() % INSERT_MANY); - /* Attempt to find existant record in root of level-4 B-tree */ + /* Attempt to find existent record in root of level-4 B-tree */ found = FALSE; if (H5B2_find(bt2, &idx, &found, find_cb, &idx) < 0) FAIL_STACK_ERROR @@ -3128,7 +3128,7 @@ HDfprintf(stderr,"curr_time=%lu\n",(unsigned long)curr_time); TEST_ERROR } /* end for */ - /* Attempt to index non-existant record in level-4 B-tree, in increasing & decreasing order */ + /* Attempt to index non-existent record in level-4 B-tree, in increasing & decreasing order */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)(INSERT_MANY * 3), find_cb, NULL); @@ -3151,12 +3151,12 @@ HDfprintf(stderr,"curr_time=%lu\n",(unsigned long)curr_time); /* Pick random record */ idx = (hsize_t)(HDrandom() % INSERT_MANY); - /* Attempt to find existant record in root of level-4 B-tree */ + /* Attempt to find existent record in root of level-4 B-tree */ /* (in increasing order) */ if (H5B2_index(bt2, H5_ITER_INC, idx, find_cb, &idx) < 0) FAIL_STACK_ERROR - /* Attempt to find existant record in root of level-4 B-tree */ + /* Attempt to find existent record in root of level-4 B-tree */ /* (in decreasing order) */ if (H5B2_index(bt2, H5_ITER_DEC, idx, find_dec_cb, &idx) < 0) FAIL_STACK_ERROR @@ -3263,7 +3263,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (H5B2_update(bt2, &record, no_modify_cb, NULL) < 0) FAIL_STACK_ERROR - /* Attempt to find non-existant record in B-tree with 1 record */ + /* Attempt to find non-existent record in B-tree with 1 record */ /* (Should not be found, but not fail) */ find.key = 10; find.val = (hsize_t)-1; @@ -3285,7 +3285,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (found) TEST_ERROR - /* Attempt to find existant record in B-tree with 1 record */ + /* Attempt to find existent record in B-tree with 1 record */ find.key = 42; found = FALSE; if (H5B2_find(bt2, &find, &found, find_rec_cb, &find) < 0) @@ -3305,7 +3305,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (!found) TEST_ERROR - /* Attempt to index non-existant record in B-tree with 1 record */ + /* Attempt to index non-existent record in B-tree with 1 record */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)1, index_rec_cb, NULL); @@ -3340,7 +3340,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (H5B2_update(bt2, &modify, modify_rec_cb, &modify) < 0) FAIL_STACK_ERROR - /* Attempt to find non-existant record in B-tree with 1 record */ + /* Attempt to find non-existent record in B-tree with 1 record */ /* (Should not be found, but not fail) */ find.key = 10; find.val = (hsize_t)-1; @@ -3382,7 +3382,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (!found) TEST_ERROR - /* Attempt to index non-existant record in B-tree with 1 record */ + /* Attempt to index non-existent record in B-tree with 1 record */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)1, index_rec_cb, NULL); @@ -3437,7 +3437,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (H5B2_update(bt2, &record, no_modify_cb, NULL) < 0) FAIL_STACK_ERROR - /* Attempt to find non-existant record in level-0 B-tree with several records */ + /* Attempt to find non-existent record in level-0 B-tree with several records */ /* (Should not be found, but not fail) */ find.key = 10; find.val = (hsize_t)-1; @@ -3449,7 +3449,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (found) TEST_ERROR - /* Attempt to find existant record in level-0 B-tree with several records */ + /* Attempt to find existent record in level-0 B-tree with several records */ find.key = 56; found = FALSE; if (H5B2_find(bt2, &find, &found, find_rec_cb, &find) < 0) @@ -3459,7 +3459,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (!found) TEST_ERROR - /* Attempt to index non-existant record in B-tree with several records */ + /* Attempt to index non-existent record in B-tree with several records */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)4, index_rec_cb, NULL); @@ -3531,7 +3531,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (H5B2_update(bt2, &record, modify_rec_cb, &modify) < 0) FAIL_STACK_ERROR - /* Attempt to find non-existant record in level-0 B-tree with several records */ + /* Attempt to find non-existent record in level-0 B-tree with several records */ /* (Should not be found, but not fail) */ find.key = 41; find.val = (hsize_t)-1; @@ -3543,7 +3543,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (found) TEST_ERROR - /* Attempt to find existant record in level-0 B-tree with several record */ + /* Attempt to find existent record in level-0 B-tree with several record */ find.key = 56; found = FALSE; if (H5B2_find(bt2, &find, &found, find_rec_cb, &find) < 0) @@ -3553,7 +3553,7 @@ test_update_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (!found) TEST_ERROR - /* Attempt to index non-existant record in B-tree with several records */ + /* Attempt to index non-existent record in B-tree with several records */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)4, index_rec_cb, NULL); @@ -3782,7 +3782,7 @@ test_update_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (idx.key != (INSERT_SPLIT_ROOT_NREC_REC + 2)) TEST_ERROR - /* Attempt to find non-existant record in level-1 B-tree */ + /* Attempt to find non-existent record in level-1 B-tree */ /* (Should not be found, but not fail) */ find.key = 800; find.val = (hsize_t)-1; @@ -3794,7 +3794,7 @@ test_update_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (found) TEST_ERROR - /* Attempt to find existant record in root of level-1 B-tree */ + /* Attempt to find existent record in root of level-1 B-tree */ find.key = 33; find.val = (hsize_t)-1; found = FALSE; @@ -3807,7 +3807,7 @@ test_update_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (!found) TEST_ERROR - /* Attempt to find existant record in leaf of level-1 B-tree */ + /* Attempt to find existent record in leaf of level-1 B-tree */ find.key = 56; find.val = (hsize_t)-1; found = FALSE; @@ -3820,7 +3820,7 @@ test_update_split_root(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_p if (!found) TEST_ERROR - /* Attempt to index non-existant record in level-1 B-tree */ + /* Attempt to index non-existent record in level-1 B-tree */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)(INSERT_SPLIT_ROOT_NREC_REC + 2), index_rec_cb, NULL); @@ -4578,7 +4578,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (idx.key != ((INSERT_SPLIT_ROOT_NREC_REC * 41) + 4)) TEST_ERROR - /* Attempt to find non-existant record in level-2 B-tree */ + /* Attempt to find non-existent record in level-2 B-tree */ /* (Should not be found, but not fail) */ find.key = INSERT_SPLIT_ROOT_NREC_REC * 42; find.val = (hsize_t)-1; @@ -4590,7 +4590,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (found) TEST_ERROR - /* Attempt to find existant record in root of level-2 B-tree */ + /* Attempt to find existent record in root of level-2 B-tree */ find.key = 1347; find.val = (hsize_t)-1; found = FALSE; @@ -4606,7 +4606,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)2) < 0) TEST_ERROR - /* Attempt to find existant record in internal node of level-2 B-tree */ + /* Attempt to find existent record in internal node of level-2 B-tree */ find.key = 513; find.val = (hsize_t)-1; found = FALSE; @@ -4622,7 +4622,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)1) < 0) TEST_ERROR - /* Attempt to find existant record in leaf of level-2 B-tree */ + /* Attempt to find existent record in leaf of level-2 B-tree */ find.key = 555; find.val = (hsize_t)-1; found = FALSE; @@ -4638,7 +4638,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)0) < 0) TEST_ERROR - /* Attempt to index non-existant record in level-2 B-tree */ + /* Attempt to index non-existent record in level-2 B-tree */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)(INSERT_SPLIT_ROOT_NREC_REC * 42), index_rec_cb, NULL); @@ -4755,7 +4755,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (idx.key != ((INSERT_SPLIT_ROOT_NREC_REC * 41) + 4)) TEST_ERROR - /* Attempt to find non-existant record in level-2 B-tree */ + /* Attempt to find non-existent record in level-2 B-tree */ /* (Should not be found, but not fail) */ find.key = INSERT_SPLIT_ROOT_NREC_REC * 42; find.val = (hsize_t)-1; @@ -4767,7 +4767,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (found) TEST_ERROR - /* Attempt to find existant record in root of level-2 B-tree */ + /* Attempt to find existent record in root of level-2 B-tree */ find.key = 1344; find.val = (hsize_t)-1; found = FALSE; @@ -4783,7 +4783,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)2) < 0) TEST_ERROR - /* Attempt to find existant record in internal node of level-2 B-tree */ + /* Attempt to find existent record in internal node of level-2 B-tree */ find.key = 512; find.val = (hsize_t)-1; found = FALSE; @@ -4799,7 +4799,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)1) < 0) TEST_ERROR - /* Attempt to find existant record in leaf of level-2 B-tree */ + /* Attempt to find existent record in leaf of level-2 B-tree */ find.key = 555; find.val = (hsize_t)-1; found = FALSE; @@ -4815,7 +4815,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)0) < 0) TEST_ERROR - /* Attempt to index non-existant record in level-2 B-tree */ + /* Attempt to index non-existent record in level-2 B-tree */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)(INSERT_SPLIT_ROOT_NREC_REC * 42), index_rec_cb, NULL); @@ -4942,7 +4942,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (idx.key != ((INSERT_SPLIT_ROOT_NREC_REC * 41) + 6)) TEST_ERROR - /* Attempt to find non-existant record in level-2 B-tree */ + /* Attempt to find non-existent record in level-2 B-tree */ /* (Should not be found, but not fail) */ find.key = INSERT_SPLIT_ROOT_NREC_REC * 42; find.val = (hsize_t)-1; @@ -4954,7 +4954,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (found) TEST_ERROR - /* Attempt to find existant record in root of level-2 B-tree */ + /* Attempt to find existent record in root of level-2 B-tree */ find.key = 1345; find.val = (hsize_t)-1; found = FALSE; @@ -4970,7 +4970,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)2) < 0) TEST_ERROR - /* Attempt to find existant record in internal node of level-2 B-tree */ + /* Attempt to find existent record in internal node of level-2 B-tree */ find.key = 513; find.val = (hsize_t)-1; found = FALSE; @@ -4986,7 +4986,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)1) < 0) TEST_ERROR - /* Attempt to find existant record in leaf of level-2 B-tree */ + /* Attempt to find existent record in leaf of level-2 B-tree */ find.key = 555; find.val = (hsize_t)-1; found = FALSE; @@ -5002,7 +5002,7 @@ test_update_make_level2(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_ if (check_node_depth(bt2, &record, (unsigned)0) < 0) TEST_ERROR - /* Attempt to index non-existant record in level-2 B-tree */ + /* Attempt to index non-existent record in level-2 B-tree */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)(INSERT_SPLIT_ROOT_NREC_REC * 42), index_rec_cb, NULL); @@ -5188,7 +5188,7 @@ HDfprintf(stderr, "curr_time = %lu\n", (unsigned long)curr_time); if (iter.key != INSERT_MANY_REC) TEST_ERROR - /* Attempt to find non-existant record in level-4 B-tree */ + /* Attempt to find non-existent record in level-4 B-tree */ /* (Should not be found, but not fail) */ find.key = INSERT_MANY_REC * 2; find.val = (hsize_t)-1; @@ -5206,7 +5206,7 @@ HDfprintf(stderr, "curr_time = %lu\n", (unsigned long)curr_time); find.key = (hsize_t)(HDrandom() % INSERT_MANY_REC); find.val = (hsize_t)-1; - /* Attempt to find existant record in level-4 B-tree */ + /* Attempt to find existent record in level-4 B-tree */ found = FALSE; if (H5B2_find(bt2, &find, &found, find_rec_cb, &find) < 0) FAIL_STACK_ERROR @@ -5216,7 +5216,7 @@ HDfprintf(stderr, "curr_time = %lu\n", (unsigned long)curr_time); TEST_ERROR } /* end for */ - /* Attempt to index non-existant record in level-4 B-tree, in increasing & decreasing order */ + /* Attempt to index non-existent record in level-4 B-tree, in increasing & decreasing order */ H5E_BEGIN_TRY { ret = H5B2_index(bt2, H5_ITER_INC, (hsize_t)(INSERT_MANY_REC * 3), find_rec_cb, NULL); @@ -5245,7 +5245,7 @@ HDfprintf(stderr, "curr_time = %lu\n", (unsigned long)curr_time); find.key = (hsize_t)-1; find.val = (hsize_t)-1; - /* Attempt to find existant record in level-4 B-tree */ + /* Attempt to find existent record in level-4 B-tree */ /* (in increasing order) */ if (H5B2_index(bt2, H5_ITER_INC, idx, index_rec_cb, &find) < 0) FAIL_STACK_ERROR @@ -5258,7 +5258,7 @@ HDfprintf(stderr, "curr_time = %lu\n", (unsigned long)curr_time); find.key = (hsize_t)-1; find.val = (hsize_t)-1; - /* Attempt to find existant record in level-4 B-tree */ + /* Attempt to find existent record in level-4 B-tree */ /* (in decreasing order) */ if (H5B2_index(bt2, H5_ITER_DEC, idx, index_rec_cb, &find) < 0) FAIL_STACK_ERROR @@ -5377,7 +5377,7 @@ test_remove_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ PASSED(); - TESTING("B-tree remove: non-existant record from 1 record B-tree"); + TESTING("B-tree remove: non-existent record from 1 record B-tree"); /* Check for closing & re-opening the B-tree */ if (reopen_btree(f, &bt2, bt2_addr, tparam) < 0) @@ -5404,7 +5404,7 @@ test_remove_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ if (!H5F_addr_defined(root_addr)) TEST_ERROR - /* Attempt to remove a non-existant record from a B-tree with 1 record */ + /* Attempt to remove a non-existent record from a B-tree with 1 record */ record = 0; H5E_BEGIN_TRY { @@ -5418,7 +5418,7 @@ test_remove_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ PASSED(); /* Attempt to remove a record from a B-tree with 1 record */ - TESTING("B-tree remove: existant record from 1 record B-tree"); + TESTING("B-tree remove: existent record from 1 record B-tree"); /* Check for closing & re-opening the B-tree */ if (reopen_btree(f, &bt2, bt2_addr, tparam) < 0) @@ -5482,8 +5482,8 @@ test_remove_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ PASSED(); - /* Attempt to remove a non-existant record from a level-0 B-tree with mult. record */ - TESTING("B-tree remove: non-existant record from level-0 B-tree"); + /* Attempt to remove a non-existent record from a level-0 B-tree with mult. record */ + TESTING("B-tree remove: non-existent record from level-0 B-tree"); /* Check for closing & re-opening the B-tree */ if (reopen_btree(f, &bt2, bt2_addr, tparam) < 0) @@ -5502,7 +5502,7 @@ test_remove_basic(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_ PASSED(); /* Attempt to remove a record from a level-0 B-tree with mult. record */ - TESTING("B-tree remove: mult. existant records from level-0 B-tree"); + TESTING("B-tree remove: mult. existent records from level-0 B-tree"); /* Check for closing & re-opening the B-tree */ if (reopen_btree(f, &bt2, bt2_addr, tparam) < 0) @@ -5660,7 +5660,7 @@ test_remove_level1_noredistrib(hid_t fapl, const H5B2_create_t *cparam, const bt herr_t ret; /* Generic error return value */ /* B-tree record removal tests */ - TESTING("B-tree remove: non-existant record from level-1 B-tree"); + TESTING("B-tree remove: non-existent record from level-1 B-tree"); /* Create the file for the test */ if (create_file(&file, &f, fapl) < 0) @@ -5693,7 +5693,7 @@ test_remove_level1_noredistrib(hid_t fapl, const H5B2_create_t *cparam, const bt if (!H5F_addr_defined(root_addr)) TEST_ERROR - /* Attempt to remove a non-existant record from a B-tree with 1 record */ + /* Attempt to remove a non-existent record from a B-tree with 1 record */ record = (INSERT_SPLIT_ROOT_NREC * 2) + 1; H5E_BEGIN_TRY { @@ -8849,8 +8849,7 @@ HDfprintf(stderr, "curr_time = %lu\n", (unsigned long)curr_time); TEST_ERROR /* Check for VFD which stores data in multiple files */ - single_file_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && - HDstrcmp(env_h5_drvr, "family") != 0); + single_file_vfd = !h5_driver_uses_multiple_files(env_h5_drvr, H5_EXCLUDE_NON_MULTIPART_DRIVERS); if (single_file_vfd) { /* Make a copy of the file in memory, in order to speed up deletion testing */ @@ -9749,7 +9748,7 @@ test_modify(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_t *tpa /* * Test modifying records */ - TESTING("B-tree modify: attempt to modify non-existant record"); + TESTING("B-tree modify: attempt to modify non-existent record"); /* Create the file for the test */ if (create_file(&file, &f, fapl) < 0) @@ -9772,7 +9771,7 @@ test_modify(hid_t fapl, const H5B2_create_t *cparam, const bt2_test_param_t *tpa if (bt2_stat.depth != 2) TEST_ERROR - /* Attempt to modify a non-existant record */ + /* Attempt to modify a non-existent record */ record = 3; modify = 4; H5E_BEGIN_TRY @@ -10127,7 +10126,7 @@ main(void) const char * envval = NULL; hbool_t api_ctx_pushed = FALSE; /* Whether API context pushed */ - envval = HDgetenv("HDF5_DRIVER"); + envval = HDgetenv(HDF5_DRIVER); if (envval == NULL) envval = "nomatch"; @@ -10135,6 +10134,11 @@ main(void) h5_reset(); fapl = h5_fileaccess(); ExpressMode = GetTestExpress(); + + /* For the Direct I/O driver, skip intensive tests due to poor performance */ + if (!HDstrcmp(envval, "direct")) + ExpressMode = 2; + if (ExpressMode > 1) HDprintf("***Express test mode on. Some tests may be skipped\n"); diff --git a/test/cache.c b/test/cache.c index 6989564..e6a2ac8 100644 --- a/test/cache.c +++ b/test/cache.c @@ -2789,8 +2789,8 @@ write_permitted_check(int * Test the behaviour with different flags. * * This test was added primarily to test basic insert - * pinned entry functionallity, but I through in explicit - * tests for other functionallity that is tested implicitly + * pinned entry functionality, but I through in explicit + * tests for other functionality that is tested implicitly * elsewhere. * * Return: void @@ -2908,7 +2908,7 @@ check_insert_entry(unsigned paged) } } - /* Thats all we can get from H5C_get_entry_status(). + /* That's all we can get from H5C_get_entry_status(). * Now start looking at the cache data structures directly. */ @@ -5146,7 +5146,7 @@ check_flush_cache__flush_ops(H5F_t *file_ptr) * * 10/8/07 -- JRM * Added a resize operation to this test to satisfy the new - * requiremnt that any resize of an entry on flush will always + * requirement that any resize of an entry on flush will always * be accompanied by a resize. Note that as a result, this * test becomes redundant with later tests. */ @@ -5238,7 +5238,7 @@ check_flush_cache__flush_ops(H5F_t *file_ptr) * * 10/8/07 -- JRM * Added a resize operation to this test to satisfy the new - * requiremnt that any resize of an entry on flush will always + * requirement that any resize of an entry on flush will always * be accompanied by a resize. Note that as a result, this * test becomes redundant with later tests. */ @@ -9603,7 +9603,7 @@ check_flush_cache__flush_op_eviction_test(H5F_t *file_ptr) * * (VET, 9) Y 10 KB N N - dirty (VET, 8) * - * Recall that in this test bed, flush operations are excuted the + * Recall that in this test bed, flush operations are executed the * first time the associated entry is flushed, and are then * deleted. */ @@ -10156,7 +10156,7 @@ check_flush_cache__flush_op_eviction_test(H5F_t *file_ptr) * returns to the serialize function for (VET, 9). * * When the serialize function for (VET, 9) is done with (VET, 8), it - * calls H5C_unprotect(VET, 8), which markes (VET, 8) as dirty and + * calls H5C_unprotect(VET, 8), which marks (VET, 8) as dirty and * unprotected, and places it at the head of the LRU. * * (VET, 0) is the next item on the LRU -- it is clean and is therefore @@ -10555,7 +10555,7 @@ check_flush_cache__flush_op_eviction_test(H5F_t *file_ptr) * Added tests for pinned entries. * * JRM -- 5/17/06 - * Complete reqrite of pinned entry tests to accomodate + * Complete reqrite of pinned entry tests to accommodate * the new H5C_mark_entry_dirty() call. * *------------------------------------------------------------------------- @@ -12980,7 +12980,7 @@ check_expunge_entry(unsigned paged) * Function: check_multiple_read_protect() * * Purpose: Verify that multiple, simultaneous read protects of a - * single entry perform as expectd. + * single entry perform as expected. * * Return: void * @@ -13326,7 +13326,7 @@ check_multiple_read_protect(unsigned paged) if (pass && (entry_ptr->header.is_pinned)) { pass = FALSE; - failure_mssg = "enty (0,4) still pinned.\n"; + failure_mssg = "entry (0,4) still pinned.\n"; } unprotect_entry(file_ptr, 0, 4, /* (0,4) unprotect */ @@ -15529,7 +15529,7 @@ check_destroy_protected_err(unsigned paged) * Function: check_duplicate_insert_err() * * Purpose: Verify that an attempt to insert and entry that is - * alread in the cache will generate an error. + * already in the cache will generate an error. * * Return: void * @@ -18085,7 +18085,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - /* eigth epoch -- force the hit rate to 100% again -- should be steady + /* eighth epoch -- force the hit rate to 100% again -- should be steady * state. */ if (pass) { @@ -18181,7 +18181,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - if (pass) { /* twelth epoch -- at minimum size so no more ageouts */ + if (pass) { /* twelfth epoch -- at minimum size so no more ageouts */ rpt_fcn_called = FALSE; i = 0; while (pass && (i < 1000)) { @@ -18462,7 +18462,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - /* eigth epoch -- still 100% hit rate + /* eighth epoch -- still 100% hit rate */ if (pass) { @@ -18565,7 +18565,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - /* twelth epoch -- force the hit rate to 100% again -- should be steady + /* twelfth epoch -- force the hit rate to 100% again -- should be steady * state. */ if (pass) { @@ -18941,7 +18941,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - /* eigth epoch -- force the hit rate to 100% again -- should be steady + /* eighth epoch -- force the hit rate to 100% again -- should be steady * state. */ if (pass) { @@ -19041,7 +19041,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - if (pass) { /* twelth epoch -- at minimum size so no more ageouts */ + if (pass) { /* twelfth epoch -- at minimum size so no more ageouts */ rpt_fcn_called = FALSE; i = 0; @@ -19692,7 +19692,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - /* twelth epoch -- hit rate 1.0 -- decrease as before. + /* twelfth epoch -- hit rate 1.0 -- decrease as before. */ if (pass) { @@ -20705,7 +20705,7 @@ check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) if (show_progress) HDfprintf(stderr, "check point %d\n", checkpoint++); - /* protect and unprotect two more large entries -- shouldnt trigger a + /* protect and unprotect two more large entries -- shouldn't trigger a * flash cache size increase. */ if (pass) { @@ -23265,7 +23265,7 @@ check_auto_cache_resize_disable(unsigned paged) * mode (add space), which is triggered whenever the size of a newly * loaded or inserted entry, or the delta between old and new entry * sizes exceeds some fraction of the current maximum cache size, and - * the cache doesn't have enough free space to accomodate the new/ + * the cache doesn't have enough free space to accommodate the new/ * resize entry without performing evictions. The range of permissible * values for the flash_threshold (0.1 to 1.0 as of this writing), and * for the flash_multiple (0.1 to 10.0) do not permit the facility to @@ -23274,7 +23274,7 @@ check_auto_cache_resize_disable(unsigned paged) * current max_cache_size equal to max_size. * * We have already tested the latter in check_auto_cache_resize(), so - * we need only thest the former here. Do this by disabling flash + * we need only test the former here. Do this by disabling flash * cache size increments via the flash_incr_mode, and then creating * situations that would trigger flash cache size increases were that * code enabled. @@ -26347,7 +26347,7 @@ check_auto_cache_resize_input_errs(unsigned paged) /*------------------------------------------------------------------------- * Function: check_auto_cache_resize_aux_fcns() * - * Purpose: Verify that the auxilary functions associated with + * Purpose: Verify that the auxiliary functions associated with * the automatic cache resize capability are operating * correctly. These functions are: * @@ -26424,13 +26424,13 @@ check_auto_cache_resize_aux_fcns(unsigned paged) }; if (paged) - TESTING("automatic cache resize auxilary functions (paged aggregation)") + TESTING("automatic cache resize auxiliary functions (paged aggregation)") else - TESTING("automatic cache resize auxilary functions") + TESTING("automatic cache resize auxiliary functions") pass = TRUE; - /* allocate a cache, and then test the various auxilary functions. + /* allocate a cache, and then test the various auxiliary functions. */ if (pass) { @@ -27077,7 +27077,7 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion, unsigned paged) if (file_ptr == NULL) { pass = FALSE; - failure_mssg = "bad return from cache intialization.\n"; + failure_mssg = "bad return from cache initialization.\n"; } else cache_ptr = file_ptr->shared->cache; @@ -27296,7 +27296,7 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion, unsigned paged) H5C__DIRTIED_FLAG); /* unsigned int flags */ } - /* Expected staus is that after each insertion, the entry + /* Expected status is that after each insertion, the entry * inserted 26 insertions ago has been flushed, and the * entry currently getting inserted is now in the cache and * dirty. @@ -29701,7 +29701,7 @@ check_flush_deps_err(unsigned paged) result = H5C_destroy_flush_dependency(&((entries[entry_type])[0]), &((entries[entry_type])[1])); if (result != FAIL) - CACHE_ERROR("Destroying [non-existant] dependency when parent isn't protected") + CACHE_ERROR("Destroying [non-existent] dependency when parent isn't protected") break; /* Verify that parent entry has flush dependency */ @@ -33057,7 +33057,7 @@ check_metadata_cork(hbool_t fill_via_insertion, unsigned paged) if (file_ptr == NULL) { pass = FALSE; - failure_mssg = "bad return from cache intialization.\n"; + failure_mssg = "bad return from cache initialization.\n"; } else cache_ptr = file_ptr->shared->cache; @@ -33567,7 +33567,7 @@ check_metadata_cork(hbool_t fill_via_insertion, unsigned paged) * an entry that is no longer in the cache, and which may have * been deleted. * - * This function contans tests for correct handling on this + * This function contains tests for correct handling on this * situation. * * Do nothing if pass is FALSE on entry. @@ -33798,7 +33798,7 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t *file_ptr) * * (HET, 3) Y 16 KB Y N - - * - * Recall that in this test bed, flush operations are excuted the + * Recall that in this test bed, flush operations are executed the * first time the associated entry is flushed, and are then * deleted. */ @@ -34048,7 +34048,7 @@ cedds__H5C_make_space_in_cache(H5F_t *file_ptr) * * (HET, 3) Y 16 KB N N - - * - * Recall that in this test bed, flush operations are excuted the + * Recall that in this test bed, flush operations are executed the * first time the associated entry is flushed, and are then * deleted. */ @@ -34463,7 +34463,7 @@ cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t *file_ptr) * * (MET, 1-31) Y 64 KB N N - - * - * Recall that in this test bed, flush operations are excuted the + * Recall that in this test bed, flush operations are executed the * first time the associated entry is flushed, and are then * deleted. */ @@ -35888,6 +35888,11 @@ main(void) HDprintf(" express_test = %d\n", express_test); HDprintf("=========================================\n"); + if (!h5_using_default_driver(NULL)) { + HDputs(" -- SKIPPED for incompatible VFD --"); + HDexit(EXIT_SUCCESS); + } + if (create_entry_arrays() < 0) { HDprintf("ERROR: Unable to create entries arrays. Aborting.\n"); diff --git a/test/cache_api.c b/test/cache_api.c index 412888e..6e897ec 100644 --- a/test/cache_api.c +++ b/test/cache_api.c @@ -306,10 +306,10 @@ check_fapl_mdc_api_calls(unsigned paged, hid_t fcpl_id) pass = FALSE; failure_mssg = "H5Fclose() failed.\n"; } - else if (HDremove(filename) < 0) { + else if (H5Fdelete(filename, H5P_DEFAULT) < 0) { pass = FALSE; - failure_mssg = "HDremove() failed.\n"; + failure_mssg = "H5Fdelete() failed.\n"; } } @@ -442,28 +442,28 @@ check_fapl_mdc_api_calls(unsigned paged, hid_t fcpl_id) } } - /* close the fapl used to create the file */ + /* close the file and delete it */ if (pass) { - if (H5Pclose(fapl_id) < 0) { + if (H5Fclose(file_id) < 0) { pass = FALSE; - failure_mssg = "H5Pclose() failed.\n"; + failure_mssg = "H5Fclose() failed.\n"; + } + else if (H5Fdelete(filename, fapl_id) < 0) { + + pass = FALSE; + failure_mssg = "H5Fdelete() failed.\n"; } } - /* close the file and delete it */ + /* close the fapl used to create the file */ if (pass) { - if (H5Fclose(file_id) < 0) { - - pass = FALSE; - failure_mssg = "H5Fclose() failed.\n"; - } - else if (HDremove(filename) < 0) { + if (H5Pclose(fapl_id) < 0) { pass = FALSE; - failure_mssg = "HDremove() failed.\n"; + failure_mssg = "H5Pclose() failed.\n"; } } @@ -662,7 +662,7 @@ check_file_mdc_api_calls(unsigned paged, hid_t fcpl_id) * configured as per the default both by looking at its internal * configuration, and via the H5Fget_mdc_config() call. * - * Then set serveral different configurations, and verify that + * Then set several different configurations, and verify that * they took as per above. */ @@ -803,10 +803,10 @@ check_file_mdc_api_calls(unsigned paged, hid_t fcpl_id) pass = FALSE; failure_mssg = "H5Fclose() failed.\n"; } - else if (HDremove(filename) < 0) { + else if (H5Fdelete(filename, H5P_DEFAULT) < 0) { pass = FALSE; - failure_mssg = "HDremove() failed.\n"; + failure_mssg = "H5Fdelete() failed.\n"; } } @@ -1054,7 +1054,7 @@ mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id) } /* set the dataset creation plist to specify that the raw data is - * to be partioned into 10X10 element chunks. + * to be partitioned into 10X10 element chunks. */ if (pass) { @@ -1082,7 +1082,7 @@ mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id) /* create the dataset */ if (pass) { - HDsprintf(dset_name, "/dset%03d", i); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", i); dataset_ids[i] = H5Dcreate2(file_id, dset_name, H5T_STD_I32BE, dataspace_id, H5P_DEFAULT, properties, H5P_DEFAULT); @@ -1445,10 +1445,10 @@ mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id) pass = FALSE; failure_mssg = "H5Fclose() failed.\n"; } - else if (HDremove(filename) < 0) { + else if (H5Fdelete(filename, H5P_DEFAULT) < 0) { pass = FALSE; - failure_mssg = "HDremove() failed.\n"; + failure_mssg = "H5Fdelete() failed.\n"; } } @@ -1885,7 +1885,7 @@ check_file_mdc_api_errs(unsigned paged, hid_t fcpl_id) pass = TRUE; - /* Create a file for test purposes, and veify that its metadata cache + /* Create a file for test purposes, and verify that its metadata cache * set to the default MDC configuration. */ @@ -2170,10 +2170,10 @@ check_file_mdc_api_errs(unsigned paged, hid_t fcpl_id) pass = FALSE; failure_mssg = "H5Fclose() failed.\n"; } - else if (HDremove(filename) < 0) { + else if (H5Fdelete(filename, H5P_DEFAULT) < 0) { pass = FALSE; - failure_mssg = "HDremove() failed.\n"; + failure_mssg = "H5Fdelete() failed.\n"; } } @@ -2254,12 +2254,17 @@ main(void) } /* end if */ /* Test with paged aggregation enabled or not */ - /* The "my_fcpl" passed to each test has the paged or non-paged strategy set up accordinly */ + /* The "my_fcpl" passed to each test has the paged or non-paged strategy set up accordingly */ for (paged = FALSE; paged <= TRUE; paged++) { hid_t my_fcpl = fcpl_id; - if (paged) - my_fcpl = fcpl2_id; + if (paged) { + /* Only run paged aggregation tests with sec2/default driver */ + if (!h5_using_default_driver(NULL)) + continue; + else + my_fcpl = fcpl2_id; + } if (!check_fapl_mdc_api_calls(paged, my_fcpl)) nerrs += 1; diff --git a/test/cache_common.c b/test/cache_common.c index 5fcbed5..8614605 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -1529,7 +1529,7 @@ add_flush_op(int target_type, int target_idx, int op_code, int type, int idx, hb * Otherwise, set up a pinned entry dependency so we can * test the pinned entry modifications to the flush routine. * - * Given the types and indicies of the pinned and pinning + * Given the types and indices of the pinned and pinning * entries, add the pinned entry to the list of pinned * entries in the pinning entry, increment the * pinning reference count of the pinned entry, and @@ -1768,7 +1768,7 @@ execute_flush_op(H5F_t *file_ptr, struct test_entry_t *entry_ptr, struct flush_o case FLUSH_OP__EXPUNGE: /* the expunge flush op exists to allow us to simulate the - * case in which an entry is removed from the cashe as the + * case in which an entry is removed from the cache as the * the result of the flush of a second entry. At present, * this can only happen via the take ownership flag, but * we will make this test feature more general to as to make @@ -2313,7 +2313,7 @@ verify_clean(void) * Function: verify_entry_status * * Purpose: Verify that a list of entries have the expected status. - * If any discrepencies are found, set the failure message + * If any discrepancies are found, set the failure message * and set pass to FALSE. * * Do nothing if pass is FALSE on entry. @@ -2329,8 +2329,8 @@ verify_clean(void) void verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_entry_status expected[]) { - static char msg[256]; - int i; + char msg[256]; + int i; i = 0; while ((pass) && (i < num_entries)) { @@ -2342,14 +2342,15 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if ((!expected[i].in_cache) && ((expected[i].is_protected) || (expected[i].is_pinned))) { pass = FALSE; - HDsprintf(msg, "%d: Contradictory data in expected[%d].\n", tag, i); + HDsnprintf(msg, sizeof(msg), "%d: Contradictory data in expected[%d].\n", tag, i); failure_mssg = msg; } if ((!expected[i].in_cache) && (expected[i].is_dirty) && (!entry_ptr->expunged)) { pass = FALSE; - HDsprintf(msg, "%d: expected[%d] specs non-expunged, dirty, non-resident.\n", tag, i); + HDsnprintf(msg, sizeof(msg), "%d: expected[%d] specs non-expunged, dirty, non-resident.\n", tag, + i); failure_mssg = msg; } @@ -2360,9 +2361,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (in_cache != expected[i].in_cache) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) in cache actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, (int)in_cache, - (int)expected[i].in_cache); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) in cache actual/expected = %d/%d.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, (int)in_cache, + (int)expected[i].in_cache); failure_mssg = msg; } } @@ -2372,9 +2373,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->size != expected[i].size) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) size actual/expected = %ld/%ld.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, (long)(entry_ptr->size), - (long)expected[i].size); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) size actual/expected = %ld/%ld.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, (long)(entry_ptr->size), + (long)expected[i].size); failure_mssg = msg; } } @@ -2384,9 +2385,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->header.size != expected[i].size) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) header size actual/expected = %ld/%ld.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (long)(entry_ptr->header.size), (long)expected[i].size); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) header size actual/expected = %ld/%ld.\n", + tag, (int)expected[i].entry_type, (int)expected[i].entry_index, + (long)(entry_ptr->header.size), (long)expected[i].size); failure_mssg = msg; } } @@ -2396,9 +2397,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->at_main_addr != expected[i].at_main_addr) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) at main addr actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->at_main_addr), (int)expected[i].at_main_addr); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) at main addr actual/expected = %d/%d.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->at_main_addr), (int)expected[i].at_main_addr); failure_mssg = msg; } } @@ -2408,9 +2409,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->is_dirty != expected[i].is_dirty) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) is_dirty actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->is_dirty), (int)expected[i].is_dirty); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) is_dirty actual/expected = %d/%d.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->is_dirty), (int)expected[i].is_dirty); failure_mssg = msg; } } @@ -2420,9 +2421,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->header.is_dirty != expected[i].is_dirty) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) header is_dirty actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->header.is_dirty), (int)expected[i].is_dirty); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) header is_dirty actual/expected = %d/%d.\n", + tag, (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->header.is_dirty), (int)expected[i].is_dirty); failure_mssg = msg; } } @@ -2432,9 +2433,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->is_protected != expected[i].is_protected) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) is_protected actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->is_protected), (int)expected[i].is_protected); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) is_protected actual/expected = %d/%d.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->is_protected), (int)expected[i].is_protected); failure_mssg = msg; } } @@ -2444,9 +2445,10 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->header.is_protected != expected[i].is_protected) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) header is_protected actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->header.is_protected), (int)expected[i].is_protected); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d, %d) header is_protected actual/expected = %d/%d.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->header.is_protected), (int)expected[i].is_protected); failure_mssg = msg; } } @@ -2456,9 +2458,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->is_pinned != expected[i].is_pinned) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) is_pinned actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->is_pinned), (int)expected[i].is_pinned); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) is_pinned actual/expected = %d/%d.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->is_pinned), (int)expected[i].is_pinned); failure_mssg = msg; } } @@ -2468,9 +2470,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->is_corked != expected[i].is_corked) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) is_corked actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->is_corked), (int)expected[i].is_corked); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) is_corked actual/expected = %d/%d.\n", tag, + (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->is_corked), (int)expected[i].is_corked); failure_mssg = msg; } } @@ -2480,9 +2482,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (entry_ptr->header.is_pinned != expected[i].is_pinned) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) header is_pinned actual/expected = %d/%d.\n", tag, - (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->header.is_pinned), (int)expected[i].is_pinned); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) header is_pinned actual/expected = %d/%d.\n", + tag, (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->header.is_pinned), (int)expected[i].is_pinned); failure_mssg = msg; } } @@ -2494,11 +2496,12 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ (entry_ptr->destroyed != expected[i].destroyed)) { pass = FALSE; - HDsprintf(msg, "%d entry (%d,%d) deserialized = %d(%d), serialized = %d(%d), dest = %d(%d)\n", - tag, (int)expected[i].entry_type, (int)expected[i].entry_index, - (int)(entry_ptr->deserialized), (int)(expected[i].deserialized), - (int)(entry_ptr->serialized), (int)(expected[i].serialized), - (int)(entry_ptr->destroyed), (int)(expected[i].destroyed)); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d,%d) deserialized = %d(%d), serialized = %d(%d), dest = %d(%d)\n", + tag, (int)expected[i].entry_type, (int)expected[i].entry_index, + (int)(entry_ptr->deserialized), (int)(expected[i].deserialized), + (int)(entry_ptr->serialized), (int)(expected[i].serialized), + (int)(entry_ptr->destroyed), (int)(expected[i].destroyed)); failure_mssg = msg; } } @@ -2509,18 +2512,19 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (pass) { if (entry_ptr->flush_dep_npar != expected[i].flush_dep_npar) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) flush_dep_npar actual/expected = %u/%u.\n", tag, - expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_dep_npar, - expected[i].flush_dep_npar); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) flush_dep_npar actual/expected = %u/%u.\n", + tag, expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_dep_npar, + expected[i].flush_dep_npar); failure_mssg = msg; } /* end if */ } /* end if */ if ((pass) && (in_cache)) { if (entry_ptr->header.flush_dep_nparents != expected[i].flush_dep_npar) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) header flush_dep_nparents actual/expected = %u/%u.\n", tag, - expected[i].entry_type, expected[i].entry_index, - entry_ptr->header.flush_dep_nparents, expected[i].flush_dep_npar); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d, %d) header flush_dep_nparents actual/expected = %u/%u.\n", tag, + expected[i].entry_type, expected[i].entry_index, + entry_ptr->header.flush_dep_nparents, expected[i].flush_dep_npar); failure_mssg = msg; } /* end if */ } /* end if */ @@ -2531,9 +2535,10 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ for (u = 0; u < entry_ptr->flush_dep_npar; u++) { if (entry_ptr->flush_dep_par_type[u] != expected[i].flush_dep_par_type[u]) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) flush_dep_par_type[%u] actual/expected = %d/%d.\n", tag, - expected[i].entry_type, expected[i].entry_index, u, - entry_ptr->flush_dep_par_type[u], expected[i].flush_dep_par_type[u]); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d, %d) flush_dep_par_type[%u] actual/expected = %d/%d.\n", tag, + expected[i].entry_type, expected[i].entry_index, u, + entry_ptr->flush_dep_par_type[u], expected[i].flush_dep_par_type[u]); failure_mssg = msg; } /* end if */ } /* end for */ @@ -2542,9 +2547,10 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ for (u = 0; u < entry_ptr->flush_dep_npar; u++) { if (entry_ptr->flush_dep_par_idx[u] != expected[i].flush_dep_par_idx[u]) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) flush_dep_par_idx[%u] actual/expected = %d/%d.\n", tag, - expected[i].entry_type, expected[i].entry_index, u, - entry_ptr->flush_dep_par_idx[u], expected[i].flush_dep_par_idx[u]); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d, %d) flush_dep_par_idx[%u] actual/expected = %d/%d.\n", tag, + expected[i].entry_type, expected[i].entry_index, u, + entry_ptr->flush_dep_par_idx[u], expected[i].flush_dep_par_idx[u]); failure_mssg = msg; } /* end if */ } /* end for */ @@ -2554,37 +2560,39 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (pass) { if (entry_ptr->flush_dep_nchd != expected[i].flush_dep_nchd) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) flush_dep_nchd actual/expected = %u/%u.\n", tag, - expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_dep_nchd, - expected[i].flush_dep_nchd); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) flush_dep_nchd actual/expected = %u/%u.\n", + tag, expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_dep_nchd, + expected[i].flush_dep_nchd); failure_mssg = msg; } /* end if */ } /* end if */ if ((pass) && (in_cache)) { if (entry_ptr->header.flush_dep_nchildren != expected[i].flush_dep_nchd) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) header flush_dep_nchildren actual/expected = %u/%u.\n", tag, - expected[i].entry_type, expected[i].entry_index, - entry_ptr->header.flush_dep_nchildren, expected[i].flush_dep_nchd); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d, %d) header flush_dep_nchildren actual/expected = %u/%u.\n", tag, + expected[i].entry_type, expected[i].entry_index, + entry_ptr->header.flush_dep_nchildren, expected[i].flush_dep_nchd); failure_mssg = msg; } /* end if */ } /* end if */ if (pass) { if (entry_ptr->flush_dep_ndirty_chd != expected[i].flush_dep_ndirty_chd) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) flush_dep_ndirty_chd actual/expected = %u/%u.\n", tag, - expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_dep_ndirty_chd, - expected[i].flush_dep_ndirty_chd); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d, %d) flush_dep_ndirty_chd actual/expected = %u/%u.\n", tag, + expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_dep_ndirty_chd, + expected[i].flush_dep_ndirty_chd); failure_mssg = msg; } /* end if */ } /* end if */ if ((pass) && (in_cache)) { if (entry_ptr->header.flush_dep_ndirty_children != expected[i].flush_dep_ndirty_chd) { pass = FALSE; - HDsprintf(msg, - "%d entry (%d, %d) header flush_dep_ndirty_children actual/expected = %u/%u.\n", - tag, expected[i].entry_type, expected[i].entry_index, - entry_ptr->header.flush_dep_ndirty_children, expected[i].flush_dep_ndirty_chd); + HDsnprintf(msg, sizeof(msg), + "%d entry (%d, %d) header flush_dep_ndirty_children actual/expected = %u/%u.\n", + tag, expected[i].entry_type, expected[i].entry_index, + entry_ptr->header.flush_dep_ndirty_children, expected[i].flush_dep_ndirty_chd); failure_mssg = msg; } /* end if */ } /* end if */ @@ -2593,9 +2601,9 @@ verify_entry_status(H5C_t *cache_ptr, int tag, int num_entries, struct expected_ if (pass) { if (expected[i].flush_order >= 0 && entry_ptr->flush_order != (unsigned)expected[i].flush_order) { pass = FALSE; - HDsprintf(msg, "%d entry (%d, %d) flush_order actual/expected = %u/%d.\n", tag, - expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_order, - expected[i].flush_order); + HDsnprintf(msg, sizeof(msg), "%d entry (%d, %d) flush_order actual/expected = %u/%d.\n", tag, + expected[i].entry_type, expected[i].entry_index, entry_ptr->flush_order, + expected[i].flush_order); failure_mssg = msg; } /* end if */ } /* end if */ @@ -3825,7 +3833,7 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDassert((!cache_ptr->slist_enabled) || (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ - } /* end elsef */ + } /* end elseif */ if (verbose) HDfprintf(stdout, "\n"); @@ -4504,7 +4512,7 @@ col_major_scan_backward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t int mile_stone = 1; int32_t type; int32_t idx; - int32_t local_max_index[NUMBER_OF_ENTRY_TYPES]; + int32_t local_max_index[NUMBER_OF_ENTRY_TYPES] = {0}; if (verbose) HDfprintf(stdout, "%s: entering.\n", __func__); @@ -5006,7 +5014,7 @@ check_and_validate_cache_hit_rate(hid_t file_id, double *hit_rate_ptr, hbool_t d } else { - expected_hit_rate = 0.0F; + expected_hit_rate = 0.0; } result = H5Fget_mdc_hit_rate(file_id, &hit_rate); @@ -5233,7 +5241,7 @@ resize_configs_are_equal(const H5C_auto_size_ctl_t *a, const H5C_auto_size_ctl_t * has both internal and external configuration matching * *config_ptr. * - * Do nothin on success. On failure, set pass to FALSE, and + * Do nothing on success. On failure, set pass to FALSE, and * load an error message into failue_mssg. Note that * failure_msg is assumed to be at least 128 bytes in length. * diff --git a/test/cache_common.h b/test/cache_common.h index 8f4e776..3a4be95 100644 --- a/test/cache_common.h +++ b/test/cache_common.h @@ -347,7 +347,7 @@ typedef struct test_entry_t { int pin_type[MAX_PINS]; /* array of the types of entries * pinned by this entry. */ - int pin_idx[MAX_PINS]; /* array of the indicies of + int pin_idx[MAX_PINS]; /* array of the indices of * entries pinned by this entry. */ int num_flush_ops; /* integer field containing the @@ -393,7 +393,7 @@ typedef struct test_entry_t { unsigned flush_dep_npar; /* Number of flush dependency parents */ unsigned flush_dep_nchd; /* Number of flush dependency children */ unsigned - flush_dep_ndirty_chd; /* Number of dirty flush dependency children (including granchildren, etc.) */ + flush_dep_ndirty_chd; /* Number of dirty flush dependency children (including grandchildren, etc.) */ hbool_t pinned_from_client; /* entry was pinned by client call */ hbool_t pinned_from_cache; /* entry was pinned by cache internally */ unsigned flush_order; /* Order that entry was flushed in */ diff --git a/test/cache_image.c b/test/cache_image.c index f63f04a..0d0f27c 100644 --- a/test/cache_image.c +++ b/test/cache_image.c @@ -138,7 +138,7 @@ create_datasets(hid_t file_id, int min_dset, int max_dset) } /* set the dataset creation plist to specify that the raw data is - * to be partioned into 10X10 element chunks. + * to be partitioned into 10X10 element chunks. */ if (pass) { @@ -166,7 +166,7 @@ create_datasets(hid_t file_id, int min_dset, int max_dset) /* create the dataset */ if (pass) { - HDsprintf(dset_name, "/dset%03d", i); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", i); dataset_ids[i] = H5Dcreate2(file_id, dset_name, H5T_STD_I32BE, dataspace_id, H5P_DEFAULT, properties, H5P_DEFAULT); @@ -445,7 +445,7 @@ delete_datasets(hid_t file_id, int min_dset, int max_dset) i = min_dset; while ((pass) && (i <= max_dset)) { - HDsprintf(dset_name, "/dset%03d", i); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", i); if (H5Ldelete(file_id, dset_name, H5P_DEFAULT) < 0) { @@ -472,7 +472,7 @@ delete_datasets(hid_t file_id, int min_dset, int max_dset) * Set pass to FALSE and issue a suitable failure * message if either the file contains a metadata cache image * superblock extension and mdci_sbem_expected is TRUE, or - * vise versa. + * vice versa. * * If mdci_sbem_expected is TRUE, also verify that the metadata * cache has been advised of this. @@ -546,7 +546,7 @@ open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, hbool_t read_onl if (show_progress) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); - /* create a file access propertly list. */ + /* create a file access property list. */ if (pass) { fapl_id = h5_fileaccess(); @@ -881,7 +881,7 @@ attempt_swmr_open_hdf5_file(const hbool_t create_file, const hbool_t set_mdci_fa H5AC_cache_image_config_t cache_image_config = {H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, TRUE, FALSE, H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; - /* create a file access propertly list. */ + /* create a file access property list. */ if (pass) { fapl_id = h5_fileaccess(); @@ -1021,7 +1021,7 @@ verify_datasets(hid_t file_id, int min_dset, int max_dset) /* open the dataset */ if (pass) { - HDsprintf(dset_name, "/dset%03d", i); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", i); dataset_ids[i] = H5Dopen2(file_id, dset_name, H5P_DEFAULT); if (dataset_ids[i] < 0) { @@ -1507,7 +1507,7 @@ check_cache_image_ctl_flow_1(hbool_t single_file_vfd) * processed as part of the first protect operation after the * superblock is loaded.) * - * In this particular test, we preform the following operations: + * In this particular test, we perform the following operations: * * 1) Create a HDF5 file with the cache image FAPL entry. * @@ -4626,7 +4626,7 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) /* 2) Create a process specific group. */ if (pass) { - HDsprintf(process_group_name, "/process_%d", min_group); + HDsnprintf(process_group_name, sizeof(process_group_name), "/process_%d", min_group); proc_gid = H5Gcreate2(file_id, process_group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); @@ -4737,7 +4737,7 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) if (pass) { max_group++; - HDsprintf(process_group_name, "/process_%d", max_group); + HDsnprintf(process_group_name, sizeof(process_group_name), "/process_%d", max_group); proc_gid = H5Gcreate2(file_id, process_group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); @@ -4815,7 +4815,7 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) /* 11) Validate all the zoos. */ i = min_group; while (pass && i <= max_group) { - HDsprintf(process_group_name, "/process_%d", i); + HDsnprintf(process_group_name, sizeof(process_group_name), "/process_%d", i); pass = validate_zoo( file_id, process_group_name, &lastmsgtime, (zoo_config_t){.proc_num = i++, .skip_varlen = false, .skip_compact = false, .msgival = {0, 0}}); @@ -4869,7 +4869,7 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) i = min_group; while ((pass) && (i <= max_group)) { - HDsprintf(process_group_name, "/process_%d", i); + HDsnprintf(process_group_name, sizeof(process_group_name), "/process_%d", i); pass = validate_zoo( file_id, process_group_name, &lastmsgtime, (zoo_config_t){.proc_num = i++, .skip_varlen = false, .skip_compact = false, .msgival = {0, 0}}); @@ -4931,7 +4931,7 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) */ i = min_group; while ((pass) && (i <= max_group)) { - HDsprintf(process_group_name, "/process_%d", i); + HDsnprintf(process_group_name, sizeof(process_group_name), "/process_%d", i); pass = validate_zoo( file_id, process_group_name, &lastmsgtime, (zoo_config_t){.proc_num = i++, .skip_varlen = false, .skip_compact = false, .msgival = {0, 0}}); @@ -7020,7 +7020,7 @@ cache_image_api_error_check_4(hbool_t single_file_vfd) * image. * * The objective of this test is to create a test file - * with both non-empty self referential presistant + * with both non-empty self referential persistent * free space managers, and a cache image, and then * verify that this situation is handled correctly if * H5Fget_free_sections() is called before the metadata @@ -7858,7 +7858,7 @@ main(void) int express_test; /* Get the VFD to use */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; @@ -7872,8 +7872,7 @@ main(void) HDprintf("=========================================\n"); /* Check for VFD which stores data in multiple files */ - single_file_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && - HDstrcmp(env_h5_drvr, "family") != 0); + single_file_vfd = !h5_driver_uses_multiple_files(env_h5_drvr, H5_EXCLUDE_NON_MULTIPART_DRIVERS); nerrs += check_cache_image_ctl_flow_1(single_file_vfd); nerrs += check_cache_image_ctl_flow_2(single_file_vfd); diff --git a/test/cache_tagging.c b/test/cache_tagging.c index 06de174..a335193 100644 --- a/test/cache_tagging.c +++ b/test/cache_tagging.c @@ -443,7 +443,7 @@ check_file_creation_tags(hid_t fcpl_id, int type) /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose test outout */ + int verbose = FALSE; /* verbose test output */ #endif /* NDEBUG */ hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; @@ -547,7 +547,7 @@ check_file_open_tags(hid_t fcpl, int type) /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t fapl = -1; /* File access prop list */ haddr_t root_tag; /* Root Group Tag */ @@ -677,7 +677,7 @@ check_group_creation_tags(void) hid_t fid = -1; /* File Identifier */ hid_t gid = -1; /* Group Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = HADDR_UNDEF; /* Root Group Tag */ @@ -800,7 +800,7 @@ check_multi_group_creation_tags(void) hid_t fid = -1; /* File Identifier */ hid_t gid = -1; /* Group Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ char gname[16]; /* group name buffer */ int i = 0; /* iterator */ @@ -952,7 +952,7 @@ check_link_iteration_tags(void) hid_t sid = -1; /* Group Identifier */ hid_t did = -1; /* Group Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ int i = 0; /* iterator */ haddr_t root_tag = 0; /* Root Group Tag Value */ @@ -1094,7 +1094,7 @@ check_dense_attribute_tags(void) hid_t did = -1; /* Group Identifier */ hid_t dcpl = -1; /* Group Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ int i = 0; /* iterator */ hid_t fapl = -1; /* File access property list */ @@ -1455,7 +1455,7 @@ check_attribute_creation_tags(hid_t fcpl, int type) hid_t gid = -1; /* Group Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; /* Root group tag */ @@ -1620,7 +1620,7 @@ check_attribute_open_tags(hid_t fcpl, int type) hid_t gid = -1; /* Group Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; @@ -1787,7 +1787,7 @@ check_attribute_rename_tags(hid_t fcpl, int type) hid_t aid = -1; /* Attribute Identifier */ hid_t sid = -1; /* Dataset Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ int * data = NULL; /* data buffer */ int i, j, k = 0; /* iterators */ @@ -2000,7 +2000,7 @@ check_attribute_delete_tags(hid_t fcpl, int type) hid_t aid = -1; /* Attribute Identifier */ hid_t sid = -1; /* Dataset Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ int * data = NULL; /* data buffer */ int i, j, k = 0; /* iterators */ @@ -2191,7 +2191,7 @@ check_dataset_creation_tags(hid_t fcpl, int type) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -2351,7 +2351,7 @@ check_dataset_creation_earlyalloc_tags(hid_t fcpl, int type) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -2517,7 +2517,7 @@ check_dataset_open_tags(void) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -2669,7 +2669,7 @@ check_dataset_write_tags(void) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -2836,7 +2836,7 @@ check_attribute_write_tags(hid_t fcpl, int type) hid_t aid = -1; /* Attribute Identifier */ hid_t sid = -1; /* Dataset Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ int * data = NULL; /* data buffer */ int i, j, k = 0; /* iterators */ @@ -3022,7 +3022,7 @@ check_dataset_read_tags(void) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -3184,7 +3184,7 @@ check_dataset_size_retrieval(void) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -3348,7 +3348,7 @@ check_dataset_extend_tags(void) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -3795,7 +3795,7 @@ check_link_removal_tags(hid_t fcpl, int type) hid_t sid = -1; /* Dataspace Identifier */ hid_t gid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -3984,7 +3984,7 @@ check_link_getname_tags(void) hid_t sid = -1; /* Dataspace Identifier */ hid_t gid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1, 1}; /* chunk dimensions */ @@ -4161,7 +4161,7 @@ check_external_link_creation_tags(void) hid_t fid2 = -1; /* File Identifier */ hid_t gid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ hid_t fapl = -1; /* File access prop list */ haddr_t root_tag = 0; @@ -4288,7 +4288,7 @@ check_external_link_open_tags(void) hid_t gid = -1; /* Dataspace Identifier */ hid_t xid = -1; /* Dataspace Identifier */ #ifndef NDEBUG - int verbose = FALSE; /* verbose file outout */ + int verbose = FALSE; /* verbose file output */ #endif /* NDEBUG */ H5O_native_info_t ninfo; /* Native object info struct */ hid_t fapl = -1; /* File access prop list */ @@ -4572,6 +4572,12 @@ main(void) /* Open the HDF5 Library */ H5open(); + /* Only run with sec2/default driver */ + if (!h5_using_default_driver(NULL)) { + HDputs(" -- SKIPPED for incompatible VFD --"); + HDexit(EXIT_SUCCESS); + } + /* ========== */ /* Test Setup */ /* ========== */ @@ -4579,7 +4585,7 @@ main(void) /* Create a standard file creation property list */ fcpl_default = H5Pcreate(H5P_FILE_CREATE); - /* Create an fcpl with shared messages and file space managment enabled */ + /* Create an fcpl with shared messages and file space management enabled */ fcpl_shmesg_all = H5Pcreate(H5P_FILE_CREATE); H5Pset_shared_mesg_nindexes(fcpl_shmesg_all, 1); H5Pset_shared_mesg_index(fcpl_shmesg_all, 0, H5O_SHMESG_ALL_FLAG, 20); diff --git a/test/chunk_info.c b/test/chunk_info.c index a1c4160..2bec42d 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -291,7 +291,7 @@ index_type_str(H5D_chunk_index_t idx_type) /*------------------------------------------------------------------------- * Function: verify_selected_chunks (helper function) * - * Purpose: Reads the chunks within the boundery {start,end} and verify + * Purpose: Reads the chunks within the boundary {start,end} and verify * the values against the populated data. * * Return: Success: SUCCEED @@ -321,7 +321,7 @@ verify_selected_chunks(hid_t dset, hid_t plist, const hsize_t *start, const hsiz for (jj = 0; jj < CHUNK_NY; jj++) expected_buf[n][ii][jj] = (int)(ii * jj) + 1; - /* Read each chunk within the boundery of {start,end} and verify the + /* Read each chunk within the boundary of {start,end} and verify the values against the expected data */ chk_index = 0; for (ii = start[0]; ii < end[0]; ii++) @@ -421,7 +421,7 @@ verify_idx_nchunks(hid_t dset, hid_t dspace, H5D_chunk_index_t exp_idx_type, hsi /* Ensure the correct chunk indexing scheme is used */ if (idx_type != exp_idx_type) { char msg[256]; - HDsprintf(msg, "Should be using %s.\n", index_type_str(idx_type)); + HDsnprintf(msg, sizeof(msg), "Should be using %s.\n", index_type_str(idx_type)); FAIL_PUTS_ERROR(msg); } diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index 8777096..024b6c0 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -713,7 +713,7 @@ test_compound(char *filename, hid_t fapl) /* *###################################################################### * Step 12: Write an array into the middle third of the dataset - * initializeing only members `b' and `d' to -1. + * initializing only members `b' and `d' to -1. */ TESTING("hyperslab part initialized write"); @@ -1766,7 +1766,7 @@ test_pack_ooo(void) /* Insert the compound members in the random order previously generated */ for (i = 0; i < PACK_NMEMBS; i++) { - HDsprintf(name, "%05d", i); + HDsnprintf(name, sizeof(name), "%05d", i); if (i == sub_cmpd_order) { if (H5Tinsert(cmpd, name, (size_t)(4 * order[i]), sub_cmpd) < 0) PACK_OOO_ERROR @@ -1799,7 +1799,7 @@ test_pack_ooo(void) /* Insert the compound members in the random order previously generated */ for (i = 0; i < PACK_NMEMBS; i++) { - HDsprintf(name, "%05d", i); + HDsnprintf(name, sizeof(name), "%05d", i); if (i == sub_cmpd_order) { if (H5Tinsert(cmpd, name, (size_t)(4 * order[i]), sub_cmpd) < 0) PACK_OOO_ERROR @@ -1834,7 +1834,7 @@ test_pack_ooo(void) /* Insert the compound members in reverse order, with compound last */ for (i = 0; i < PACK_NMEMBS; i++) { - HDsprintf(name, "%05d", i); + HDsnprintf(name, sizeof(name), "%05d", i); if (i == PACK_NMEMBS - 1) { if (H5Tinsert(cmpd, name, (size_t)(4 * (PACK_NMEMBS - i - 1)), sub_cmpd) < 0) PACK_OOO_ERROR @@ -1867,7 +1867,7 @@ test_pack_ooo(void) /* Insert the compound members in reverse order, with compound last */ for (i = 0; i < PACK_NMEMBS; i++) { - HDsprintf(name, "%05d", i); + HDsnprintf(name, sizeof(name), "%05d", i); if (i == PACK_NMEMBS - 1) { if (H5Tinsert(cmpd, name, (size_t)(4 * (PACK_NMEMBS - i - 1)), sub_cmpd) < 0) PACK_OOO_ERROR @@ -1902,7 +1902,7 @@ test_pack_ooo(void) /* Insert the compound members in forward order, with compound first */ for (i = 0; i < PACK_NMEMBS; i++) { - HDsprintf(name, "%05d", i); + HDsnprintf(name, sizeof(name), "%05d", i); if (i == 0) { if (H5Tinsert(cmpd, name, (size_t)(4 * i), sub_cmpd) < 0) PACK_OOO_ERROR @@ -1935,7 +1935,7 @@ test_pack_ooo(void) /* Insert the compound members in forward order */ for (i = 0; i < PACK_NMEMBS; i++) { - HDsprintf(name, "%05d", i); + HDsnprintf(name, sizeof(name), "%05d", i); if (i == 0) { if (H5Tinsert(cmpd, name, (size_t)(4 * i), sub_cmpd) < 0) PACK_OOO_ERROR diff --git a/test/cork.c b/test/cork.c index 28c0604..b1f28c1 100644 --- a/test/cork.c +++ b/test/cork.c @@ -26,7 +26,11 @@ */ #define H5C_FRIEND /*suppress error about including H5Cpkg */ #define H5C_TESTING /*suppress warning about H5C testing funcs*/ +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ +#define H5FD_TESTING + #include "H5Cpkg.h" /* Cache */ +#include "H5FDpkg.h" /* ============ */ /* Test Defines */ @@ -97,7 +101,7 @@ verify_old_dset_cork(void) hsize_t dims[2] = {100, 20}; /* Dataset dimension sizes */ hsize_t max_dims[2] = {100, H5S_UNLIMITED}; /* Dataset maximum dimension sizes */ hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ - int ** buf = NULL; /* Data bufer (pointers to fake 2D array) */ + int ** buf = NULL; /* Data buffer (pointers to fake 2D array) */ int * buf_data = NULL; /* Data buffer (actual data) */ int i = 0, j = 0; /* Local index variables */ H5O_info2_t oinfo, oinfo2, oinfo3; /* Object metadata information */ @@ -509,7 +513,7 @@ verify_dset_cork(hbool_t swmr, hbool_t new_format) hsize_t dims[2] = {100, 20}; /* Dataset dimension sizes */ hsize_t max_dims[2] = {100, H5S_UNLIMITED}; /* Dataset maximum dimension sizes */ hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ - int ** buf = NULL; /* Data bufer (pointers to fake 2D array) */ + int ** buf = NULL; /* Data buffer (pointers to fake 2D array) */ int * buf_data = NULL; /* Data buffer (actual data) */ int i = 0, j = 0; /* Local index variables */ H5O_info2_t oinfo, oinfo2, oinfo3; /* Object metadata information */ @@ -1883,7 +1887,7 @@ test_dset_cork(hbool_t swmr, hbool_t new_format) { hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t fapl; /* File access property list */ - hid_t gid = H5I_INVALID_HID; /* Groupd ID */ + hid_t gid = H5I_INVALID_HID; /* Grouped ID */ hid_t did1 = H5I_INVALID_HID, did2 = H5I_INVALID_HID; /* Dataset IDs */ hid_t tid1 = H5I_INVALID_HID, tid2 = H5I_INVALID_HID; /* Datatype IDs */ hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ @@ -2244,6 +2248,19 @@ main(void) nerrs += verify_old_dset_cork(); for (swmr = 0; swmr <= 1; swmr++) { + if (swmr) { + char *driver = NULL; + + /* Skip these tests if SWMR I/O is not supported for the VFD specified + * by the environment variable. + */ + driver = HDgetenv(HDF5_DRIVER); + if (!H5FD__supports_swmr_test(driver)) { + HDputs("-- SKIPPED SWMR tests for SWMR-incompatible VFD --"); + continue; + } + } + /* Tests with new/old library format */ /* This is the test moved from th5o.c: test_h5o_cork() */ nerrs += test_objs_cork(swmr, TRUE); diff --git a/test/dangle.c b/test/dangle.c index d41507b..95d3105 100644 --- a/test/dangle.c +++ b/test/dangle.c @@ -660,7 +660,21 @@ error: int main(void) { - int nerrors = 0; + const char *env_h5_drvr; /* File Driver value from environment */ + int nerrors = 0; + + /* Get the VFD to use */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* Don't run this test with the multi/split VFD. A bug in library shutdown + * ordering causes problems with the multi VFD when IDs are left dangling. + */ + if (!HDstrcmp(env_h5_drvr, "multi") || !HDstrcmp(env_h5_drvr, "split")) { + HDputs(" -- SKIPPED for incompatible VFD --"); + return 0; + } /* Run tests w/weak file close */ HDputs("Testing dangling objects with weak file close:"); diff --git a/test/del_many_dense_attrs.c b/test/del_many_dense_attrs.c index cf7f607..78aba3b 100644 --- a/test/del_many_dense_attrs.c +++ b/test/del_many_dense_attrs.c @@ -119,7 +119,7 @@ main(void) /* Create attributes in the group */ for (i = ATTR_COUNT; i >= 0; i--) { /* Set up the attribute name */ - HDsprintf(aname, "%s%d", basename, i); + HDsnprintf(aname, sizeof(aname), "%s%d", basename, i); /* Create the attribute */ if ((aid = H5Acreate2(gid, aname, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -165,7 +165,7 @@ main(void) /* Delete the attributes */ for (i = 0; i <= ATTR_COUNT; i++) { /* Set up the attribute name */ - HDsprintf(aname, "%s%d", basename, i); + HDsnprintf(aname, sizeof(aname), "%s%d", basename, i); /* Delete the attribute */ if (H5Adelete(gid, aname) < 0) diff --git a/test/direct_chunk.c b/test/direct_chunk.c index 8960784..ac4ef8d 100644 --- a/test/direct_chunk.c +++ b/test/direct_chunk.c @@ -686,7 +686,7 @@ filter_bogus1(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, size_t buf_left = *buf_size; /* Amount of data buffer left to process */ if (flags & H5Z_FLAG_REVERSE) { /* read */ - /* Substract the "add on" value to all the data values */ + /* Subtract the "add on" value to all the data values */ while (buf_left > 0) { *int_ptr++ -= (int)ADD_ON; buf_left -= sizeof(int); @@ -722,7 +722,7 @@ filter_bogus2(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, size_t buf_left = *buf_size; /* Amount of data buffer left to process */ if (flags & H5Z_FLAG_REVERSE) { /* read */ - /* Substract the "add on" value to all the data values */ + /* Subtract the "add on" value to all the data values */ while (buf_left > 0) { *int_ptr++ /= (int)FACTOR; buf_left -= sizeof(int); @@ -1427,7 +1427,7 @@ error: * Function: test_direct_chunk_read_no_cache * * Purpose: Test the basic functionality of H5Dread_chunk with the - * chunk cache diabled. + * chunk cache disabled. * * Return: Success: 0 * Failure: 1 @@ -1963,7 +1963,7 @@ error: * Function: test_read_unallocated_chunk * * Purpose: Tests the H5Dread_chunk and H5Dget_chunk_storage_size with valid - * offets to chunks that have not been written to the dataset and are + * offsets to chunks that have not been written to the dataset and are * not allocated in the chunk storage on disk. * * Return: Success: 0 @@ -2014,7 +2014,7 @@ test_read_unallocated_chunk(hid_t file) if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) FAIL_STACK_ERROR; - /* Write a single chunk to intialize the chunk storage */ + /* Write a single chunk to initialize the chunk storage */ HDmemset(direct_buf, 0, CHUNK_NX * CHUNK_NY * sizeof(int)); offset[0] = 0; offset[1] = 0; @@ -2031,7 +2031,7 @@ test_read_unallocated_chunk(hid_t file) offset[0] = i * CHUNK_NX; offset[1] = j * CHUNK_NY; - /* Read a non-existant chunk using the direct read function. */ + /* Read a non-existent chunk using the direct read function. */ H5E_BEGIN_TRY { status = H5Dread_chunk(dataset, dxpl, offset, &filter_mask, &direct_buf); @@ -2042,7 +2042,7 @@ test_read_unallocated_chunk(hid_t file) if (status != -1) TEST_ERROR - /* Query the size of the non-existant chunk */ + /* Query the size of the non-existent chunk */ direct_chunk_nbytes = ULONG_MAX; H5E_BEGIN_TRY { diff --git a/test/dsets.c b/test/dsets.c index e2d281a..88e3ce0 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -835,7 +835,11 @@ error: static herr_t test_compact_io(hid_t fapl) { - hid_t file, dataset, space, plist; + hid_t file = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t plist = H5I_INVALID_HID; + hid_t verfile = -1, new_fapl = -1; hsize_t dims[2]; int wbuf[16][8], rbuf[16][8]; @@ -2007,7 +2011,7 @@ test_filter_internal(hid_t fid, const char *name, hid_t dcpl, int if_fletcher32, if (status >= 0) TEST_ERROR; - /* Callback decides to continue inspite data is corrupted. */ + /* Callback decides to continue in spite data is corrupted. */ if (H5Pset_filter_callback(dxpl, filter_cb_cont, NULL) < 0) TEST_ERROR; if (H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, check_data) < 0) @@ -2074,7 +2078,7 @@ test_filter_internal(hid_t fid, const char *name, hid_t dcpl, int if_fletcher32, if (status >= 0) TEST_ERROR; - /* Callback decides to continue inspite data is corrupted. */ + /* Callback decides to continue in spite data is corrupted. */ if (H5Pset_filter_callback(dxpl, filter_cb_cont, NULL) < 0) TEST_ERROR; if (H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, check_data) < 0) @@ -2138,7 +2142,7 @@ test_filter_internal(hid_t fid, const char *name, hid_t dcpl, int if_fletcher32, if (status >= 0) TEST_ERROR; - /* Callback decides to continue inspite data is corrupted. */ + /* Callback decides to continue in spite data is corrupted. */ if (H5Pset_filter_callback(dxpl, filter_cb_cont, NULL) < 0) TEST_ERROR; if (H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, check_data) < 0) @@ -2205,7 +2209,7 @@ test_filter_internal(hid_t fid, const char *name, hid_t dcpl, int if_fletcher32, if (status >= 0) TEST_ERROR; - /* Callback decides to continue inspite data is corrupted. */ + /* Callback decides to continue in spite data is corrupted. */ if (H5Pset_filter_callback(dxpl, filter_cb_cont, NULL) < 0) TEST_ERROR; if (H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl, check_data) < 0) @@ -3240,7 +3244,7 @@ test_nbit_int(hid_t file) power = HDpow(2.0, (double)(precision - 1)); orig_data[i][j] = (int)(((long long)HDrandom() % (long long)power) << offset); - /* even-numbered values are negtive */ + /* even-numbered values are negative */ if ((i * size[1] + j + 1) % 2 == 0) orig_data[i][j] = -orig_data[i][j]; } @@ -3818,7 +3822,7 @@ test_nbit_compound(hid_t file) orig_data[i][j].s = (short)(((long long)HDrandom() % (long long)power) << offset[2]); orig_data[i][j].f = float_val[i][j]; - /* some even-numbered integer values are negtive */ + /* some even-numbered integer values are negative */ if ((i * size[1] + j + 1) % 2 == 0) { orig_data[i][j].i = -orig_data[i][j].i; orig_data[i][j].s = (short)-orig_data[i][j].s; @@ -4532,7 +4536,7 @@ test_nbit_int_size(hid_t file) goto error; } - /* Initiliaze data buffer with random data within correct range + /* Initialize data buffer with random data within correct range * corresponding to the memory datatype's precision and offset. */ for (i = 0; i < DSET_DIM1; i++) @@ -4745,7 +4749,7 @@ test_nbit_flt_size(hid_t file) } /* end if */ /* - * Initiliaze data buffer with random data + * Initialize data buffer with random data */ for (i = 0; i < DSET_DIM1; i++) for (j = 0; j < DSET_DIM2; j++) @@ -4904,7 +4908,7 @@ test_scaleoffset_int(hid_t file) for (j = 0; j < (size_t)size[1]; j++) { orig_data[i][j] = HDrandom() % 10000; - /* even-numbered values are negtive */ + /* even-numbered values are negative */ if ((i * size[1] + j + 1) % 2 == 0) orig_data[i][j] = -orig_data[i][j]; } @@ -5048,7 +5052,7 @@ test_scaleoffset_int_2(hid_t file) for (j = 0; j < (size_t)size[1]; j++) { orig_data[0][j] = (int)HDrandom() % 10000; - /* even-numbered values are negtive */ + /* even-numbered values are negative */ if ((j + 1) % 2 == 0) orig_data[0][j] = -orig_data[0][j]; } @@ -5170,7 +5174,7 @@ test_scaleoffset_float(hid_t file) for (j = 0; j < (size_t)size[1]; j++) { orig_data[i][j] = (float)(HDrandom() % 100000) / 1000.0F; - /* even-numbered values are negtive */ + /* even-numbered values are negative */ if ((i * size[1] + j + 1) % 2 == 0) orig_data[i][j] = -orig_data[i][j]; } @@ -5316,7 +5320,7 @@ test_scaleoffset_float_2(hid_t file) for (j = 0; j < (size_t)size[1]; j++) { orig_data[0][j] = (float)(HDrandom() % 100000) / 1000.0F; - /* even-numbered values are negtive */ + /* even-numbered values are negative */ if ((j + 1) % 2 == 0) orig_data[0][j] = -orig_data[0][j]; } @@ -5437,7 +5441,7 @@ test_scaleoffset_double(hid_t file) for (j = 0; j < (size_t)size[1]; j++) { orig_data[i][j] = (HDrandom() % 10000000) / 10000000.0; - /* even-numbered values are negtive */ + /* even-numbered values are negative */ if ((i * size[1] + j + 1) % 2 == 0) orig_data[i][j] = -orig_data[i][j]; } @@ -5583,7 +5587,7 @@ test_scaleoffset_double_2(hid_t file) for (j = 0; j < (size_t)size[1]; j++) { orig_data[0][j] = (HDrandom() % 10000000) / 10000000.0; - /* even-numbered values are negtive */ + /* even-numbered values are negative */ if ((j + 1) % 2 == 0) orig_data[0][j] = -orig_data[0][j]; } @@ -7587,9 +7591,9 @@ test_missing_chunk(hid_t file) hsize_t csize2[2] = {5, 5}; size_t u, i, j; /* Local Index variable */ - hid_t fapl; /* File access property list */ - H5F_libver_t low; /* File format low bound */ - H5D_chunk_index_t idx_type, idx_type2; /* Dataset chunk index types */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + H5F_libver_t low; /* File format low bound */ + H5D_chunk_index_t idx_type, idx_type2; /* Dataset chunk index types */ TESTING("Read dataset with unwritten chunk & undefined fill value"); @@ -8895,7 +8899,7 @@ test_chunk_cache(hid_t fapl) if (H5Pclose(dapl2) < 0) FAIL_STACK_ERROR - /* Similary, test use of H5Dcreate2 with H5P_DEFAULT */ + /* Similarly, test use of H5Dcreate2 with H5P_DEFAULT */ if (H5Dclose(dsid) < 0) FAIL_STACK_ERROR if ((dsid = H5Dcreate2(fid, "dset2", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) @@ -10722,8 +10726,8 @@ test_fixed_array(hid_t fapl) const hsize_t dim2_big[2] = {500, 60}; /* Big dataset dimensions */ hsize_t dim2_max[2] = {120, 50}; /* Maximum dataset dimensions */ - hid_t mem_id; /* Memory space ID */ - hid_t big_mem_id; /* Memory space ID for big dataset */ + hid_t mem_id = H5I_INVALID_HID; /* Memory space ID */ + hid_t big_mem_id; /* Memory space ID for big dataset */ hsize_t msize[1] = {POINTS}; /* Size of memory space */ hsize_t msize_big[1] = {POINTS_BIG}; /* Size of memory space for big dataset */ @@ -11980,7 +11984,7 @@ error: * Purpose: Checks that a file created with either: * (a) SWMR-write + non-latest-format * (b) write + latest format - * will generate datset with latest chunk indexing type. + * will generate dataset with latest chunk indexing type. * * Return: Success: 0 * Failure: -1 @@ -14714,7 +14718,7 @@ test_versionbounds(void) hid_t dcpl = -1; /* Dataset creation property list */ hid_t srcspace = -1; /* Source dataspaces */ hid_t vspace = -1; /* Virtual dset dataspaces */ - hid_t srcdset = -1; /* Source datset */ + hid_t srcdset = -1; /* Source dataset */ hid_t vdset = -1; /* Virtual dataset */ hsize_t dims[1] = {3}; /* Data space current size */ char srcfilename[FILENAME_BUF_SIZE]; @@ -14850,7 +14854,7 @@ error: /*----------------------------------------------------------------------------- * Function: test_object_header_minimization_dcpl * - * Purpose: Test the "datset object header minimization" property as part of + * Purpose: Test the "dataset object header minimization" property as part of * the DCPL. * * Return: Success/pass: 0 @@ -15557,15 +15561,16 @@ main(void) double rdcc_w0; int nerrors = 0; const char *envval; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ + hbool_t driver_uses_modified_filename = h5_driver_uses_modified_filename(); int i; /* Don't run this test using certain file drivers */ - envval = HDgetenv("HDF5_DRIVER"); + envval = HDgetenv(HDF5_DRIVER); if (envval == NULL) envval = "nomatch"; - /* Current VFD that does not support contigous address space */ + /* Current VFD that does not support contiguous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") != 0 && HDstrcmp(envval, "multi") != 0); /* Set the random # seed */ @@ -15718,7 +15723,11 @@ main(void) nerrors += (test_multiopen(file) < 0 ? 1 : 0); nerrors += (test_types(file) < 0 ? 1 : 0); nerrors += (test_userblock_offset(envval, my_fapl, new_format) < 0 ? 1 : 0); - nerrors += (test_missing_filter(file) < 0 ? 1 : 0); + + if (!driver_uses_modified_filename) { + nerrors += (test_missing_filter(file) < 0 ? 1 : 0); + } + nerrors += (test_can_apply(file) < 0 ? 1 : 0); nerrors += (test_can_apply2(file) < 0 ? 1 : 0); nerrors += (test_optional_filters(file) < 0 ? 1 : 0); @@ -15727,7 +15736,11 @@ main(void) nerrors += (test_compare_dcpl(file) < 0 ? 1 : 0); nerrors += (test_copy_dcpl(file, my_fapl) < 0 ? 1 : 0); nerrors += (test_filter_delete(file) < 0 ? 1 : 0); - nerrors += (test_filters_endianess() < 0 ? 1 : 0); + + if (!driver_uses_modified_filename) { + nerrors += (test_filters_endianess() < 0 ? 1 : 0); + } + nerrors += (test_zero_dims(file) < 0 ? 1 : 0); nerrors += (test_missing_chunk(file) < 0 ? 1 : 0); nerrors += (test_random_chunks(my_fapl) < 0 ? 1 : 0); @@ -15745,7 +15758,11 @@ main(void) nerrors += (test_chunk_expand(my_fapl) < 0 ? 1 : 0); nerrors += (test_layout_extend(my_fapl) < 0 ? 1 : 0); nerrors += (test_fixed_array(my_fapl) < 0 ? 1 : 0); - nerrors += (test_idx_compatible() < 0 ? 1 : 0); + + if (!driver_uses_modified_filename) { + nerrors += (test_idx_compatible() < 0 ? 1 : 0); + } + nerrors += (test_unfiltered_edge_chunks(my_fapl) < 0 ? 1 : 0); nerrors += (test_single_chunk(my_fapl) < 0 ? 1 : 0); nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); @@ -15779,7 +15796,9 @@ main(void) nerrors += (test_gather_error() < 0 ? 1 : 0); /* Tests version bounds using its own file */ - nerrors += (test_versionbounds() < 0 ? 1 : 0); + if (!driver_uses_modified_filename) { + nerrors += (test_versionbounds() < 0 ? 1 : 0); + } /* Tests that use their own file */ nerrors += (test_object_header_minimization_dcpl() < 0 ? 1 : 0); diff --git a/test/dt_arith.c b/test/dt_arith.c index 7e96d29..993048a 100644 --- a/test/dt_arith.c +++ b/test/dt_arith.c @@ -27,7 +27,7 @@ #define FP_EPSILON 0.000001F /* - * Offset from alinged memory returned by malloc(). This can be used to test + * Offset from aligned memory returned by malloc(). This can be used to test * that type conversions handle non-aligned buffers correctly. */ #define ALIGNMENT 1 @@ -79,7 +79,7 @@ typedef enum dtype_t { * be allowed to continue (cf. Posix signals) so in order to recover from a * SIGFPE we run tests that might generate one in a child process. */ -#ifdef H5_HAVE_UNISTD_H +#if defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) #define HANDLE_SIGFPE #endif @@ -223,7 +223,7 @@ static int without_hardware_g = 0; /*The number of values if multiplied by 10 for each step.*/ \ num_norm = (SRC_MAX_10_EXP - SRC_MIN_10_EXP); \ /*Reduce the number of values by 2^factor. MULTIPLY=10^(2^factor). Using this algorithm \ - *instead of arithmatic operation to avoid any conversion*/ \ + *instead of arithmetic operation to avoid any conversion*/ \ num_norm >>= factor; \ \ /*Total number of values*/ \ @@ -640,7 +640,7 @@ test_particular_fp_integer(void) float src_f = (float)INT_MAX; int dst_i; int fill_value = 13; - int endian; /*endianess */ + int endian; /*endianness */ unsigned int fails_this_test = 0; size_t j; @@ -818,7 +818,7 @@ test_derived_flt(void) size_t src_size, dst_size; unsigned char *buf = NULL, *saved_buf = NULL; int * aligned = NULL; - int endian; /*endianess */ + int endian; /*endianness */ size_t nelmts = NTESTELEM; unsigned int fails_this_test = 0; const size_t max_fails = 40; /*max number of failures*/ @@ -1267,7 +1267,7 @@ test_derived_integer(void) char filename[1024]; size_t src_size, dst_size; unsigned char *buf = NULL, *saved_buf = NULL; - int endian; /*endianess */ + int endian; /*endianness */ size_t nelmts = NTESTELEM; unsigned int fails_this_test = 0; const size_t max_fails = 40; /*max number of failures*/ @@ -1591,7 +1591,7 @@ test_conv_int_1(const char *name, hid_t src, hid_t dst) dtype_t src_type, dst_type; /*data types */ const char * src_type_name = NULL; /*source type name */ const char * dst_type_name = NULL; /*destination type name */ - int endian; /*machine endianess */ + int endian; /*machine endianness */ size_t src_size, dst_size; /*type sizes */ unsigned char * buf = NULL; /*buffer for conversion */ unsigned char * saved = NULL; /*original values */ @@ -2831,8 +2831,8 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) int overflow = 0; /*overflow occurred */ int uflow = 0; /*underflow debug counters*/ size_t j, k; /*counters */ - int sendian; /* source type endianess */ - int dendian; /* Destination type endianess */ + int sendian; /* source type endianness */ + int dendian; /* Destination type endianness */ size_t dst_ebias; /* Destination type's exponent bias */ size_t src_epos; /* Source type's exponent position */ size_t src_esize; /* Source type's exponent size */ @@ -3147,7 +3147,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) /* * Assume same if both results are NaN. There are many NaN bit - * patterns and the software doesn't attemt to emulate the + * patterns and the software doesn't attempt to emulate the * hardware in this regard. Instead, software uses a single bit * pattern for NaN by setting the significand to all ones. */ @@ -3426,8 +3426,8 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) dtype_t dst_type; /*data types */ const char * src_type_name = NULL; /*source type name */ const char * dst_type_name = NULL; /*destination type name */ - int sendian; /*source endianess */ - int dendian; /*destination endianess */ + int sendian; /*source endianness */ + int dendian; /*destination endianness */ size_t src_size, dst_size; /*type sizes */ unsigned char * buf = NULL; /*buffer for conversion */ unsigned char * saved = NULL; /*original values */ @@ -4485,7 +4485,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) break; } - HDprintf(" ans = "); + HDprintf(" and = "); for (k = 0; k < dst_size; k++) HDprintf(" %02x", hw[ENDIAN(dst_size, k, dendian)]); HDprintf("%*s", (int)(3 * MAX(0, (ssize_t)src_size - (ssize_t)dst_size)), ""); diff --git a/test/dtransform.c b/test/dtransform.c index 50c33ee..1f6d48b 100644 --- a/test/dtransform.c +++ b/test/dtransform.c @@ -122,9 +122,8 @@ const int transformData[ROWS][COLS] = {{36, 31, 25, 19, 13, 7, 1, 5, 11, 16, 22, { \ TYPE array[ROWS][COLS]; \ const char *f_to_c = "(5/9.0)*(x-32)"; \ - /* utrans is a transform for unsigned types: no negative numbers involved and results are < 255 to \ - * fit into uchar */ \ - const char *utrans = "((x+100)/4)*3"; \ + /* utrans is a transform for char types: numbers are restricted from -128 to 127, fits into char */ \ + const char *utrans = "(x/4+25)*3"; \ \ hid_t dataspace, dxpl_id_f_to_c, dxpl_id_utrans, dset, dset_nn, dt_nn; \ H5T_order_t order; \ @@ -199,6 +198,8 @@ const int transformData[ROWS][COLS] = {{36, 31, 25, 19, 13, 7, 1, 5, 11, 16, 22, COMPARE(TYPE, array, COMPARE_DATA, 2) \ } \ \ + if (H5Dclose(dset_nn) < 0) \ + TEST_ERROR; \ if (H5Dclose(dset) < 0) \ TEST_ERROR; \ if (H5Sclose(dataspace) < 0) \ @@ -209,9 +210,8 @@ const int transformData[ROWS][COLS] = {{36, 31, 25, 19, 13, 7, 1, 5, 11, 16, 22, { \ TYPE array[ROWS][COLS]; \ const char *f_to_c = "(5/9.0)*(x-32)"; \ - /* utrans is a transform for unsigned types: no negative numbers involved and results are < 255 to \ - * fit into uchar */ \ - const char *utrans = "((x+100)/4)*3"; \ + /* utrans is a transform for char types: numbers are restricted from -128 to 127, fits into char */ \ + const char *utrans = "(x/4+25)*3"; \ \ hid_t dataspace, dxpl_id_f_to_c, dxpl_id_utrans, cparms, memspace, dset_chunk, filespace; \ hsize_t dim[2] = {ROWS, COLS}; \ @@ -312,7 +312,7 @@ main(void) const char *simple = "(4/2) * ( (2 + 4)/(5 - 2.5))"; /* this equals 4.8 */ const char *polynomial = "(2+x)* ((x-8)/2)"; /* inverses the utrans transform in init_test to get back original array */ - const char *utrans_inv = "(x/3)*4 - 100"; + const char *utrans_inv = "(x/3 - 25)*4"; if ((file_id = H5Fcreate("dtransform.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; @@ -434,9 +434,8 @@ static int init_test(hid_t file_id) { const char *f_to_c = "(5/9.0)*(x-32)"; - /* utrans is a transform for unsigned types: no negative numbers involved and results are < 255 to fit - * into uchar */ - const char *utrans = "((x+100)/4)*3"; + /* utrans is a transform for char types: numbers are restricted from -128 to 127, fits into char */ + const char *utrans = "(x/4+25)*3"; hid_t dataspace = -1; hid_t dxpl_id_f_to_c = -1; @@ -463,7 +462,7 @@ init_test(hid_t file_id) if ((dataspace = H5Screate_simple(2, dim, NULL)) < 0) TEST_ERROR - TESTING("Intializing test...") + TESTING("Initializing test...") if ((dset_id_int = H5Dcreate2(file_id, "/default_int", H5T_NATIVE_INT, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) diff --git a/test/dtypes.c b/test/dtypes.c index f9b4e8c..d2064b7 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -29,7 +29,7 @@ #define ARRAY_DIM 4 /* - * Offset from alinged memory returned by malloc(). This can be used to test + * Offset from aligned memory returned by malloc(). This can be used to test * that type conversions handle non-aligned buffers correctly. */ #define ALIGNMENT 1 @@ -2359,7 +2359,7 @@ test_compound_11(void) TEST_ERROR } /* end if */ if (((big_t *)buf_orig)[u].s1 == NULL || ((little_t *)buf)[u].s1 == NULL) { - HDprintf("Error, line #%d: buf_orig[%u].s1=%p, buf[%u].s1=%p\n", __LINE__, (unsigned)u, + HDprintf("Error, line #%d: buf_orig[%u].s1=%s, buf[%u].s1=%s\n", __LINE__, (unsigned)u, ((big_t *)buf_orig)[u].s1, (unsigned)u, ((little_t *)buf)[u].s1); TEST_ERROR } /* end if */ @@ -2410,7 +2410,7 @@ test_compound_11(void) TEST_ERROR } /* end if */ if (((big_t *)buf_orig)[u].s1 == NULL || ((little_t *)buf)[u].s1 == NULL) { - HDprintf("Error, line #%d: buf_orig[%u].s1=%p, buf[%u].s1=%p\n", __LINE__, (unsigned)u, + HDprintf("Error, line #%d: buf_orig[%u].s1=%s, buf[%u].s1=%s\n", __LINE__, (unsigned)u, ((big_t *)buf_orig)[u].s1, (unsigned)u, ((little_t *)buf)[u].s1); TEST_ERROR } /* end if */ @@ -2451,7 +2451,7 @@ test_compound_11(void) TEST_ERROR } /* end if */ if (((big_t *)buf_orig)[u].s1 == NULL || ((little_t *)buf)[u].s1 == NULL) { - HDprintf("Error, line #%d: buf_orig[%u].s1=%p, buf[%u].s1=%p\n", __LINE__, (unsigned)u, + HDprintf("Error, line #%d: buf_orig[%u].s1=%s, buf[%u].s1=%s\n", __LINE__, (unsigned)u, ((big_t *)buf_orig)[u].s1, (unsigned)u, ((little_t *)buf)[u].s1); TEST_ERROR } /* end if */ @@ -2635,9 +2635,9 @@ test_compound_13(void) HDmemset(&data_out, 0, sizeof(data_out)); for (u = 0; u < COMPOUND13_ARRAY_SIZE + 1; u++) data_out.x[u] = (unsigned char)u; - data_out.y = 99.99f; + data_out.y = 99.99F; - /* Set latest_format in access propertly list to enable the latest + /* Set latest_format in access property list to enable the latest * compound datatype format. */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -3723,53 +3723,55 @@ test_compound_18(void) if (H5Fclose(file) < 0) FAIL_STACK_ERROR - /* Open Generated File */ - /* (generated with gen_bad_compound.c) */ - if ((file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + if (!h5_driver_uses_modified_filename()) { + /* Open Generated File */ + /* (generated with gen_bad_compound.c) */ + if ((file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR - /* Try to open the datatype */ - H5E_BEGIN_TRY - { - tid = H5Topen2(file, "cmpnd", H5P_DEFAULT); - } - H5E_END_TRY; - if (tid > 0) { - H5Tclose(tid); - FAIL_PUTS_ERROR("opened named datatype with bad compound datatype") - } /* end if */ + /* Try to open the datatype */ + H5E_BEGIN_TRY + { + tid = H5Topen2(file, "cmpnd", H5P_DEFAULT); + } + H5E_END_TRY; + if (tid > 0) { + H5Tclose(tid); + FAIL_PUTS_ERROR("opened named datatype with bad compound datatype") + } /* end if */ - /* Try to open the dataset */ - H5E_BEGIN_TRY - { - did = H5Dopen2(file, "dataset", H5P_DEFAULT); - } - H5E_END_TRY; - if (did > 0) { - H5Dclose(did); - FAIL_PUTS_ERROR("opened dataset with bad compound datatype") - } /* end if */ + /* Try to open the dataset */ + H5E_BEGIN_TRY + { + did = H5Dopen2(file, "dataset", H5P_DEFAULT); + } + H5E_END_TRY; + if (did > 0) { + H5Dclose(did); + FAIL_PUTS_ERROR("opened dataset with bad compound datatype") + } /* end if */ - /* Open the group with the attribute */ - if ((gid = H5Gopen2(file, "group", H5P_DEFAULT)) < 0) - TEST_ERROR + /* Open the group with the attribute */ + if ((gid = H5Gopen2(file, "group", H5P_DEFAULT)) < 0) + TEST_ERROR - /* Try to open the dataset */ - H5E_BEGIN_TRY - { - aid = H5Aopen(gid, "attr", H5P_DEFAULT); - } - H5E_END_TRY; - if (aid > 0) { - H5Aclose(aid); - FAIL_PUTS_ERROR("opened attribute with bad compound datatype") - } /* end if */ + /* Try to open the dataset */ + H5E_BEGIN_TRY + { + aid = H5Aopen(gid, "attr", H5P_DEFAULT); + } + H5E_END_TRY; + if (aid > 0) { + H5Aclose(aid); + FAIL_PUTS_ERROR("opened attribute with bad compound datatype") + } /* end if */ - /* Close IDs */ - if (H5Gclose(gid) < 0) - FAIL_STACK_ERROR - if (H5Fclose(file) < 0) - FAIL_STACK_ERROR + /* Close IDs */ + if (H5Gclose(gid) < 0) + FAIL_STACK_ERROR + if (H5Fclose(file) < 0) + FAIL_STACK_ERROR + } PASSED(); return 0; @@ -3792,7 +3794,7 @@ error: * * Modifications: * Raymond Lu - * Wednesday, Febuary 9, 2005 + * Wednesday, February 9, 2005 * Added test for H5Tenum_valueof, H5Tenum_nameof, and * H5Tget_member_value. *------------------------------------------------------------------------- @@ -4967,7 +4969,7 @@ test_conv_str_2(void) } /* end for */ /* Do the conversions */ - HDsprintf(s, "Testing random string conversion speed"); + HDsnprintf(s, sizeof(s), "Testing random string conversion speed"); HDprintf("%-70s", s); HDfflush(stdout); if (H5Tconvert(c_type, f_type, nelmts, buf, NULL, H5P_DEFAULT) < 0) @@ -5182,14 +5184,14 @@ test_conv_enum_1(void) buf[u] = HDrand() % 26; /* Conversions */ - HDsprintf(s, "Testing random enum conversion O(N)"); + HDsnprintf(s, sizeof(s), "Testing random enum conversion O(N)"); HDprintf("%-70s", s); HDfflush(stdout); if (H5Tconvert(t1, t2, nelmts, buf, NULL, H5P_DEFAULT) < 0) goto error; PASSED(); - HDsprintf(s, "Testing random enum conversion O(N log N)"); + HDsnprintf(s, sizeof(s), "Testing random enum conversion O(N log N)"); HDprintf("%-70s", s); HDfflush(stdout); if (H5Tconvert(t2, t1, nelmts, buf, NULL, H5P_DEFAULT) < 0) @@ -6845,7 +6847,7 @@ test_set_order(void) { hid_t dtype; /* Datatype ID */ H5T_order_t order; /* Byte order */ - hsize_t dims[2] = {3, 4}; /* Array dimenstions */ + hsize_t dims[2] = {3, 4}; /* Array dimensions */ herr_t ret; /* Generic return value */ TESTING("H5Tset/get_order"); @@ -7093,7 +7095,7 @@ test_set_order_compound(hid_t fapl) hid_t file = -1; hid_t cmpd = -1, memb_cmpd = -1, memb_array1 = -1, memb_array2 = -1, cmpd_array = -1; hid_t vl_id = -1; - hsize_t dims[2] = {3, 4}; /* Array dimenstions */ + hsize_t dims[2] = {3, 4}; /* Array dimensions */ char filename[1024]; herr_t ret; /* Generic return value */ @@ -7948,7 +7950,7 @@ test_deprec(hid_t fapl) unsigned u; /* Local index variable */ herr_t status; /* Generic routine value */ - TESTING("deprected API routines for datatypes"); + TESTING("deprecated API routines for datatypes"); /* Create an array datatype with an atomic base type */ /* (dimension permutations allowed, but not stored) */ diff --git a/test/earray.c b/test/earray.c index 337e818..7c8dd1f 100644 --- a/test/earray.c +++ b/test/earray.c @@ -1284,7 +1284,7 @@ typedef struct eiter_fw_t { /*------------------------------------------------------------------------- * Function: eiter_fw_init * - * Purpose: Initialize element interator (forward iteration) + * Purpose: Initialize element iterator (forward iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -1428,7 +1428,7 @@ eiter_fw_state(void *in_eiter, const H5EA_create_t *cparam, const earray_test_pa /*------------------------------------------------------------------------- * Function: eiter_fw_term * - * Purpose: Shut down element interator (forward iteration) + * Purpose: Shut down element iterator (forward iteration) * * Return: Success: 0 * Failure: -1 @@ -1472,7 +1472,7 @@ typedef struct eiter_rv_t { /*------------------------------------------------------------------------- * Function: eiter_rv_init * - * Purpose: Initialize element interator (reverse iteration) + * Purpose: Initialize element iterator (reverse iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -1653,7 +1653,7 @@ eiter_rv_state(void *in_eiter, const H5EA_create_t *cparam, const earray_test_pa /*------------------------------------------------------------------------- * Function: eiter_rv_term * - * Purpose: Shut down element interator (reverse iteration) + * Purpose: Shut down element iterator (reverse iteration) * * Return: Success: 0 * Failure: -1 @@ -1694,7 +1694,7 @@ typedef struct eiter_rnd_t { /*------------------------------------------------------------------------- * Function: eiter_rnd_init * - * Purpose: Initialize element interator (random iteration) + * Purpose: Initialize element iterator (random iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -1803,7 +1803,7 @@ eiter_rnd_max(const void *in_eiter) /*------------------------------------------------------------------------- * Function: eiter_rnd_term * - * Purpose: Shut down element interator (random iteration) + * Purpose: Shut down element iterator (random iteration) * * Return: Success: 0 * Failure: -1 @@ -1843,7 +1843,7 @@ static const earray_iter_t ea_iter_rnd = { /*------------------------------------------------------------------------- * Function: eiter_rnd2_init * - * Purpose: Initialize element interator (random #2 iteration) + * Purpose: Initialize element iterator (random #2 iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -1926,7 +1926,7 @@ typedef struct eiter_cyc_t { /*------------------------------------------------------------------------- * Function: eiter_cyc_init * - * Purpose: Initialize element interator (cyclic iteration) + * Purpose: Initialize element iterator (cyclic iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -2019,7 +2019,7 @@ eiter_cyc_max(const void *in_eiter) /*------------------------------------------------------------------------- * Function: eiter_cyc_term * - * Purpose: Shut down element interator (cyclic iteration) + * Purpose: Shut down element iterator (cyclic iteration) * * Return: Success: 0 * Failure: -1 @@ -2543,15 +2543,16 @@ main(void) /* Test first element in data block */ nelmts = (hsize_t)((hsize_t)1 + cparam.idx_blk_elmts + tparam.sblk_info[sblk].start_idx + (tparam.sblk_info[sblk].dblk_nelmts * dblk)); - HDsprintf(test_str, "setting first element of array's data block #%llu", - (unsigned long long)ndblks); + HDsnprintf(test_str, sizeof(test_str), + "setting first element of array's data block #%llu", + (unsigned long long)ndblks); nerrors += test_set_elmts(fapl, &cparam, &tparam, nelmts, test_str); /* Test all elements in data block */ nelmts = (hsize_t)(cparam.idx_blk_elmts + tparam.sblk_info[sblk].start_idx + (tparam.sblk_info[sblk].dblk_nelmts * (dblk + 1))); - HDsprintf(test_str, "setting all elements of array's data block #%llu", - (unsigned long long)ndblks); + HDsnprintf(test_str, sizeof(test_str), "setting all elements of array's data block #%llu", + (unsigned long long)ndblks); nerrors += test_set_elmts(fapl, &cparam, &tparam, nelmts, test_str); /* Increment data block being tested */ diff --git a/test/enc_dec_plist.c b/test/enc_dec_plist.c index 60b229a..0a73273 100644 --- a/test/enc_dec_plist.c +++ b/test/enc_dec_plist.c @@ -199,8 +199,8 @@ main(void) /* Display testing info */ low_string = h5_get_version_string(low); high_string = h5_get_version_string(high); - HDsprintf(msg, "Testing ENCODE/DECODE with file version bounds: (%s, %s):", low_string, - high_string); + HDsnprintf(msg, sizeof(msg), + "Testing ENCODE/DECODE with file version bounds: (%s, %s):", low_string, high_string); HDputs(msg); if (VERBOSE_MED) diff --git a/test/enc_dec_plist_cross_platform.c b/test/enc_dec_plist_cross_platform.c index 9ea5cf9..3030d7c 100644 --- a/test/enc_dec_plist_cross_platform.c +++ b/test/enc_dec_plist_cross_platform.c @@ -28,7 +28,7 @@ int main(void) { if (VERBOSE_MED) - HDprintf("Encode/Decode property list endianess\n"); + HDprintf("Encode/Decode property list endianness\n"); /******* ENCODE/DECODE DCPLS *****/ TESTING("Default DCPL Encoding/Decoding"); diff --git a/test/enum.c b/test/enum.c index 3923892..512fd9d 100644 --- a/test/enum.c +++ b/test/enum.c @@ -552,7 +552,7 @@ test_value_dsnt_exist(void) if ((datatype_id = H5Tenum_create(H5T_NATIVE_INT)) < 0) goto error; - /* These calls should fail, since no memebrs exist yet */ + /* These calls should fail, since no members exist yet */ if (H5Tenum_valueof(datatype_id, "SAX", &val) >= 0) goto error; val = 3; diff --git a/test/error_test.c b/test/error_test.c index f4dc340..9ac76dd 100644 --- a/test/error_test.c +++ b/test/error_test.c @@ -738,7 +738,7 @@ error: *------------------------------------------------------------------------- */ static herr_t -test_filter_error(const char *fname) +test_filter_error(const char *fname, hid_t fapl) { const char *pathname = H5_get_srcdir_filename(fname); /* Corrected test file name */ hid_t file = -1; @@ -748,7 +748,7 @@ test_filter_error(const char *fname) HDfprintf(stderr, "\nTesting error message during data reading when filter isn't registered\n"); /* Open the file */ - if ((file = H5Fopen(pathname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + if ((file = H5Fopen(pathname, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR; /* Open the regular dataset */ @@ -785,9 +785,15 @@ main(void) hid_t fapl = -1; hid_t estack_id = -1; char filename[1024]; + const char *env_h5_drvr; /* File driver value from environment */ const char *FUNC_main = "main"; int i; + /* Get the VFD to use */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + HDfprintf(stderr, " This program tests the Error API. There're supposed to be some error messages\n"); /* Initialize errors */ @@ -877,7 +883,15 @@ main(void) * the test file was pre-generated. */ h5_fixname(DATAFILE, H5P_DEFAULT, filename, sizeof filename); - if (test_filter_error(filename) < 0) + if (!h5_using_default_driver(env_h5_drvr) && HDstrcmp(env_h5_drvr, "stdio")) { + /* If not using the library's default VFD or the stdio VFD, force + * the library's default VFD here. The test file was pre-generated + * and can cause issues with many VFDs. + */ + if (H5Pset_driver(fapl, H5_DEFAULT_VFD, NULL) < 0) + TEST_ERROR; + } + if (test_filter_error(filename, fapl) < 0) TEST_ERROR; h5_clean_files(FILENAME, fapl); diff --git a/test/event_set.c b/test/event_set.c index 5df49e9..c2a17b1 100644 --- a/test/event_set.c +++ b/test/event_set.c @@ -19,8 +19,157 @@ #include "h5test.h" #include "H5srcdir.h" +#define EVENT_SET_NUM_CONNECTOR_IDS 2 + const char *FILENAME[] = {"event_set_1", NULL}; +hid_t connector_ids_g[EVENT_SET_NUM_CONNECTOR_IDS]; + +herr_t fake_wait_request_wait(void *req, uint64_t timeout, H5VL_request_status_t *status); +herr_t fake_wait_request_free(void *req); + +/* A VOL class struct that describes a VOL class with no + * functionality, other than a wait that returns success. + */ +static const H5VL_class_t fake_wait_vol_g = { + H5VL_VERSION, /* VOL class struct version */ + ((H5VL_class_value_t)501), /* value */ + "fake_wait", /* name */ + 0, /* connector version */ + 0, /* capability flags */ + NULL, /* initialize */ + NULL, /* terminate */ + { + /* info_cls */ + (size_t)0, /* size */ + NULL, /* copy */ + NULL, /* compare */ + NULL, /* free */ + NULL, /* to_str */ + NULL, /* from_str */ + }, + { + /* wrap_cls */ + NULL, /* get_object */ + NULL, /* get_wrap_ctx */ + NULL, /* wrap_object */ + NULL, /* unwrap_object */ + NULL, /* free_wrap_ctx */ + }, + { + /* attribute_cls */ + NULL, /* create */ + NULL, /* open */ + NULL, /* read */ + NULL, /* write */ + NULL, /* get */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* close */ + }, + { + /* dataset_cls */ + NULL, /* create */ + NULL, /* open */ + NULL, /* read */ + NULL, /* write */ + NULL, /* get */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* close */ + }, + { + /* datatype_cls */ + NULL, /* commit */ + NULL, /* open */ + NULL, /* get */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* close */ + }, + { + /* file_cls */ + NULL, /* create */ + NULL, /* open */ + NULL, /* get */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* close */ + }, + { + /* group_cls */ + NULL, /* create */ + NULL, /* open */ + NULL, /* get */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* close */ + }, + { + /* link_cls */ + NULL, /* create */ + NULL, /* copy */ + NULL, /* move */ + NULL, /* get */ + NULL, /* specific */ + NULL /* optional */ + }, + { + /* object_cls */ + NULL, /* open */ + NULL, /* copy */ + NULL, /* get */ + NULL, /* specific */ + NULL /* optional */ + }, + { + /* introspect_cls */ + NULL, /* get_conn_cls */ + NULL, /* get_cap_flags */ + NULL, /* opt_query */ + }, + { + /* request_cls */ + fake_wait_request_wait, /* wait */ + NULL, /* notify */ + NULL, /* cancel */ + NULL, /* specific */ + NULL, /* optional */ + fake_wait_request_free /* free */ + }, + { + /* blob_cls */ + NULL, /* put */ + NULL, /* get */ + NULL, /* specific */ + NULL /* optional */ + }, + { + /* token_cls */ + NULL, /* cmp */ + NULL, /* to_str */ + NULL /* from_str */ + }, + NULL /* optional */ +}; + +herr_t +fake_wait_request_wait(void H5_ATTR_UNUSED *req, uint64_t H5_ATTR_UNUSED timeout, + H5VL_request_status_t *status) +{ + /* Set status if requested */ + if (status) + *status = H5VL_REQUEST_STATUS_SUCCEED; + + return 0; +} /* end H5_daos_req_wait() */ + +herr_t +fake_wait_request_free(void H5_ATTR_UNUSED *req) +{ + return 0; +} /* end fake_wait_request_free() */ + /*------------------------------------------------------------------------- * Function: test_es_create * @@ -159,6 +308,348 @@ error: } /*------------------------------------------------------------------------- + * Function: test_es_get_requests + * + * Purpose: Tests getting requests from event set. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Neil Fortner + * Wednesday, November 24, 2021 + * + *------------------------------------------------------------------------- + */ +static int +test_es_get_requests(void) +{ + hid_t es_id; /* Event set ID */ + hid_t connector_ids[2]; /* Connector IDs */ + void * requests[2]; /* Requests */ + int req_targets[2]; /* Dummy targets for void * requests */ + size_t count; /* # of events in set */ + hbool_t op_failed; /* Whether an operation failed (unused) */ + + TESTING("event set get requests"); + + /* Create an event set */ + if ((es_id = H5EScreate()) < 0) + TEST_ERROR + + /* Get number of requests in event set */ + count = 3; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, NULL, NULL, 0, &count) < 0) + TEST_ERROR + if (count != 0) + TEST_ERROR + + /* Get only connector IDs */ + count = 3; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, connector_ids, NULL, 2, &count) < 0) + TEST_ERROR + if (count != 0) + TEST_ERROR + if (connector_ids[0] != H5I_INVALID_HID) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + + /* Get only requests */ + count = 3; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, NULL, requests, 2, &count) < 0) + TEST_ERROR + if (count != 0) + TEST_ERROR + if (requests[0] != NULL) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Get both */ + count = 3; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, connector_ids, requests, 2, &count) < 0) + TEST_ERROR + if (count != 0) + TEST_ERROR + if (connector_ids[0] != H5I_INVALID_HID) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + if (requests[0] != NULL) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Insert event into event set */ + if (H5ESinsert_request(es_id, connector_ids_g[0], &req_targets[0]) < 0) + TEST_ERROR + + /* Get number of requests in event set */ + count = 0; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, NULL, NULL, 0, &count) < 0) + TEST_ERROR + if (count != 1) + TEST_ERROR + + /* Get only connector IDs */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, connector_ids, NULL, 2, &count) < 0) + TEST_ERROR + if (count != 1) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[0]) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + + /* Get only requests */ + count = 0; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, NULL, requests, 2, &count) < 0) + TEST_ERROR + if (count != 1) + TEST_ERROR + if (requests[0] != &req_targets[0]) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Get both */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, connector_ids, requests, 2, &count) < 0) + TEST_ERROR + if (count != 1) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[0]) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + if (requests[0] != &req_targets[0]) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Insert second event into event set */ + if (H5ESinsert_request(es_id, connector_ids_g[1], &req_targets[1]) < 0) + TEST_ERROR + + /* Get number of requests in event set */ + count = 0; + if (H5ESget_requests(es_id, H5_ITER_NATIVE, NULL, NULL, 0, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + + /* Get only connector IDs */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + if (H5ESget_requests(es_id, H5_ITER_INC, connector_ids, NULL, 2, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[0]) + TEST_ERROR + if (connector_ids[1] != connector_ids_g[1]) + TEST_ERROR + + /* Try with H5_ITER_DEC */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + if (H5ESget_requests(es_id, H5_ITER_DEC, connector_ids, NULL, 2, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[1]) + TEST_ERROR + if (connector_ids[1] != connector_ids_g[0]) + TEST_ERROR + + /* Get only requests */ + count = 0; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_INC, NULL, requests, 2, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (requests[0] != &req_targets[0]) + TEST_ERROR + if (requests[1] != &req_targets[1]) + TEST_ERROR + + /* Try with H5_ITER_DEC */ + count = 0; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_DEC, NULL, requests, 2, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (requests[0] != &req_targets[1]) + TEST_ERROR + if (requests[1] != &req_targets[0]) + TEST_ERROR + + /* Get both */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_INC, connector_ids, requests, 2, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[0]) + TEST_ERROR + if (connector_ids[1] != connector_ids_g[1]) + TEST_ERROR + if (requests[0] != &req_targets[0]) + TEST_ERROR + if (requests[1] != &req_targets[1]) + TEST_ERROR + + /* Try with H5_ITER_DEC */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_DEC, connector_ids, requests, 2, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[1]) + TEST_ERROR + if (connector_ids[1] != connector_ids_g[0]) + TEST_ERROR + if (requests[0] != &req_targets[1]) + TEST_ERROR + if (requests[1] != &req_targets[0]) + TEST_ERROR + + /* Get only first connector ID */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + if (H5ESget_requests(es_id, H5_ITER_INC, connector_ids, NULL, 1, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[0]) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + + /* Try with H5_ITER_DEC */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + if (H5ESget_requests(es_id, H5_ITER_DEC, connector_ids, NULL, 1, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[1]) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + + /* Get only first request */ + count = 0; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_INC, NULL, requests, 1, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (requests[0] != &req_targets[0]) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Try with H5_ITER_DEC */ + count = 0; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_DEC, NULL, requests, 1, &count) < 0) + TEST_ERROR + if (count != 2) + TEST_ERROR + if (requests[0] != &req_targets[1]) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Get only first of both */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_INC, connector_ids, requests, 1, &count) < 0) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[0]) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + if (requests[0] != &req_targets[0]) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Try with H5_ITER_DEC */ + count = 0; + connector_ids[0] = H5I_INVALID_HID; + connector_ids[1] = H5I_INVALID_HID; + requests[0] = NULL; + requests[1] = NULL; + if (H5ESget_requests(es_id, H5_ITER_DEC, connector_ids, requests, 1, &count) < 0) + TEST_ERROR + if (connector_ids[0] != connector_ids_g[1]) + TEST_ERROR + if (connector_ids[1] != H5I_INVALID_HID) + TEST_ERROR + if (requests[0] != &req_targets[1]) + TEST_ERROR + if (requests[1] != NULL) + TEST_ERROR + + /* Close the event set */ + if (H5ESwait(es_id, 10000000, &count, &op_failed) < 0) + TEST_ERROR + if (H5ESclose(es_id) < 0) + TEST_ERROR + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5ESclose(es_id); + } + H5E_END_TRY; + return 1; +} + +/*------------------------------------------------------------------------- * Function: main * * Purpose: Tests event sets @@ -175,27 +666,41 @@ int main(void) { hid_t fapl_id = H5I_INVALID_HID; /* File access property list */ + int i; /* Local index variable */ int nerrors = 0; /* Error count */ /* Setup */ h5_reset(); fapl_id = h5_fileaccess(); + /* Register dummy connector IDs */ + for (i = 0; i < EVENT_SET_NUM_CONNECTOR_IDS; i++) + if ((connector_ids_g[i] = H5VLregister_connector(&fake_wait_vol_g, H5P_DEFAULT)) < 0) + TEST_ERROR + /* Tests */ nerrors += test_es_create(); nerrors += test_es_none(); + nerrors += test_es_get_requests(); + + /* Unregister dummy connectors */ + for (i = 0; i < EVENT_SET_NUM_CONNECTOR_IDS; i++) + if (H5VLunregister_connector(connector_ids_g[i]) < 0) + TEST_ERROR /* Cleanup */ h5_cleanup(FILENAME, fapl_id); /* Check for any errors */ - if (nerrors) { - HDputs("***** EVENT SET TESTS FAILED *****"); - HDexit(EXIT_FAILURE); - } /* end if */ + if (nerrors) + goto error; /* Report status */ HDputs("All event set tests passed."); HDexit(EXIT_SUCCESS); + +error: + HDputs("***** EVENT SET TESTS FAILED *****"); + HDexit(EXIT_FAILURE); } /* end main() */ diff --git a/test/external.c b/test/external.c index c31701d..11185b3 100644 --- a/test/external.c +++ b/test/external.c @@ -890,8 +890,8 @@ test_write_file_set(hid_t fapl) for (i = 0; i < N_EXT_FILES; i++) { char name1[64], name2[64]; - HDsprintf(name1, "extern_%dr.raw", i + 1); - HDsprintf(name2, "extern_%dw.raw", i + 1); + HDsnprintf(name1, sizeof(name1), "extern_%dr.raw", i + 1); + HDsnprintf(name2, sizeof(name2), "extern_%dw.raw", i + 1); if (!files_have_same_contents(name1, name2)) FAIL_PUTS_ERROR(" Output differs from expected value.") } /* end for */ @@ -1217,7 +1217,7 @@ test_path_relative_cwd(hid_t fapl) } H5E_END_TRY; if (dset3 >= 0) - FAIL_PUTS_ERROR("reopening the dataset with a different efile_prefix succeded"); + FAIL_PUTS_ERROR("reopening the dataset with a different efile_prefix succeeded"); /* Read the entire dataset and compare with the original */ HDmemset(whole, 0, sizeof(whole)); @@ -1233,7 +1233,7 @@ test_path_relative_cwd(hid_t fapl) if (H5Dclose(dset) < 0) FAIL_STACK_ERROR - /* Open dataset (use a differend prefix than for create. + /* Open dataset (use a different prefix than for create. * This works because the dataset was closed. */ if (H5Pset_efile_prefix(dapl2, "${ORIGIN}/.") < 0) @@ -1254,7 +1254,7 @@ test_path_relative_cwd(hid_t fapl) } H5E_END_TRY; if (dset3 >= 0) - FAIL_PUTS_ERROR("reopening the dataset with a different efile_prefix succeded"); + FAIL_PUTS_ERROR("reopening the dataset with a different efile_prefix succeeded"); /* Read the entire dataset and compare with the original */ HDmemset(whole, 0, sizeof(whole)); diff --git a/test/external_common.c b/test/external_common.c index c37fd16..3f04260 100644 --- a/test/external_common.c +++ b/test/external_common.c @@ -59,9 +59,9 @@ reset_raw_data_files(hbool_t is_env) /* Open file */ if (is_env) - HDsprintf(filename, "extern_env_%lur.raw", (unsigned long)i + 1); + HDsnprintf(filename, sizeof(filename), "extern_env_%lur.raw", (unsigned long)i + 1); else - HDsprintf(filename, "extern_%lur.raw", (unsigned long)i + 1); + HDsnprintf(filename, sizeof(filename), "extern_%lur.raw", (unsigned long)i + 1); if ((fd = HDopen(filename, O_RDWR | O_CREAT | O_TRUNC, H5_POSIX_CREATE_MODE_RW)) < 0) goto error; @@ -96,9 +96,9 @@ reset_raw_data_files(hbool_t is_env) /* Open file */ if (is_env) - HDsprintf(filename, "extern_env_%luw.raw", (unsigned long)i + 1); + HDsnprintf(filename, sizeof(filename), "extern_env_%luw.raw", (unsigned long)i + 1); else - HDsprintf(filename, "extern_%luw.raw", (unsigned long)i + 1); + HDsnprintf(filename, sizeof(filename), "extern_%luw.raw", (unsigned long)i + 1); if ((fd = HDopen(filename, O_RDWR | O_CREAT | O_TRUNC, H5_POSIX_CREATE_MODE_RW)) < 0) goto error; diff --git a/test/family_v16-000000.h5 b/test/family_v16-000000.h5 new file mode 100644 index 0000000..ac75ea9 Binary files /dev/null and b/test/family_v16-000000.h5 differ diff --git a/test/family_v16-000001.h5 b/test/family_v16-000001.h5 new file mode 100644 index 0000000..bf55b32 Binary files /dev/null and b/test/family_v16-000001.h5 differ diff --git a/test/family_v16-000002.h5 b/test/family_v16-000002.h5 new file mode 100644 index 0000000..9bd0891 Binary files /dev/null and b/test/family_v16-000002.h5 differ diff --git a/test/family_v16-000003.h5 b/test/family_v16-000003.h5 new file mode 100644 index 0000000..fedce2a Binary files /dev/null and b/test/family_v16-000003.h5 differ diff --git a/test/family_v16_00000.h5 b/test/family_v16_00000.h5 deleted file mode 100644 index ac75ea9..0000000 Binary files a/test/family_v16_00000.h5 and /dev/null differ diff --git a/test/family_v16_00001.h5 b/test/family_v16_00001.h5 deleted file mode 100644 index bf55b32..0000000 Binary files a/test/family_v16_00001.h5 and /dev/null differ diff --git a/test/family_v16_00002.h5 b/test/family_v16_00002.h5 deleted file mode 100644 index 9bd0891..0000000 Binary files a/test/family_v16_00002.h5 and /dev/null differ diff --git a/test/family_v16_00003.h5 b/test/family_v16_00003.h5 deleted file mode 100644 index fedce2a..0000000 Binary files a/test/family_v16_00003.h5 and /dev/null differ diff --git a/test/farray.c b/test/farray.c index 950c7a7..d31d77a 100644 --- a/test/farray.c +++ b/test/farray.c @@ -969,7 +969,7 @@ typedef struct fiter_fw_t { /*------------------------------------------------------------------------- * Function: fiter_fw_init * - * Purpose: Initialize element interator (forward iteration) + * Purpose: Initialize element iterator (forward iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -1021,7 +1021,7 @@ fiter_fw_next(void *_fiter) /*------------------------------------------------------------------------- * Function: fiter_term * - * Purpose: Shut down element interator (simple iterators) + * Purpose: Shut down element iterator (simple iterators) * * Return: Success: 0 * Failure: -1 @@ -1055,7 +1055,7 @@ typedef struct fiter_rv_t { /*------------------------------------------------------------------------- * Function: fiter_rv_init * - * Purpose: Initialize element interator (reverse iteration) + * Purpose: Initialize element iterator (reverse iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -1120,7 +1120,7 @@ typedef struct fiter_rnd_t { /*------------------------------------------------------------------------- * Function: fiter_rnd_init * - * Purpose: Initialize element interator (random iteration) + * Purpose: Initialize element iterator (random iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL @@ -1193,7 +1193,7 @@ fiter_rnd_next(void *_fiter) /*------------------------------------------------------------------------- * Function: fiter_rnd_term * - * Purpose: Shut down element interator (random iteration) + * Purpose: Shut down element iterator (random iteration) * * Return: Success: 0 * Failure: -1 @@ -1235,7 +1235,7 @@ typedef struct fiter_cyc_t { /*------------------------------------------------------------------------- * Function: fiter_cyc_init * - * Purpose: Initialize element interator (cyclic iteration) + * Purpose: Initialize element iterator (cyclic iteration) * * Return: Success: Pointer to iteration status object * Failure: NULL diff --git a/test/fheap.c b/test/fheap.c index 86a555d..b897063 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -466,40 +466,24 @@ error: return (1); } /* add_obj() */ -/*------------------------------------------------------------------------- - * Function: get_del_string - * - * Purpose: Return string describing the kind of deletion to perform - * - * Return: Success: 0 - * - * Failure: 1 - * - * Programmer: Quincey Koziol - * Monday, June 6, 2006 - * - *------------------------------------------------------------------------- - */ -static char * +/* Return a string describing the kind of deletion to perform. */ +static const char * get_del_string(const fheap_test_param_t *tparam) { - char *str; - /* Remove half of total objects from heap */ if (tparam->del_dir == FHEAP_DEL_FORWARD) if (tparam->drain_half == FHEAP_DEL_DRAIN_ALL) - str = H5MM_strdup("(all - forward)"); + return "(all - forward)"; else - str = H5MM_strdup("(half, refill, all - forward)"); + return "(half, refill, all - forward)"; else if (tparam->del_dir == FHEAP_DEL_REVERSE) if (tparam->drain_half == FHEAP_DEL_DRAIN_ALL) - str = H5MM_strdup("(all - reverse)"); + return "(all - reverse)"; else - str = H5MM_strdup("(half, refill, all - reverse)"); + return "(half, refill, all - reverse)"; else - str = H5MM_strdup("(all - deleting heap)"); + return "(all - deleting heap)"; - return (str); } /* get_del_string() */ /*------------------------------------------------------------------------- @@ -547,28 +531,20 @@ get_fill_size(const fheap_test_param_t *tparam) * *------------------------------------------------------------------------- */ -/* Disable warning for "format not a string literal" here -QAK */ -/* - * This pragma only needs to surround the snprintf() calls with - * test_desc in the code below, but early (4.4.7, at least) gcc only - * allows diagnostic pragmas to be toggled outside of functions. - */ -H5_GCC_CLANG_DIAG_OFF("format-nonliteral") static int begin_test(fheap_test_param_t *tparam, const char *base_desc, fheap_heap_ids_t *keep_ids, size_t *fill_size) { - char *del_str = NULL; /* Deletion order description */ - char *test_desc = NULL; /* Test description */ + char * test_desc; /* Test description */ + const char *del_str = get_del_string(tparam); /* * Test filling & removing all (small) objects from root direct block of absolute heap */ - del_str = get_del_string(tparam); - HDassert(del_str); - test_desc = (char *)H5MM_malloc(HDstrlen(del_str) + HDstrlen(base_desc)); - HDsprintf(test_desc, base_desc, del_str); + size_t test_desc_len = strlen(base_desc) + sizeof(" ") + strlen(del_str); + test_desc = H5MM_malloc(test_desc_len); + (void)HDsnprintf(test_desc, test_desc_len, "%s %s", base_desc, del_str); + TESTING(test_desc); - H5MM_xfree(del_str); H5MM_xfree(test_desc); /* Initialize the heap ID structure */ @@ -581,7 +557,6 @@ begin_test(fheap_test_param_t *tparam, const char *base_desc, fheap_heap_ids_t * /* Success */ return (0); } /* end begin_test() */ -H5_GCC_CLANG_DIAG_ON("format-nonliteral") /*------------------------------------------------------------------------- * Function: reopen_file @@ -2469,7 +2444,7 @@ test_id_limits(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) size_t id_len; /* Size of fractal heap IDs */ size_t tiny_max_len; /* Max. length of tiny objects */ hbool_t tiny_len_extended; /* Do tiny objects use two bytes for the length? */ - hbool_t huge_ids_direct; /* Are 'huge' objects directly acccessed? */ + hbool_t huge_ids_direct; /* Are 'huge' objects directly accessed? */ /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -7769,7 +7744,7 @@ test_man_remove_root_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_ size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from root direct block of absolute heap %s"; + const char *base_desc = "removing all objects from root direct block of absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -7842,7 +7817,7 @@ test_man_remove_two_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from two direct blocks of absolute heap %s"; + const char *base_desc = "removing all objects from two direct blocks of absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -7930,7 +7905,7 @@ test_man_remove_first_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from first row of direct blocks of absolute heap %s"; + const char *base_desc = "removing all objects from first row of direct blocks of absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8000,7 +7975,7 @@ test_man_remove_first_two_rows(hid_t fapl, H5HF_create_t *cparam, fheap_test_par size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from first two rows of direct blocks of absolute heap %s"; + const char *base_desc = "removing all objects from first two rows of direct blocks of absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8072,7 +8047,7 @@ test_man_remove_first_four_rows(hid_t fapl, H5HF_create_t *cparam, fheap_test_pa size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from first four rows of direct blocks of absolute heap %s"; + const char *base_desc = "removing all objects from first four rows of direct blocks of absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8148,7 +8123,7 @@ test_man_remove_all_root_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_pa size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from all direct blocks of root group in absolute heap %s"; + const char *base_desc = "removing all objects from all direct blocks of root group in absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8218,7 +8193,7 @@ test_man_remove_2nd_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test_param size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from 2nd level indirect blocks of absolute heap %s"; + const char *base_desc = "removing all objects from 2nd level indirect blocks of absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8292,7 +8267,7 @@ test_man_remove_3rd_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test_param size_t fill_size; /* Size of objects for "bulk" filled blocks */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "removing all objects from 3rd level indirect blocks of absolute heap %s"; + const char *base_desc = "removing all objects from 3rd level indirect blocks of absolute heap"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8373,8 +8348,7 @@ test_man_skip_start_block(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t size_t obj_size; /* Size of object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = - "inserting object that is too large for starting block, then remove all objects %s"; + const char *base_desc = "inserting object that is too large for starting block, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -8453,7 +8427,7 @@ test_man_skip_start_block_add_back(hid_t fapl, H5HF_create_t *cparam, fheap_test fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ const char *base_desc = - "skipping starting block, then adding object back to first block, then remove all objects %s"; + "skipping starting block, then adding object back to first block, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -8553,7 +8527,7 @@ test_man_skip_start_block_add_skipped(hid_t fapl, H5HF_create_t *cparam, fheap_t fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ const char *base_desc = - "skipping starting block, then adding objects to backfill and extend, then remove all objects %s"; + "skipping starting block, then adding objects to backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8662,7 +8636,7 @@ test_man_skip_2nd_block(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *t fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ const char *base_desc = "insert object to initial block, then add object too large for starting direct " - "blocks, then remove all objects %s"; + "blocks, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -8761,7 +8735,7 @@ test_man_skip_2nd_block_add_skipped(hid_t fapl, H5HF_create_t *cparam, fheap_tes unsigned v; /* Local index variables */ /* Test description */ const char *base_desc = "insert object to initial block, then add object too large for starting direct " - "blocks, then backfill and extend, then remove all objects %s"; + "blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -8907,7 +8881,7 @@ test_man_fill_one_partial_skip_2nd_block_add_skipped(hid_t fapl, H5HF_create_t * unsigned u; /* Local index variable */ /* Test description */ const char *base_desc = - "skipping blocks with indirect root, then backfill and extend, then remove all objects %s"; + "skipping blocks with indirect root, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -9072,7 +9046,7 @@ test_man_fill_row_skip_add_skipped(hid_t fapl, H5HF_create_t *cparam, fheap_test fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ const char *base_desc = - "filling first row, then skipping rows, then backfill and extend, then remove all objects %s"; + "filling first row, then skipping rows, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -9202,7 +9176,7 @@ test_man_skip_direct_skip_indirect_two_rows_add_skipped(hid_t fapl, H5HF_create_ unsigned v; /* Local index variables */ /* Test description */ const char *base_desc = "skipping direct blocks to last row and skipping two rows of root indirect " - "block, then backfill and extend, then remove all objects %s"; + "block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -9328,7 +9302,7 @@ test_man_fill_direct_skip_indirect_start_block_add_skipped(hid_t fapl, H5HF_crea fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ const char *base_desc = "filling direct blocks and skipping blocks in non-root indirect block, then " - "backfill and extend, then remove all objects %s"; + "backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -9457,7 +9431,7 @@ test_man_fill_direct_skip_2nd_indirect_start_block_add_skipped(hid_t fapl, H5HF_ unsigned u; /* Local index variable */ /* Test description */ const char *base_desc = "filling direct blocks and skipping row of non-root indirect blocks, then " - "backfill and extend, then remove all objects %s"; + "backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -9589,7 +9563,7 @@ test_man_fill_2nd_direct_less_one_wrap_start_block_add_skipped(hid_t fapl, H5HF_ /* Test description */ const char *base_desc = "filling direct blocks, filling 2nd level indirect blocks, except last one, and insert object too " - "large for 2nd level indirect blocks, then backfill and extend, then remove all objects %s"; + "large for 2nd level indirect blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -9735,7 +9709,7 @@ test_man_fill_direct_skip_2nd_indirect_skip_2nd_block_add_skipped(hid_t fapl, H5 unsigned u; /* Local index variable */ /* Test description */ const char *base_desc = "filling direct blocks and skipping row of non-root indirect blocks, then skip " - "row of direct blocks, then backfill and extend, then remove all objects %s"; + "row of direct blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -9898,7 +9872,7 @@ test_man_fill_direct_skip_indirect_two_rows_add_skipped(hid_t fapl, H5HF_create_ unsigned u, v; /* Local index variables */ /* Test description */ const char *base_desc = "filling direct blocks and skipping two rows of root indirect block, then " - "backfill and extend, then remove all objects %s"; + "backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -10055,7 +10029,7 @@ test_man_fill_direct_skip_indirect_two_rows_skip_indirect_row_add_skipped(hid_t /* Test description */ const char *base_desc = "filling direct blocks and skipping two rows of root indirect block, skip one row of root indirect " - "block, then backfill and extend, then remove all objects %s"; + "block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -10235,7 +10209,7 @@ test_man_fill_2nd_direct_skip_start_block_add_skipped(hid_t fapl, H5HF_create_t /* Test description */ const char *base_desc = "filling direct blocks, filling 2nd level indirect blocks, and skip first rows of direct blocks of " - "3rd level indirect block, then backfill and extend, then remove all objects %s"; + "3rd level indirect block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -10367,7 +10341,7 @@ test_man_fill_2nd_direct_skip_2nd_indirect_start_block_add_skipped(hid_t fapl, H const char *base_desc = "filling direct blocks, filling 2nd level indirect blocks, filling 3rd level indirect block's direct " "blocks, and skip first rows of direct blocks of 3rd level indirect block's 2nd level indirect " - "block, then backfill and extend, then remove all objects %s"; + "block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -10511,7 +10485,7 @@ test_man_fill_2nd_direct_fill_direct_skip_3rd_indirect_start_block_add_skipped(h /* Test description */ const char *base_desc = "filling direct blocks, filling 2nd level indirect blocks, filling 3rd level " "indirect block's direct blocks, and skip first row of indirect blocks of 3rd " - "level indirect block, then backfill and extend, then remove all objects %s"; + "level indirect block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -10664,7 +10638,7 @@ test_man_fill_2nd_direct_fill_direct_skip2_3rd_indirect_start_block_add_skipped( /* Test description */ const char *base_desc = "filling direct blocks, filling 2nd level indirect blocks, filling 3rd level " "indirect block's direct blocks, and skip first two rows of indirect blocks of " - "3rd level indirect block, then backfill and extend, then remove all objects %s"; + "3rd level indirect block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -10822,7 +10796,7 @@ test_man_fill_3rd_direct_less_one_fill_direct_wrap_start_block_add_skipped(hid_t const char *base_desc = "filling direct blocks, filling 2nd level indirect blocks, filling first row of 3rd level indirect " "blocks, except last one, fill all direct blocks in last 3rd level indirect block, and insert object " - "too large for it's 2nd level indirect blocks, then backfill and extend, then remove all objects %s"; + "too large for it's 2nd level indirect blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -10986,7 +10960,7 @@ test_man_fill_1st_row_3rd_direct_fill_2nd_direct_less_one_wrap_start_block_add_s "filling direct blocks, filling 2nd level indirect blocks, filling first row of 3rd level indirect " "blocks, fill all direct blocks in next 3rd level indirect block, fill all 1st row of 2nd level " "indirect blocks, except last one, and insert object too large for 2nd level indirect block, then " - "backfill and extend, then remove all objects %s"; + "backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -11154,7 +11128,7 @@ test_man_fill_3rd_direct_fill_direct_skip_start_block_add_skipped(hid_t fapl, H5 const char *base_desc = "filling direct blocks, filling 2nd level indirect blocks, filling 3rd level indirect blocks, fill " "4th level indirect block's direct blocks, and skip first row of 2nd indirect blocks of 4th level " - "indirect block, then backfill and extend, then remove all objects %s"; + "indirect block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -11318,7 +11292,7 @@ test_man_fill_3rd_direct_fill_2nd_direct_fill_direct_skip_3rd_indirect_start_blo "filling direct blocks, filling 2nd level indirect blocks, filling 3rd level indirect blocks, fill " "4th level indirect block's direct, 2nd level indirect blocks and 3rd level direct block, and skip " "first row of 2nd indirect blocks of 4th level indirect block's 3rd level indirect block, then " - "backfill and extend, then remove all objects %s"; + "backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -11502,7 +11476,7 @@ test_man_fill_3rd_direct_fill_2nd_direct_fill_direct_skip_3rd_indirect_two_rows_ "first row of 4th level indirect blocks, fill 2nd row 4th level indirect block's direct, 2nd level " "indirect blocks, first row of 3rd level indirect blocks, 3rd level direct block in 2nd row, and " "skip first row of 2nd indirect blocks of 4th level indirect block's 3rd level indirect block, then " - "backfill and extend, then remove all objects %s"; + "backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -11721,7 +11695,7 @@ test_man_fill_3rd_direct_fill_2nd_direct_fill_direct_skip_3rd_indirect_wrap_star "filling direct blocks, filling 2nd level indirect blocks, filling 3rd level indirect blocks, fill " "first row of 3rd level indirect blocks in 4th level indirect block except last 3rd level block, " "fill direct blocks in 3rd level block, and skip row of 2nd indirect blocks of 4th level indirect " - "block's 3rd level indirect block, then backfill and extend, then remove all objects %s"; + "block's 3rd level indirect block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -11922,7 +11896,7 @@ test_man_fill_4th_direct_less_one_fill_2nd_direct_fill_direct_skip_3rd_indirect_ "first row of 4th level indirect blocks, except last one, fill first row of 3rd level indirect " "blocks in last 4th level indirect block except last 3rd level block, fill direct blocks in 3rd " "level block, and skip row of 2nd indirect blocks of 4th level indirect block's 3rd level indirect " - "block, then backfill and extend, then remove all objects %s"; + "block, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -12143,7 +12117,7 @@ test_man_frag_simple(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar fheap_heap_state_t state; /* State of fractal heap */ unsigned u; /* Local index variables */ /* Test description */ - const char *base_desc = "fragmenting small blocks, then backfill and extend, then remove all objects %s"; + const char *base_desc = "fragmenting small blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -12275,7 +12249,7 @@ test_man_frag_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar fheap_heap_state_t state; /* State of fractal heap */ unsigned u, v; /* Local index variables */ /* Test description */ - const char *base_desc = "fragmenting direct blocks, then backfill and extend, then remove all objects %s"; + const char *base_desc = "fragmenting direct blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -12449,7 +12423,7 @@ test_man_frag_2nd_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * unsigned u, v; /* Local index variables */ /* Test description */ const char *base_desc = "fill root direct blocks, then fragment 2nd level indirect block's direct " - "blocks, then backfill and extend, then remove all objects %s"; + "blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -12561,7 +12535,7 @@ test_man_frag_3rd_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * /* Test description */ const char *base_desc = "fill root direct blocks and 2nd level indirect blocks, then fragment 3rd level indirect block's " - "direct blocks, then backfill and extend, then remove all objects %s"; + "direct blocks, then backfill and extend, then remove all objects"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, &fill_size) < 0) @@ -12675,7 +12649,7 @@ test_huge_insert_one(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar unsigned char obj_type; /* Type of storage for object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "insert one huge object, then remove %s"; + const char *base_desc = "insert one huge object, then remove"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -12826,7 +12800,7 @@ test_huge_insert_two(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar unsigned char obj_type; /* Type of storage for object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "insert two huge objects, then remove %s"; + const char *base_desc = "insert two huge objects, then remove"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -13057,7 +13031,7 @@ test_huge_insert_three(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tp unsigned char obj_type; /* Type of storage for object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "insert three huge objects, then remove %s"; + const char *base_desc = "insert three huge objects, then remove"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -13364,7 +13338,7 @@ test_huge_insert_mix(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar unsigned char obj_type; /* Type of storage for object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "insert mix of normal & huge objects, then remove %s"; + const char *base_desc = "insert mix of normal & huge objects, then remove"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -13785,10 +13759,10 @@ test_filtered_huge(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam fheap_heap_state_t state; /* State of fractal heap */ unsigned deflate_level; /* Deflation level */ size_t old_actual_id_len = 0; /* Old actual ID length */ - hbool_t huge_ids_direct; /* Are 'huge' objects directly acccessed? */ + hbool_t huge_ids_direct; /* Are 'huge' objects directly accessed? */ hbool_t pline_init = FALSE; /* Whether the I/O pipeline has been initialized */ /* Test description */ - const char *base_desc = "insert 'huge' object into heap with I/O filters, then remove %s"; + const char *base_desc = "insert 'huge' object into heap with I/O filters, then remove"; /* Copy heap creation properties */ HDmemcpy(&tmp_cparam, cparam, sizeof(H5HF_create_t)); @@ -13996,7 +13970,7 @@ test_tiny_insert_one(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar unsigned char obj_type; /* Type of storage for object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "insert one tiny object, then remove %s"; + const char *base_desc = "insert one tiny object, then remove"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -14147,7 +14121,7 @@ test_tiny_insert_two(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar unsigned char obj_type; /* Type of storage for object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "insert two tiny objects, then remove %s"; + const char *base_desc = "insert two tiny objects, then remove"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -14383,7 +14357,7 @@ test_tiny_insert_mix(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpar unsigned char obj_type; /* Type of storage for object */ fheap_heap_state_t state; /* State of fractal heap */ /* Test description */ - const char *base_desc = "insert mix of normal, huge & tiny objects, then remove %s"; + const char *base_desc = "insert mix of normal, huge & tiny objects, then remove"; /* Perform common test initialization operations */ if (begin_test(tparam, base_desc, &keep_ids, NULL) < 0) @@ -14984,7 +14958,7 @@ test_filtered_man_root_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_para fheap_heap_state_t state; /* State of fractal heap */ unsigned deflate_level; /* Deflation level */ /* Test description */ - const char *base_desc = "insert one 'managed' object into heap with I/O filters, then remove %s"; + const char *base_desc = "insert one 'managed' object into heap with I/O filters, then remove"; /* Copy heap creation properties */ HDmemcpy(&tmp_cparam, cparam, sizeof(H5HF_create_t)); @@ -15158,7 +15132,7 @@ test_filtered_man_root_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test_pa fheap_heap_state_t state; /* State of fractal heap */ unsigned deflate_level; /* Deflation level */ /* Test description */ - const char *base_desc = "insert two 'managed' objects into heap with I/O filters, then remove %s"; + const char *base_desc = "insert two 'managed' objects into heap with I/O filters, then remove"; /* Copy heap creation properties */ HDmemcpy(&tmp_cparam, cparam, sizeof(H5HF_create_t)); @@ -16361,15 +16335,15 @@ main(void) unsigned num_pb_fs = 1; /* The number of settings to test for page buffering and file space handling */ int ExpressMode; /* Express testing level */ const char *envval; /* Environment variable */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ hbool_t api_ctx_pushed = FALSE; /* Whether API context pushed */ /* Don't run this test using certain file drivers */ - envval = HDgetenv("HDF5_DRIVER"); + envval = HDgetenv(HDF5_DRIVER); if (envval == NULL) envval = "nomatch"; - /* Current VFD that does not support contigous address space */ + /* Current VFD that does not support contiguous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") != 0 && HDstrcmp(envval, "multi") != 0); /* Reset library */ diff --git a/test/file_image.c b/test/file_image.c index 0d031f5..7f6a83e 100644 --- a/test/file_image.c +++ b/test/file_image.c @@ -145,7 +145,7 @@ test_properties(void) /* Check that sizes are the same, and that the buffers are identical but separate */ VERIFY(temp_size == size, "Sizes of buffers don't match"); - VERIFY(temp2 != NULL, "Recieved buffer not set"); + VERIFY(temp2 != NULL, "Received buffer not set"); VERIFY(temp2 != buffer, "Retrieved buffer is the same as original"); VERIFY(temp2 != temp, "Retrieved buffer is the same as previously retrieved buffer"); VERIFY(0 == HDmemcmp(temp2, buffer, size), "Buffers contain different data"); @@ -260,7 +260,7 @@ free_cb(void *ptr, H5FD_file_image_op_t op, void *udata) * Function: udata_copy_cb * * Purpose: This function allows calls to the udata_copy callback to be tracked. - * No copying actualy takes place; it is easier to deal with only one + * No copying actually takes place; it is easier to deal with only one * instance of the udata. * * Returns: A pointer to the same udata that was passed in. @@ -954,7 +954,7 @@ test_get_file_image(const char *test_banner, const int file_name_num, hid_t fapl err = H5Fclose(core_file_id); VERIFY(err == SUCCEED, "H5Fclose(core_file_id) failed."); - /* dicard core fapl */ + /* discard core fapl */ err = H5Pclose(core_fapl_id); VERIFY(err == SUCCEED, "H5Pclose(core_fapl_id) failed."); @@ -1078,7 +1078,7 @@ test_get_file_image_error_rejection(void) VERIFY(bytes_read < 0, "H5Fget_file_image(2 -- test 1) succeeded."); /* Call H5Fget_file_image() with good buffer and buffer size, - * but non-existant file_id. Should fail. + * but non-existent file_id. Should fail. */ H5E_BEGIN_TRY { @@ -1361,7 +1361,10 @@ main(void) errors += test_properties(); errors += test_callbacks(); - errors += test_core(); + + if (!h5_driver_uses_modified_filename()) { + errors += test_core(); + } /* Perform tests with/without user block */ for (user = FALSE; user <= TRUE; user++) { diff --git a/test/filenotclosed.c b/test/filenotclosed.c index b5def9b..58edc06 100644 --- a/test/filenotclosed.c +++ b/test/filenotclosed.c @@ -68,7 +68,7 @@ main(void) hbool_t contig_addr_vfd; /* Contiguous address vfd */ /* Get the VFD to use */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; diff --git a/test/fillval.c b/test/fillval.c index 8de6ef1..1a0c02f 100644 --- a/test/fillval.c +++ b/test/fillval.c @@ -1918,7 +1918,7 @@ test_extend_cases(hid_t file, hid_t _dcpl, const char *dset_name, const hsize_t if ((fspace = H5Dget_space(dset)) < 0) TEST_ERROR - /* Create dataspace for single element sized bufer */ + /* Create dataspace for single element sized buffer */ if ((mspace = H5Screate_simple(5, one, NULL)) < 0) TEST_ERROR @@ -2142,7 +2142,7 @@ test_extend(hid_t fapl, const char *base_name, H5D_layout_t layout) #if 1 /* - * Remove this when contiguous datasets can be exended to some + * Remove this when contiguous datasets can be extended to some * predetermined fininte size, even if it's just in the slowest varying * dimension. If it's removed prematurely then you'll get one of the * errors described above or `unable to select fill value region'. @@ -2722,7 +2722,10 @@ main(int argc, char *argv[]) nerrors += test_create(my_fapl, FILENAME[1], H5D_CONTIGUOUS); nerrors += test_rdwr(my_fapl, FILENAME[3], H5D_CONTIGUOUS); nerrors += test_extend(my_fapl, FILENAME[5], H5D_CONTIGUOUS); - nerrors += test_compatible(); + + if (!h5_driver_uses_modified_filename()) { + nerrors += test_compatible(); + } } /* end if */ /* Compact dataset storage tests */ diff --git a/test/filter_fail.c b/test/filter_fail.c index a1bcf28..64cc41a 100644 --- a/test/filter_fail.c +++ b/test/filter_fail.c @@ -160,7 +160,7 @@ test_filter_write(char *file_name, hid_t my_fapl, hbool_t cache_enabled) points[i] = i; /* Write data. If the chunk cache is enabled, H5Dwrite should succeed. If it is - * diabled, H5Dwrite should fail. */ + * disabled, H5Dwrite should fail. */ if (cache_enabled) { if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, sid, H5P_DEFAULT, points) < 0) TEST_ERROR @@ -186,7 +186,7 @@ test_filter_write(char *file_name, hid_t my_fapl, hbool_t cache_enabled) TEST_ERROR /* Close dataset. If the chunk cache is enabled, the flushing of chunks should fail - * during H5Dclose. If it is diabled, H5Dwrite should fail but H5Dclose should succeed. */ + * during H5Dclose. If it is disabled, H5Dwrite should fail but H5Dclose should succeed. */ if (cache_enabled) { H5E_BEGIN_TRY { diff --git a/test/filter_plugin1_dsets.c b/test/filter_plugin1_dsets.c index 5d6c1ef..29b6e73 100644 --- a/test/filter_plugin1_dsets.c +++ b/test/filter_plugin1_dsets.c @@ -79,7 +79,7 @@ add_sub_value(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_value value = (int)cd_values[0]; if (flags & H5Z_FLAG_REVERSE) { - /* READ - Substract the given value from all the data values */ + /* READ - Subtract the given value from all the data values */ while (buf_left > 0) { *int_ptr++ -= value; buf_left -= sizeof(int); diff --git a/test/filter_plugin3_dsets.c b/test/filter_plugin3_dsets.c index 618ce06..1d399e5 100644 --- a/test/filter_plugin3_dsets.c +++ b/test/filter_plugin3_dsets.c @@ -108,7 +108,7 @@ add_sub_value_hdf5(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_ value = (int)cd_values[0]; if (flags & H5Z_FLAG_REVERSE) { - /* READ - Substract the given value from all the data values */ + /* READ - Subtract the given value from all the data values */ while (buf_left > 0) { *int_ptr++ -= value; buf_left -= sizeof(int); diff --git a/test/flush1.c b/test/flush1.c index 456f5f4..0ddceb6 100644 --- a/test/flush1.c +++ b/test/flush1.c @@ -81,7 +81,7 @@ create_file(const char *filename, hid_t fapl_id, hbool_t swmr) if ((top_gid = H5Gcreate2(fid, "top_group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) STACK_ERROR for (i = 0; i < NGROUPS; i++) { - HDsprintf(group_name, "group%02d", i); + HDsnprintf(group_name, sizeof(group_name), "group%02d", i); if ((gid = H5Gcreate2(top_gid, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) STACK_ERROR if (H5Gclose(gid) < 0) @@ -201,7 +201,7 @@ main(void) TEST_ERROR /* Check if the current VFD supports SWMR */ - driver = HDgetenv("HDF5_DRIVER"); + driver = HDgetenv(HDF5_DRIVER); vfd_supports_swmr = H5FD__supports_swmr_test(driver); /*************************************************/ diff --git a/test/flush2.c b/test/flush2.c index 3262b14..b6c7761 100644 --- a/test/flush2.c +++ b/test/flush2.c @@ -144,7 +144,7 @@ file_ok(const char *filename, hid_t fapl_id, hbool_t check_second_dset) if ((top_gid = H5Gopen2(fid, "top_group", H5P_DEFAULT)) < 0) goto error; for (i = 0; i < NGROUPS; i++) { - HDsprintf(group_name, "group%02d", i); + HDsnprintf(group_name, sizeof(group_name), "group%02d", i); if ((gid = H5Gopen2(top_gid, group_name, H5P_DEFAULT)) < 0) goto error; if (H5Gclose(gid) < 0) @@ -251,7 +251,7 @@ main(void) PUTS_ERROR("bad vfd-dependent fapl") /* Check if the current VFD supports SWMR */ - driver = HDgetenv("HDF5_DRIVER"); + driver = HDgetenv(HDF5_DRIVER); vfd_supports_swmr = H5FD__supports_swmr_test(driver); /* TEST 1 */ diff --git a/test/flushrefresh.c b/test/flushrefresh.c index 5efff2f..0c10238 100644 --- a/test/flushrefresh.c +++ b/test/flushrefresh.c @@ -63,10 +63,10 @@ #define NOT_FLUSHED "NOT_FLUSHED" /* Error Handling */ -/* For errors occuring in the main process, use the standard TEST_ERROR macro. +/* For errors occurring in the main process, use the standard TEST_ERROR macro. For errors occurring in the spawned process (from the test script), use the PROCESS_ERROR macro, which will send a signal to the main process so the - main process can propogate errors correctly. */ + main process can propagate errors correctly. */ FILE *errorfile; #define ERRFILE "flushrefresh_ERROR" #define PROCESS_ERROR \ @@ -93,7 +93,7 @@ FILE *errorfile; /* ===================== */ /* Main */ -int main(int argc, const char *argv[]); +int main(int argc, char *argv[]); /* Flush Test Framework */ herr_t test_flush(void); @@ -137,7 +137,7 @@ herr_t end_verification(void); *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { /* Variables */ const char *envval = NULL; @@ -153,7 +153,7 @@ main(int argc, const char *argv[]) * anything. */ /* Determine driver being used */ - envval = HDgetenv("HDF5_DRIVER"); + envval = HDgetenv(HDF5_DRIVER); if (envval == NULL || H5FD__supports_swmr_test(envval)) { if (test_flush() != SUCCEED) @@ -222,7 +222,7 @@ test_flush(void) * object, a series of verifications will occur on each object in the file. * * Each verification consists of spawning off a new process and determining - * if the object can be opened and its information retreived in said + * if the object can be opened and its information retrieved in said * alternate process. It reports the results, which are compared to an * expected value (either that the object can be found on disk, or that it * cannot). @@ -695,7 +695,7 @@ test_refresh(void) * that an attribute has been added, and is only visible after a * successful call to a H5*refresh function). * - * As with the flush case, the implemention is a bit tricky as it's + * As with the flush case, the implementation is a bit tricky as it's * dealing with signals going back and forth between the two processes * to ensure the timing is correct, but basically, an example: * @@ -1166,7 +1166,7 @@ error: * Purpose: This function is used to communicate with the verification * process spawned by the start_refresh_verification_process * function. It gives it the go-ahead to call H5*refresh - * on an object and conlcude the refresh verification. + * on an object and conclude the refresh verification. * * Return: 0 on Success, 1 on Failure * @@ -1371,7 +1371,7 @@ error: * * Purpose: This function checks the status of external verification * processes to see if they've succeeded. It checks for the - * existance of flushrefresh_ERROR file. If present, that indicates + * existence of flushrefresh_ERROR file. If present, that indicates * an external verification process has failed, and this function * thus fails as well. If not present, then nothing else has * failed, and this function succeeds. diff --git a/test/freespace.c b/test/freespace.c index e888b2b..1107609 100644 --- a/test/freespace.c +++ b/test/freespace.c @@ -661,13 +661,16 @@ test_fs_sect_add(hid_t fapl) /* Close the file */ if (H5Fclose(file) < 0) FAIL_STACK_ERROR + sect_node = NULL; - /* Get the size of a file w/empty heap*/ - if ((tmp_file_size = h5_get_file_size(filename, fapl)) < 0) - TEST_ERROR + if (h5_using_default_driver(NULL)) { + /* Get the size of a file w/empty heap*/ + if ((tmp_file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR - if (tmp_file_size <= (file_size + fr_meta_size)) - TEST_ERROR + if (tmp_file_size <= (file_size + fr_meta_size)) + TEST_ERROR + } PASSED(); @@ -723,13 +726,16 @@ test_fs_sect_add(hid_t fapl) /* Close the file */ if (H5Fclose(file) < 0) FAIL_STACK_ERROR + sect_node = NULL; - /* Get the size of a file w/empty heap*/ - if ((tmp_file_size = h5_get_file_size(filename, fapl)) < 0) - TEST_ERROR + if (h5_using_default_driver(NULL)) { + /* Get the size of a file w/empty heap*/ + if ((tmp_file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR - if (tmp_file_size != (file_size + fr_meta_size)) - TEST_ERROR + if (tmp_file_size != (file_size + fr_meta_size)) + TEST_ERROR + } PASSED(); @@ -799,6 +805,7 @@ test_fs_sect_add(hid_t fapl) /* Close the file */ if (H5Fclose(file) < 0) FAIL_STACK_ERROR + sect_node = NULL; PASSED(); @@ -878,6 +885,7 @@ test_fs_sect_add(hid_t fapl) /* Close the file */ if (H5Fclose(file) < 0) FAIL_STACK_ERROR + sect_node = NULL; PASSED(); diff --git a/test/gen_bounds.c b/test/gen_bounds.c index 8148f16..87a9c18 100644 --- a/test/gen_bounds.c +++ b/test/gen_bounds.c @@ -858,12 +858,12 @@ gen_sel_files(const char *filename, H5F_libver_t low_bound, H5F_libver_t high_bo if (H5Rcreate(&ref_wbuf[0], fid, DATASET, H5R_DATASET_REGION, sid) < 0) TEST_ERROR; - /* Create the reference datset */ + /* Create the reference dataset */ if ((ref_did = H5Dcreate2(fid, SEL_EX_REG_DSET, H5T_STD_REF_DSETREG, ref_sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; - /* Write to the reference datset */ + /* Write to the reference dataset */ if (H5Dwrite(ref_did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_wbuf) < 0) TEST_ERROR; @@ -886,12 +886,12 @@ gen_sel_files(const char *filename, H5F_libver_t low_bound, H5F_libver_t high_bo if (H5Rcreate(&ref_wbuf[0], fid, DATASET, H5R_DATASET_REGION, sid) < 0) TEST_ERROR; - /* Create the reference datset */ + /* Create the reference dataset */ if ((ref_did = H5Dcreate2(fid, SEL_EX_IRR_DSET, H5T_STD_REF_DSETREG, ref_sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; - /* Write to the reference datset */ + /* Write to the reference dataset */ if (H5Dwrite(ref_did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_wbuf) < 0) TEST_ERROR; @@ -915,12 +915,12 @@ gen_sel_files(const char *filename, H5F_libver_t low_bound, H5F_libver_t high_bo if (H5Rcreate(&ref_wbuf[0], fid, DATASET, H5R_DATASET_REGION, sid) < 0) TEST_ERROR; - /* Create the reference datset */ + /* Create the reference dataset */ if ((ref_did = H5Dcreate2(fid, SEL_EX_PT_DSET, H5T_STD_REF_DSETREG, ref_sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; - /* Write to the reference datset */ + /* Write to the reference dataset */ if (H5Dwrite(ref_did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_wbuf) < 0) TEST_ERROR; diff --git a/test/gen_new_array.c b/test/gen_new_array.c index 5811106..8161c4e 100644 --- a/test/gen_new_array.c +++ b/test/gen_new_array.c @@ -89,7 +89,7 @@ main(void) if (ret < 0) printf("field 1 insert<0!\n"); - /* Creat the array datatype */ + /* Create the array datatype */ arr_type = H5Tarray_create2(H5T_NATIVE_FLOAT, ARRAY1_RANK, tdims1); if (arr_type < 0) printf("arr_type<0!\n"); @@ -104,7 +104,7 @@ main(void) if (ret < 0) printf("field 3 array close<0!\n"); - /* Creat the array datatype */ + /* Create the array datatype */ arr_type = H5Tarray_create2(H5T_NATIVE_LONG, ARRAY1_RANK, tdims1); if (arr_type < 0) printf("arr_type<0!\n"); diff --git a/test/gen_old_group.c b/test/gen_old_group.c index 1b4e0b2..15e67a4 100644 --- a/test/gen_old_group.c +++ b/test/gen_old_group.c @@ -19,7 +19,7 @@ * empty "symbol table" group. * * This file is used in the v1.7 branch (after the "compact group" - * checkin) to test compatability. Compile and run this + * checkin) to test compatibility. Compile and run this * program (with the 1.6.x branch), it will generate a file* * called "group_old.h5". You need to move it to the test * directory in the HDF5 v1.7 source tree. The test/stab.c diff --git a/test/gen_plist.c b/test/gen_plist.c index 10c6423..25d48cf 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -81,7 +81,7 @@ main(void) H5AC_cache_image_config_t my_cache_image_config = {H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, TRUE, FALSE, -1}; - /* check endianess */ + /* check endianness */ { short int word = 0x0001; char * byte = (char *)&word; diff --git a/test/genall5.c b/test/genall5.c index f1189fb..d7bef23 100644 --- a/test/genall5.c +++ b/test/genall5.c @@ -14,7 +14,7 @@ /* Programmer: John Mainzer * 9/23/15 * - * This file contains a heavily edited and functionaly reduce + * This file contains a heavily edited and functionally reduce * version of the test code first written by Quincey in a file * of the same name. */ diff --git a/test/getname.c b/test/getname.c index 200c426..ba33280 100644 --- a/test/getname.c +++ b/test/getname.c @@ -1526,7 +1526,7 @@ test_main(hid_t file_id, hid_t fapl) { char name[NAME_BUF_SIZE]; /* Buffer to hold name and its size */ - /* Get name for non commited datatype, it should fail */ + /* Get name for non committed datatype, it should fail */ H5E_BEGIN_TRY { if (H5Iget_name(type_id, name, NAME_BUF_SIZE) > 0) @@ -3775,8 +3775,12 @@ main(void) /* Call "main" test routine */ nerrors += test_main(file_id, fapl); - nerrors += test_obj_ref(fapl); - nerrors += test_reg_ref(fapl); + + if (!h5_using_parallel_driver(NULL)) { + nerrors += test_obj_ref(fapl); + nerrors += test_reg_ref(fapl); + } + nerrors += test_elinks(fapl); /* Close file */ diff --git a/test/gheap.c b/test/gheap.c index 567fd65..f1182bd 100644 --- a/test/gheap.c +++ b/test/gheap.c @@ -353,7 +353,7 @@ error: * Function: test_4 * * Purpose: Tests the H5HG_remove() feature by writing lots of objects - * and occassionally removing some. When we're done they're all + * and occasionally removing some. When we're done they're all * removed. * * Return: Success: 0 diff --git a/test/h5test.c b/test/h5test.c index 09fb5b5..0528623 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -40,10 +40,7 @@ * to the cpp constant. If neither is defined then use some default value. * * HDF5_DRIVER: This string describes what low level file driver to - * use for HDF5 file access. The first word in the - * value is the name of the driver and subsequent data - * is interpreted according to the driver. See - * h5_get_vfd_fapl() for details. + * use for HDF5 file access. * * HDF5_LIBVER_BOUNDS: This string describes what library version bounds to * use for HDF5 file access. See h5_get_libver_fapl() for details. @@ -525,8 +522,8 @@ static char * h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fullname, size_t size, hbool_t nest_printf, hbool_t subst_for_superblock) { - const char *prefix = NULL; - const char *env = NULL; /* HDF5_DRIVER environment variable */ + const char *prefix = NULL; + const char *driver_env_var = NULL; /* HDF5_DRIVER environment variable */ char * ptr, last = '\0'; const char *suffix = _suffix; size_t i, j; @@ -538,6 +535,13 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fu HDmemset(fullname, 0, size); + /* Determine if driver is set by environment variable. If it is, + * only generate a suffix if fixing the filename for the superblock + * file. */ + driver_env_var = HDgetenv(HDF5_DRIVER); + if (driver_env_var && (H5P_DEFAULT == fapl) && subst_for_superblock) + fapl = H5P_FILE_ACCESS_DEFAULT; + /* figure out the suffix */ if (H5P_DEFAULT != fapl) { if ((driver = H5Pget_driver(fapl)) < 0) @@ -546,28 +550,25 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fu if (suffix) { if (H5FD_FAMILY == driver) { if (subst_for_superblock) - suffix = "00000.h5"; + suffix = "-000000.h5"; else - suffix = nest_printf ? "%%05d.h5" : "%05d.h5"; + suffix = nest_printf ? "-%%06d.h5" : "-%06d.h5"; } else if (H5FD_MULTI == driver) { - /* Get the environment variable, if it exists, in case + /* Check the HDF5_DRIVER environment variable in case * we are using the split driver since both of those * use the multi VFD under the hood. */ - env = HDgetenv("HDF5_DRIVER"); #ifdef HDF5_DRIVER /* Use the environment variable, then the compile-time constant */ - if (!env) - env = HDF5_DRIVER; + if (!driver_env_var) + driver_env_var = HDF5_DRIVER; #endif - if (env && !HDstrcmp(env, "split")) { + if (driver_env_var && !HDstrcmp(driver_env_var, "split")) { /* split VFD */ if (subst_for_superblock) - suffix = "-m.h5"; - else - suffix = NULL; + suffix = ".h5.meta"; } else { /* multi VFD */ @@ -583,7 +584,7 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fu /* Must first check fapl is not H5P_DEFAULT (-1) because H5FD_XXX * could be of value -1 if it is not defined. */ - isppdriver = H5P_DEFAULT != fapl && (H5FD_MPIO == driver); + isppdriver = ((H5P_DEFAULT != fapl) || driver_env_var) && (H5FD_MPIO == driver); /* Check HDF5_NOCLEANUP environment setting. * (The #ifdef is needed to prevent compile failure in case MPI is not @@ -591,12 +592,12 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fu */ if (isppdriver) { #ifdef H5_HAVE_PARALLEL - if (getenv_all(MPI_COMM_WORLD, 0, "HDF5_NOCLEANUP")) + if (getenv_all(MPI_COMM_WORLD, 0, HDF5_NOCLEANUP)) SetTestNoCleanup(); #endif /* H5_HAVE_PARALLEL */ } else { - if (HDgetenv("HDF5_NOCLEANUP")) + if (HDgetenv(HDF5_NOCLEANUP)) SetTestNoCleanup(); } @@ -798,11 +799,7 @@ h5_fileaccess(void) if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; - /* Attempt to set up a file driver first */ - if (h5_get_vfd_fapl(fapl_id) < 0) - goto error; - - /* Finally, check for libver bounds */ + /* Check for libver bounds */ if (h5_get_libver_fapl(fapl_id) < 0) goto error; @@ -837,11 +834,7 @@ h5_fileaccess_flags(unsigned flags) if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; - /* Attempt to set up a file driver first */ - if ((flags & H5_FILEACCESS_VFD) && h5_get_vfd_fapl(fapl_id) < 0) - goto error; - - /* Finally, check for libver bounds */ + /* Check for libver bounds */ if ((flags & H5_FILEACCESS_LIBVER) && h5_get_libver_fapl(fapl_id) < 0) goto error; @@ -854,150 +847,6 @@ error: } /* end h5_fileaccess_flags() */ /*------------------------------------------------------------------------- - * Function: h5_get_vfd_fapl - * - * Purpose: Sets the file driver for a FAPL according to the value specified - * in the constant or environment variable "HDF5_DRIVER". - * - * Return: Success: 0 - * Failure: -1 - * - * Programmer: Dana Robinson - * February 2016 - * - *------------------------------------------------------------------------- - */ -herr_t -h5_get_vfd_fapl(hid_t fapl) -{ - const char *env = NULL; /* HDF5_DRIVER environment variable */ - const char *tok = NULL; /* strtok pointer */ - char * lasts = NULL; /* Context pointer for strtok_r() call */ - char buf[1024]; /* buffer for tokenizing HDF5_DRIVER */ - - /* Get the environment variable, if it exists */ - env = HDgetenv("HDF5_DRIVER"); -#ifdef HDF5_DRIVER - /* Use the environment variable, then the compile-time constant */ - if (!env) - env = HDF5_DRIVER; -#endif - - /* If the environment variable was not set, just return - * without modifying the FAPL. - */ - if (!env || !*env) - goto done; - - /* Get the first 'word' of the environment variable. - * If it's nothing (environment variable was whitespace) - * just return the default fapl. - */ - HDstrncpy(buf, env, sizeof(buf)); - buf[sizeof(buf) - 1] = '\0'; - if (NULL == (tok = HDstrtok_r(buf, " \t\n\r", &lasts))) - goto done; - - if (!HDstrcmp(tok, "sec2")) { - /* POSIX (section 2) read() and write() system calls */ - if (H5Pset_fapl_sec2(fapl) < 0) - goto error; - } - else if (!HDstrcmp(tok, "stdio")) { - /* Standard C fread() and fwrite() system calls */ - if (H5Pset_fapl_stdio(fapl) < 0) - goto error; - } - else if (!HDstrcmp(tok, "core")) { - /* In-memory driver settings (backing store on, 1 MB increment) */ - if (H5Pset_fapl_core(fapl, (size_t)H5_MB, TRUE) < 0) - goto error; - } - else if (!HDstrcmp(tok, "core_paged")) { - /* In-memory driver with write tracking and paging on */ - if (H5Pset_fapl_core(fapl, (size_t)H5_MB, TRUE) < 0) - goto error; - if (H5Pset_core_write_tracking(fapl, TRUE, (size_t)4096) < 0) - goto error; - } - else if (!HDstrcmp(tok, "split")) { - /* Split meta data and raw data each using default driver */ - if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) - goto error; - } - else if (!HDstrcmp(tok, "multi")) { - /* Multi-file driver, general case of the split driver */ - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; - hid_t memb_fapl[H5FD_MEM_NTYPES]; - const char *memb_name[H5FD_MEM_NTYPES]; - char * sv[H5FD_MEM_NTYPES]; - haddr_t memb_addr[H5FD_MEM_NTYPES]; - H5FD_mem_t mt; - - HDmemset(memb_map, 0, sizeof(memb_map)); - HDmemset(memb_fapl, 0, sizeof(memb_fapl)); - HDmemset(memb_name, 0, sizeof(memb_name)); - HDmemset(memb_addr, 0, sizeof(memb_addr)); - - HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { - memb_fapl[mt] = H5P_DEFAULT; - sv[mt] = (char *)HDmalloc(H5TEST_MULTI_FILENAME_LEN); - HDassert(sv[mt]); - HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); - memb_name[mt] = sv[mt]; - memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); - } /* end for */ - - if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0) - goto error; - - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) - HDfree(sv[mt]); - } - else if (!HDstrcmp(tok, "family")) { - /* Family of files, each 1MB and using the default driver */ - hsize_t fam_size = 100 * 1024 * 1024; /* 100 MB */ - - /* Was a family size specified in the environment variable? */ - if ((tok = HDstrtok_r(NULL, " \t\n\r", &lasts))) - fam_size = (hsize_t)(HDstrtod(tok, NULL) * 1024 * 1024); - if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT) < 0) - goto error; - } - else if (!HDstrcmp(tok, "log")) { - /* Log file access */ - unsigned log_flags = H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC; - - /* Were special log file flags specified in the environment variable? */ - if ((tok = HDstrtok_r(NULL, " \t\n\r", &lasts))) - log_flags = (unsigned)HDstrtol(tok, NULL, 0); - - if (H5Pset_fapl_log(fapl, NULL, log_flags, (size_t)0) < 0) - goto error; -#ifdef H5_HAVE_DIRECT - } - else if (!HDstrcmp(tok, "direct")) { - /* Linux direct read() and write() system calls. Set memory boundary, - * file block size, and copy buffer size to the default values. - */ - if (H5Pset_fapl_direct(fapl, 1024, 4096, 8 * 4096) < 0) - goto error; -#endif - } - else { - /* Unknown driver */ - goto error; - } /* end if */ - -done: - return 0; - -error: - return -1; -} /* end h5_get_vfd_fapl() */ - -/*------------------------------------------------------------------------- * Function: h5_get_libver_fapl * * Purpose: Sets the library version bounds for a FAPL according to the @@ -1333,30 +1182,58 @@ h5_get_file_size(const char *filename, hid_t fapl) #ifdef H5_HAVE_DIRECT driver == H5FD_DIRECT || #endif /* H5_HAVE_DIRECT */ - driver == H5FD_LOG) { + driver == H5FD_LOG || driver == H5FD_SPLITTER) { /* Get the file's statistics */ if (0 == HDstat(filename, &sb)) return ((h5_stat_size_t)sb.st_size); } /* end if */ else if (driver == H5FD_MULTI) { H5FD_mem_t mt; - h5_stat_size_t tot_size = 0; - - HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { - /* Create the filename to query */ - HDsnprintf(temp, sizeof temp, "%s-%c.h5", filename, multi_letters[mt]); - - /* Check for existence of file */ - if (0 == HDaccess(temp, F_OK)) { - /* Get the file's statistics */ - if (0 != HDstat(temp, &sb)) - return (-1); - - /* Add to total size */ - tot_size += (h5_stat_size_t)sb.st_size; - } /* end if */ - } /* end for */ + h5_stat_size_t tot_size = 0; + char * driver_env_var = NULL; + + driver_env_var = HDgetenv(HDF5_DRIVER); + if (driver_env_var && !HDstrcmp(driver_env_var, "split")) { + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { + if (mt != H5FD_MEM_DRAW && mt != H5FD_MEM_SUPER) + continue; + + /* Create the filename to query */ + if (mt == H5FD_MEM_DRAW) { + HDsnprintf(temp, sizeof temp, "%s.raw", filename); + } + else { + HDsnprintf(temp, sizeof temp, "%s.meta", filename); + } + + /* Check for existence of file */ + if (0 == HDaccess(temp, F_OK)) { + /* Get the file's statistics */ + if (0 != HDstat(temp, &sb)) + return (-1); + + /* Add to total size */ + tot_size += (h5_stat_size_t)sb.st_size; + } /* end if */ + } /* end for */ + } + else { + HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { + /* Create the filename to query */ + HDsnprintf(temp, sizeof temp, "%s-%c.h5", filename, multi_letters[mt]); + + /* Check for existence of file */ + if (0 == HDaccess(temp, F_OK)) { + /* Get the file's statistics */ + if (0 != HDstat(temp, &sb)) + return (-1); + + /* Add to total size */ + tot_size += (h5_stat_size_t)sb.st_size; + } /* end if */ + } /* end for */ + } /* Return total size */ return (tot_size); @@ -1404,7 +1281,9 @@ h5_get_file_size(const char *filename, hid_t fapl) return (tot_size); } /* end if */ else { - HDassert(0 && "Unknown VFD!"); + /* Get the file's statistics */ + if (0 == HDstat(filename, &sb)) + return ((h5_stat_size_t)sb.st_size); } /* end else */ } /* end else */ @@ -1786,7 +1665,7 @@ h5_send_message(const char *send, const char *arg1, const char *arg2) * signal file from disk, and only continues once it has * successfully done so (i.e., only after another process has * called the "h5_send_message" function to write the signal file). - * This functon will then immediately remove the file (i.e., + * This function will then immediately remove the file (i.e., * to indicate that it has been received and can be reused), * and then exits, allowing the calling function to continue. * @@ -1892,7 +1771,9 @@ dummy_vfd_write(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hi } /* Dummy VFD with the minimum parameters to make a VFD that can be registered */ +#define DUMMY_VFD_VALUE (H5FD_class_value_t)155 static const H5FD_class_t H5FD_dummy_g = { + DUMMY_VFD_VALUE, /* value */ "dummy", /* name */ 1, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ @@ -1925,6 +1806,7 @@ static const H5FD_class_t H5FD_dummy_g = { NULL, /* lock */ NULL, /* unlock */ NULL, /* del */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -2287,3 +2169,137 @@ error: } return FAIL; } /* end h5_check_if_file_locking_enabled() */ + +/*------------------------------------------------------------------------- + * Function: h5_using_default_driver + * + * Purpose: Checks if the specified VFD name matches the library's + * default VFD. If `drv_name` is NULL, the HDF5_DRIVER + * environment is checked instead (if it is set). + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +h5_using_default_driver(const char *drv_name) +{ + hbool_t ret_val = TRUE; + + HDassert(H5_DEFAULT_VFD == H5FD_SEC2); + + if (!drv_name) + drv_name = HDgetenv(HDF5_DRIVER); + + if (drv_name) + return (!HDstrcmp(drv_name, "sec2") || !HDstrcmp(drv_name, "nomatch")); + + return ret_val; +} + +/*------------------------------------------------------------------------- + * Function: h5_using_parallel_driver + * + * Purpose: Checks if the specified VFD name matches a parallel-enabled + * VFD (usually `mpio`). If `drv_name` is NULL, the + * HDF5_DRIVER environment is checked instead (if it is set). + * + * This is mostly useful for avoiding tests that use features + * which are not currently supported for parallel HDF5, such + * as writing of VL or region reference datatypes. + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +h5_using_parallel_driver(const char *drv_name) +{ + hbool_t ret_val = FALSE; + + if (!drv_name) + drv_name = HDgetenv(HDF5_DRIVER); + + if (drv_name) + return (!HDstrcmp(drv_name, "mpio")); + + return ret_val; +} + +/*------------------------------------------------------------------------- + * Function: h5_driver_uses_modified_filename + * + * Purpose: Checks if the current VFD set by use of the HDF5_DRIVER + * environment variable uses a modified filename. Examples + * are the multi and family drivers. + * + * This routine is helpful for skipping tests that use + * pre-generated files. VFDs that use a modified filename will + * not be able to find these files and those tests will fail. + * Eventually, HDF5's testing framework should be modified to + * not run VFD testing against tests that use pre-generated + * files. + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +h5_driver_uses_modified_filename(void) +{ + hbool_t ret_val = FALSE; + char * driver = HDgetenv(HDF5_DRIVER); + + if (driver) { + ret_val = !HDstrcmp(driver, "multi") || !HDstrcmp(driver, "split") || !HDstrcmp(driver, "family") || + !HDstrcmp(driver, "splitter"); + } + + return ret_val; +} /* end h5_driver_uses_modified_filename() */ + +/*------------------------------------------------------------------------- + * Function: h5_driver_uses_multiple_files + * + * Purpose: Checks if the specified VFD name matches a driver that + * stores data using multiple files. + * + * The following flags can be used to control what types of + * drivers are checked for by this routine: + * + * H5_EXCLUDE_MULTIPART_DRIVERS - This flag excludes any + * drivers which store data using multiple files which, + * together, make up a single logical file. These are + * drivers like the split, multi and family drivers. + * + * H5_EXCLUDE_NON_MULTIPART_DRIVERS - This flag excludes any + * drivers which store data using multiple files which are + * separate logical files. The splitter driver is an example + * of this type of driver. + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +h5_driver_uses_multiple_files(const char *drv_name, unsigned flags) +{ + hbool_t ret_val = FALSE; + + if (!drv_name) + drv_name = HDgetenv(HDF5_DRIVER); + + if (drv_name) { + if ((flags & H5_EXCLUDE_MULTIPART_DRIVERS) == 0) { + if (!HDstrcmp(drv_name, "split") || !HDstrcmp(drv_name, "multi") || !HDstrcmp(drv_name, "family")) + return TRUE; + } + + if ((flags & H5_EXCLUDE_NON_MULTIPART_DRIVERS) == 0) { + if (!HDstrcmp(drv_name, "splitter")) + return TRUE; + } + } + + return ret_val; +} diff --git a/test/h5test.h b/test/h5test.h index 2855fb5..7b82b68 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -166,63 +166,25 @@ H5TEST_DLLVAR MPI_Info h5_io_info_g; /* MPI INFO object for IO */ #define H5_ALARM_SEC 1200 /* default is 20 minutes */ /* Flags for h5_fileaccess_flags() */ -#define H5_FILEACCESS_VFD 0x01 -#define H5_FILEACCESS_LIBVER 0x02 +#define H5_FILEACCESS_LIBVER 0x01 -/* Macros to create and fill 2D arrays with a single heap allocation. - * These can be used to replace large stack and global arrays which raise - * warnings. - * - * The macros make a single heap allocation large enough to hold all the - * pointers and the data elements. The first part of the allocation holds - * the pointers, and the second part holds the data as a contiguous block - * in row-major order. - * - * To pass the data block to calls like H5Dread(), pass a pointer to the - * first array element as the data pointer (e.g., array[0] in a 2D array). - * - * The fill macro just fills the array with an increasing count value. - * - * Usage: - * - * int **array; - * - * H5TEST_ALLOCATE_2D_ARRAY(array, int, 5, 10); - * - * H5TEST_FILL_2D_ARRAY(array, int, 5, 10); - * - * (do stuff) - * - * HDfree(array); - */ -#define H5TEST_ALLOCATE_2D_ARRAY(ARR, TYPE, DIMS_I, DIMS_J) \ - do { \ - /* Prefix with h5taa to avoid shadow warnings */ \ - size_t h5taa_pointers_size = 0; \ - size_t h5taa_data_size = 0; \ - int h5taa_i; \ - \ - h5taa_pointers_size = (DIMS_I) * sizeof(TYPE *); \ - h5taa_data_size = (DIMS_I) * (DIMS_J) * sizeof(TYPE); \ - \ - ARR = (TYPE **)HDmalloc(h5taa_pointers_size + h5taa_data_size); \ - \ - ARR[0] = (TYPE *)(ARR + (DIMS_I)); \ - \ - for (h5taa_i = 1; h5taa_i < (DIMS_I); h5taa_i++) \ - ARR[h5taa_i] = ARR[h5taa_i - 1] + (DIMS_J); \ - } while (0) +/* Flags for h5_driver_uses_multiple_files() */ +#define H5_EXCLUDE_MULTIPART_DRIVERS 0x01 +#define H5_EXCLUDE_NON_MULTIPART_DRIVERS 0x02 -#define H5TEST_FILL_2D_ARRAY(ARR, TYPE, DIMS_I, DIMS_J) \ +/* Fill an array on the heap with an increasing count value. BUF + * is expected to point to a `struct { TYPE arr[...][...]; }`. + */ +#define H5TEST_FILL_2D_HEAP_ARRAY(BUF, TYPE) \ do { \ /* Prefix with h5tfa to avoid shadow warnings */ \ - int h5tfa_i = 0; \ - int h5tfa_j = 0; \ - TYPE h5tfa_count = 0; \ + size_t h5tfa_i = 0; \ + size_t h5tfa_j = 0; \ + TYPE h5tfa_count = 0; \ \ - for (h5tfa_i = 0; h5tfa_i < (DIMS_I); h5tfa_i++) \ - for (h5tfa_j = 0; h5tfa_j < (DIMS_J); h5tfa_j++) { \ - ARR[h5tfa_i][h5tfa_j] = h5tfa_count; \ + for (h5tfa_i = 0; h5tfa_i < NELMTS((BUF)->arr); h5tfa_i++) \ + for (h5tfa_j = 0; h5tfa_j < NELMTS((BUF)->arr[0]); h5tfa_j++) { \ + (BUF)->arr[h5tfa_i][h5tfa_j] = h5tfa_count; \ h5tfa_count++; \ } \ } while (0) @@ -255,9 +217,12 @@ H5TEST_DLL const char * h5_get_version_string(H5F_libver_t libver); H5TEST_DLL int h5_compare_file_bytes(char *fname1, char *fname2); H5TEST_DLL int h5_duplicate_file_by_bytes(const char *orig, const char *dest); H5TEST_DLL herr_t h5_check_if_file_locking_enabled(hbool_t *are_enabled); +H5TEST_DLL hbool_t h5_using_default_driver(const char *drv_name); +H5TEST_DLL hbool_t h5_using_parallel_driver(const char *drv_name); +H5TEST_DLL hbool_t h5_driver_uses_modified_filename(void); +H5TEST_DLL hbool_t h5_driver_uses_multiple_files(const char *drv_name, unsigned flags); /* Functions that will replace components of a FAPL */ -H5TEST_DLL herr_t h5_get_vfd_fapl(hid_t fapl_id); H5TEST_DLL herr_t h5_get_libver_fapl(hid_t fapl_id); /* h5_clean_files() replacements */ diff --git a/test/lheap.c b/test/lheap.c index 2609510..283c3e3 100644 --- a/test/lheap.c +++ b/test/lheap.c @@ -174,27 +174,29 @@ main(void) goto error; PASSED(); - /* Check opening existing file non-default sizes of lengths and addresses */ - TESTING("opening pre-created file with non-default sizes"); - { - const char *testfile = H5_get_srcdir_filename(TESTFILE); /* Corrected test file name */ - hid_t dset = -1; - file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); - if (file >= 0) { - if ((dset = H5Dopen2(file, "/Dataset1", H5P_DEFAULT)) < 0) - TEST_ERROR - if (H5Dclose(dset) < 0) - TEST_ERROR - if (H5Fclose(file) < 0) - TEST_ERROR + if (!h5_driver_uses_modified_filename()) { + /* Check opening existing file non-default sizes of lengths and addresses */ + TESTING("opening pre-created file with non-default sizes"); + { + const char *testfile = H5_get_srcdir_filename(TESTFILE); /* Corrected test file name */ + hid_t dset = -1; + file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file >= 0) { + if ((dset = H5Dopen2(file, "/Dataset1", H5P_DEFAULT)) < 0) + TEST_ERROR + if (H5Dclose(dset) < 0) + TEST_ERROR + if (H5Fclose(file) < 0) + TEST_ERROR + } + else { + H5_FAILED(); + HDprintf("***cannot open the pre-created non-default sizes test file (%s)\n", testfile); + goto error; + } /* end else */ } - else { - H5_FAILED(); - HDprintf("***cannot open the pre-created non-default sizes test file (%s)\n", testfile); - goto error; - } /* end else */ + PASSED(); } - PASSED(); /* Verify symbol table messages are cached */ if (h5_verify_cached_stabs(FILENAME, fapl) < 0) diff --git a/test/links.c b/test/links.c index f022783..658f8d6 100644 --- a/test/links.c +++ b/test/links.c @@ -2144,6 +2144,7 @@ cklinks_deprec(hid_t fapl, hbool_t new_format) char linkval[LINK_BUF_SIZE]; char filename[NAME_BUF_SIZE]; herr_t status; + htri_t exists; if (new_format) TESTING("link queries using deprecated routines (w/new group format)") @@ -2183,20 +2184,20 @@ cklinks_deprec(hid_t fapl, hbool_t new_format) FAIL_STACK_ERROR H5E_BEGIN_TRY { - status = H5Lexists(file, "no_grp1/hard", H5P_DEFAULT); + exists = H5Lexists(file, "no_grp1/hard", H5P_DEFAULT); } H5E_END_TRY; - if (status >= 0) { + if (exists >= 0) { H5_FAILED(); HDputs(" H5Lexists() should have failed for a path with missing components."); TEST_ERROR } /* end if */ H5E_BEGIN_TRY { - status = H5Lexists(file, "/no_grp1/hard", H5P_DEFAULT); + exists = H5Lexists(file, "/no_grp1/hard", H5P_DEFAULT); } H5E_END_TRY; - if (status >= 0) { + if (exists >= 0) { H5_FAILED(); HDputs(" H5Lexists() should have failed for a path with missing components."); TEST_ERROR @@ -8201,7 +8202,7 @@ external_link_dangling(hid_t fapl, hbool_t new_format) if (status >= 0) { H5_FAILED(); HDputs( - " Retreiving name of object by index through dangling file external link should have failed."); + " Retrieving name of object by index through dangling file external link should have failed."); } /* end if */ /* Close root group */ @@ -8674,7 +8675,7 @@ external_link_abstar(hid_t fapl, hbool_t new_format) } H5E_END_TRY; - /* should be able to find the target file with abolute path */ + /* should be able to find the target file with absolute path */ if (gid < 0) { H5_FAILED(); HDputs(" Should have found the file in tmp_links directory."); @@ -9897,7 +9898,7 @@ external_set_elink_cb(hid_t fapl, hbool_t new_format) would report FALSE, causing problems */ base_driver = H5Pget_driver(fapl); op_data.base_fapl = (base_driver == H5FD_FAMILY || base_driver == H5FD_MULTI || - base_driver == H5FD_MPIO || base_driver == H5FD_CORE) + base_driver == H5FD_MPIO || base_driver == H5FD_CORE || base_driver == H5FD_DIRECT) ? H5P_DEFAULT : fapl; op_data.fam_size = ELINK_CB_FAM_SIZE; @@ -10071,8 +10072,11 @@ external_reset_register(void) if (H5Fclose(file) < 0) TEST_ERROR - if (HDremove(filename) != 0) - TEST_ERROR + H5E_BEGIN_TRY + { + H5Fdelete(filename, H5P_DEFAULT); + } + H5E_END_TRY; PASSED(); return SUCCEED; @@ -16595,7 +16599,7 @@ link_filters(hid_t fapl, hbool_t new_format) TEST_ERROR filesize_unfiltered = h5_get_file_size(filename, fapl); - /* Set deflate fitler */ + /* Set deflate filter */ if (H5Pset_deflate(fcpl, 6) < 0) TEST_ERROR @@ -16629,11 +16633,14 @@ link_filters(hid_t fapl, hbool_t new_format) /* Close file, get file size */ if (H5Fclose(fid) < 0) TEST_ERROR - filesize_filtered = h5_get_file_size(filename, fapl); - /* Check that the file size is smaller with the filter */ - if ((double)filesize_filtered > ((double)filesize_unfiltered * FILTER_FILESIZE_MAX_FRACTION)) - TEST_ERROR + if (h5_using_default_driver(NULL)) { + filesize_filtered = h5_get_file_size(filename, fapl); + + /* Check that the file size is smaller with the filter */ + if ((double)filesize_filtered > ((double)filesize_unfiltered * FILTER_FILESIZE_MAX_FRACTION)) + TEST_ERROR + } /* Close */ if (H5Pclose(fcpl) < 0) @@ -22544,8 +22551,9 @@ main(void) unsigned minimize_dset_oh; unsigned efc; /* Whether to use the external file cache */ const char *env_h5_drvr; /* File Driver value from environment */ + hbool_t driver_uses_modified_filename = h5_driver_uses_modified_filename(); - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; @@ -22560,7 +22568,7 @@ main(void) for (minimize_dset_oh = 0; minimize_dset_oh <= 1; minimize_dset_oh++) { if (minimize_dset_oh) { - HDprintf("\n-Testing with minimzed dataset object headers-\n"); + HDprintf("\n-Testing with minimized dataset object headers-\n"); dcpl_g = H5Pcreate(H5P_DATASET_CREATE); if (0 > dcpl_g) TEST_ERROR @@ -22611,88 +22619,100 @@ main(void) nerrors += test_deprec(my_fapl, new_format); #endif /* H5_NO_DEPRECATED_SYMBOLS */ - /* tests for external link */ - /* Test external file cache first, so it sees the default efc setting on the fapl - */ - nerrors += external_file_cache(my_fapl, new_format) < 0 ? 1 : 0; + /* Skip external link tests for splitter VFD, which has external link-related bugs */ + if (HDstrcmp(env_h5_drvr, "splitter")) { - /* This test cannot run with the EFC because it assumes that an - * intermediate file is not held open - */ - nerrors += external_link_mult(my_fapl, new_format) < 0 ? 1 : 0; + /* tests for external link */ + /* Test external file cache first, so it sees the default efc setting on the fapl + */ + nerrors += external_file_cache(my_fapl, new_format) < 0 ? 1 : 0; - /* This test cannot run with the EFC because the EFC cannot currently - * reopen a cached file with a different intent - */ - nerrors += external_set_elink_acc_flags(env_h5_drvr, my_fapl, new_format) < 0 ? 1 : 0; + /* This test cannot run with the EFC because it assumes that an + * intermediate file is not held open + */ + nerrors += external_link_mult(my_fapl, new_format) < 0 ? 1 : 0; - /* Try external link tests both with and without the external file cache */ - for (efc = FALSE; efc <= TRUE; efc++) { - if (efc) { - if (H5Pset_elink_file_cache_size(my_fapl, 8) < 0) - TEST_ERROR - HDprintf("\n---Testing with external file cache---\n"); - } /* end if */ - else { - if (H5Pset_elink_file_cache_size(my_fapl, 0) < 0) - TEST_ERROR - HDprintf("\n---Testing without external file cache---\n"); - } /* end else */ + /* This test cannot run with the EFC because the EFC cannot currently + * reopen a cached file with a different intent + */ + nerrors += external_set_elink_acc_flags(env_h5_drvr, my_fapl, new_format) < 0 ? 1 : 0; + + /* Try external link tests both with and without the external file cache */ + for (efc = FALSE; efc <= TRUE; efc++) { + if (efc) { + if (H5Pset_elink_file_cache_size(my_fapl, 8) < 0) + TEST_ERROR + HDprintf("\n---Testing with external file cache---\n"); + } /* end if */ + else { + if (H5Pset_elink_file_cache_size(my_fapl, 0) < 0) + TEST_ERROR + HDprintf("\n---Testing without external file cache---\n"); + } /* end else */ - nerrors += external_link_root(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_root(my_fapl, new_format) < 0 ? 1 : 0; #ifndef H5_NO_DEPRECATED_SYMBOLS - nerrors += external_link_root_deprec(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_root_deprec(my_fapl, new_format) < 0 ? 1 : 0; #endif /* H5_NO_DEPRECATED_SYMBOLS */ - nerrors += external_link_path(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_self(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_pingpong(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_toomany(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_dangling(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_recursive(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_query(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_path(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_self(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_pingpong(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_toomany(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_dangling(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_recursive(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_query(my_fapl, new_format) < 0 ? 1 : 0; #ifndef H5_NO_DEPRECATED_SYMBOLS - nerrors += external_link_query_deprec(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_query_deprec(my_fapl, new_format) < 0 ? 1 : 0; #endif /* H5_NO_DEPRECATED_SYMBOLS */ - nerrors += external_link_unlink_compact(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_unlink_dense(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_move(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_ride(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_closing(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_unlink_compact(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_unlink_dense(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_move(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_ride(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_closing(my_fapl, new_format) < 0 ? 1 : 0; #ifndef H5_NO_DEPRECATED_SYMBOLS - nerrors += external_link_closing_deprec(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_closing_deprec(my_fapl, new_format) < 0 ? 1 : 0; #endif /* H5_NO_DEPRECATED_SYMBOLS */ - nerrors += external_link_endian(new_format) < 0 ? 1 : 0; - nerrors += external_link_strong(my_fapl, new_format) < 0 ? 1 : 0; - - nerrors += external_link_prefix(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_abs_mainpath(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_rel_mainpath(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_cwd(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_abstar(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_abstar_cur(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_reltar(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_chdir(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_set_elink_fapl1(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_set_elink_fapl2(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_set_elink_fapl3(new_format) < 0 ? 1 : 0; - nerrors += external_set_elink_cb(my_fapl, new_format) < 0 ? 1 : 0; + + if (!driver_uses_modified_filename) { + nerrors += external_link_endian(new_format) < 0 ? 1 : 0; + } + + nerrors += external_link_strong(my_fapl, new_format) < 0 ? 1 : 0; + + nerrors += external_link_prefix(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_abs_mainpath(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_rel_mainpath(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_cwd(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_abstar(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_abstar_cur(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_reltar(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_chdir(my_fapl, new_format) < 0 ? 1 : 0; + + if (!driver_uses_modified_filename) { + nerrors += external_set_elink_fapl1(my_fapl, new_format) < 0 ? 1 : 0; + } + + nerrors += external_set_elink_fapl2(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_set_elink_fapl3(new_format) < 0 ? 1 : 0; + nerrors += external_set_elink_cb(my_fapl, new_format) < 0 ? 1 : 0; #ifdef H5_HAVE_WINDOW_PATH - nerrors += external_link_win1(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win2(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win3(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win4(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win5(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win6(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win7(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win8(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_win9(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win1(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win2(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win3(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win4(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win5(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win6(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win7(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win8(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_win9(my_fapl, new_format) < 0 ? 1 : 0; #endif - nerrors += external_symlink(env_h5_drvr, my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_copy_invalid_object(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_dont_fail_to_source(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_open_twice(my_fapl, new_format) < 0 ? 1 : 0; - nerrors += external_link_with_committed_datatype(my_fapl, new_format) < 0 ? 1 : 0; - } /* with/without external file cache */ + nerrors += external_symlink(env_h5_drvr, my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_copy_invalid_object(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_dont_fail_to_source(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_open_twice(my_fapl, new_format) < 0 ? 1 : 0; + nerrors += external_link_with_committed_datatype(my_fapl, new_format) < 0 ? 1 : 0; + } /* with/without external file cache */ + } /* These tests assume that external links are a form of UD links, * so assume that everything that passed for external links diff --git a/test/links_env.c b/test/links_env.c index 590be20..7170132 100644 --- a/test/links_env.c +++ b/test/links_env.c @@ -152,8 +152,20 @@ error: int main(void) { - hid_t fapl; /* File access property lists */ - int nerrors = 0; /* Error from tests */ + const char *env_h5_drvr; /* File driver value from environment */ + hid_t fapl; /* File access property lists */ + int nerrors = 0; /* Error from tests */ + + /* Get the VFD to use */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* Splitter VFD has issues with external links */ + if (!HDstrcmp(env_h5_drvr, "splitter")) { + HDputs(" -- SKIPPED for incompatible VFD --"); + HDexit(EXIT_SUCCESS); + } h5_reset(); fapl = h5_fileaccess(); diff --git a/test/mf.c b/test/mf.c index c753c2b..242e319 100644 --- a/test/mf.c +++ b/test/mf.c @@ -223,15 +223,16 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2; haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF; hsize_t ma_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t suitable_vfd; TESTING("H5MM_alloc() of file allocation"); /* Skip test when using VFDs that has different address spaces for each - * type of metadata allocation. + * type of metadata allocation. Also skip test for Direct VFD. */ - contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); - if (contig_addr_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && + HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -371,15 +372,16 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) haddr_t addr = 0; haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t suitable_vfd; TESTING("H5MF_try_shrink() of file allocation: test 1"); /* Skip test when using VFDs that has different address spaces for each - * type of metadata allocation. + * type of metadata allocation. Also skip test for Direct VFD. */ - contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); - if (contig_addr_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && + HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -478,9 +480,9 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TESTING("H5MF_try_shrink() of file allocation: test 2"); /* Skip test when using VFDs that has different address spaces for each - * type of metadata allocation. + * type of metadata allocation. Also skip test for Direct VFD. */ - if (contig_addr_vfd) { + if (suitable_vfd) { /* Re-open the file with meta/small data setting */ if ((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) FAIL_STACK_ERROR @@ -528,9 +530,9 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TESTING("H5MF_try_shrink() of file allocation: test 3"); /* Skip test when using VFDs that has different address spaces for each - * type of metadata allocation. + * type of metadata allocation. Also skip test for Direct VFD. */ - if (contig_addr_vfd) { + if (suitable_vfd) { /* Re-open the file with meta/small data setting */ if ((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) FAIL_STACK_ERROR @@ -573,9 +575,9 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TESTING("H5MF_try_shrink() of file allocation: test 4"); /* Skip test when using VFDs that has different address spaces for each - * type of metadata allocation. + * type of metadata allocation. Also skip test for Direct VFD. */ - if (contig_addr_vfd) { + if (suitable_vfd) { /* Re-open the file with meta/small data setting */ if ((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) FAIL_STACK_ERROR @@ -658,15 +660,16 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) htri_t was_extended; haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t suitable_vfd; TESTING("H5MF_try_extend() of file allocation: test 1"); /* Skip test when using VFDs that has different address spaces for each - * type of metadata allocation. + * type of metadata allocation. Also skip test for Direct VFD. */ - contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); - if (contig_addr_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && + HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -763,9 +766,9 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) TESTING("H5MF_try_extend() of file allocation: test 2"); /* Skip test when using VFDs that has different address spaces for each - * type of metadata allocation. + * type of metadata allocation. Also skip test for Direct VFD. */ - if (contig_addr_vfd) { + if (suitable_vfd) { /* Re-open the file with meta/small data setting */ if ((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) FAIL_STACK_ERROR @@ -1934,7 +1937,7 @@ test_mf_fs_extend(hid_t fapl) if (check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR - /* Remove section A from free-space manger */ + /* Remove section A from free-space manager */ if (H5MF__find_sect(f, H5FD_MEM_SUPER, (hsize_t)(TBLOCK_SIZE30 - 10), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) TEST_ERROR @@ -2012,7 +2015,7 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) haddr_t ma_addr = HADDR_UNDEF; hsize_t ma_size = 0; H5MF_free_section_t *sect_node = NULL; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("A free-space section absorbs an aggregator: test 1"); @@ -2188,13 +2191,14 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2; haddr_t ma_addr = HADDR_UNDEF; hsize_t ma_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator:test 1"); - /* Skip test when using VFDs that don't use the metadata aggregator */ - contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); - if (contig_addr_vfd) { + /* Skip test when using VFDs that don't use the metadata aggregator. Also skip test for Direct VFD. */ + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && + HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -2339,13 +2343,14 @@ test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2, addr3; haddr_t ma_addr = HADDR_UNDEF; hsize_t ma_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator:test 2"); - /* Skip test when using VFDs that don't use the metadata aggregator */ - contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); - if (contig_addr_vfd) { + /* Skip test when using VFDs that don't use the metadata aggregator. Also skip test for Direct VFD. */ + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && + HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -2497,7 +2502,7 @@ test_mf_aggr_alloc3(const char *env_h5_drvr, hid_t fapl) hsize_t ma_size = 0, new_ma_size = 0; haddr_t sdata_addr = HADDR_UNDEF; hsize_t sdata_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator: test 3"); @@ -2665,7 +2670,7 @@ test_mf_aggr_alloc4(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2, saddr1, saddr2, saddr3; haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF, sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0, sdata_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 4"); @@ -2814,7 +2819,7 @@ test_mf_aggr_alloc5(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2, addr3; haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 5"); @@ -2952,7 +2957,7 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF, sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0, sdata_size = 0; H5FS_stat_t state; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 6"); @@ -3126,7 +3131,7 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) haddr_t ma_addr = HADDR_UNDEF, sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, sdata_size = 0; H5FS_stat_t state; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 7"); @@ -3297,7 +3302,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF, sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0, sdata_size = 0; htri_t was_extended; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("H5MF_try_extend() of meta/sdata aggregator: test 1"); @@ -3595,7 +3600,7 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) haddr_t new_sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0; hsize_t sdata_size = 0, new_sdata_size = 0; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ TESTING("H5MF_try_shrink() of meta/sdata aggregator: test 1"); @@ -3847,16 +3852,17 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) htri_t was_extended; H5FS_stat_t state; hsize_t alignment = 0, mis_align = 0, tmp = 0, accum = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MM_alloc() of file allocation with alignment: test 1"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -3966,9 +3972,10 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TESTING("H5MF_try_shrink() of file allocation with alignment: test 2"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - if (have_alloc_vfd) { + if (suitable_vfd) { /* Re-open the file with alignment and meta/sdata setting */ if ((file = H5Fopen(filename, H5F_ACC_RDWR, fapl1)) < 0) FAIL_STACK_ERROR @@ -4023,9 +4030,10 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TESTING("H5MF_try_extend() of file allocation with alignment: test 3"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - if (have_alloc_vfd) { + if (suitable_vfd) { /* Re-open the file with alignment and meta/sdata setting */ if ((file = H5Fopen(filename, H5F_ACC_RDWR, fapl1)) < 0) FAIL_STACK_ERROR @@ -4080,6 +4088,9 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) HDputs(" Current VFD doesn't support mis-aligned fragments"); } /* end else */ + if (fapl1 >= 0 && H5Pclose(fapl1) < 0) + FAIL_STACK_ERROR + return (0); error: @@ -4135,7 +4146,7 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5FS_stat_t state; htri_t was_extended; hsize_t alignment = 0, tmp = 0, mis_align = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of free-space manager with alignment: test 1"); @@ -4297,10 +4308,11 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* Skip test when using VFDs that have their own 'alloc' callback, which * don't push mis-aligned space fragments on the file free space list + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { if ((file_size = h5_get_file_size(filename, new_fapl)) < 0) TEST_ERROR @@ -4497,16 +4509,17 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t ma_addr = HADDR_UNDEF; hsize_t ma_size = 0, mis_align = 0; hsize_t alignment = 0, tmp = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator with alignment: test 1"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -4759,16 +4772,17 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t ma_addr = HADDR_UNDEF, sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, sdata_size = 0, mis_align = 0; hsize_t alignment = 0, tmp = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator with alignment: test 2"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -5104,16 +5118,17 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t ma_addr = HADDR_UNDEF, sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, sdata_size = 0, mis_align = 0; hsize_t alignment = 0, tmp = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator with alignment: test 3"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -5417,16 +5432,17 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t ma_addr = HADDR_UNDEF; hsize_t ma_size = 0, saved_ma_size = 0; hsize_t alignment = 0, mis_align = 0, tmp = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator with alignment: test 4"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -5635,16 +5651,17 @@ test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t sdata_addr = HADDR_UNDEF, new_sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0, sdata_size = 0, new_sdata_size = 0; hsize_t alignment = 0, mis_align = 0, tmp = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator with alignment: test 5"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -5838,7 +5855,7 @@ error: * * Alignment = 4096 aggr->alloc_size = 2048 * - * Allocate first block (30) from meta_aggr: (meta_aggr is emtpy) + * Allocate first block (30) from meta_aggr: (meta_aggr is empty) * request-size is > meta_aggr->size and < meta_aggr->alloc_size * Result: * A block of meta_aggr->alloc_size is allocated from file allocation @@ -5905,16 +5922,17 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t ma_addr = HADDR_UNDEF, new_ma_addr = HADDR_UNDEF, sdata_addr = HADDR_UNDEF; hsize_t ma_size = 0, new_ma_size = 0, sdata_size = 0; hsize_t alignment = 0, mis_align = 0, tmp = 0; - hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ + hbool_t suitable_vfd; TESTING("H5MF_alloc() of meta/sdata aggregator with alignment: test 6"); /* Skip test when using VFDs that have their own 'alloc' callback, which - * don't push mis-aligned space fragments on the file free space list + * don't push mis-aligned space fragments on the file free space list. + * Also skip test for Direct VFD. */ - have_alloc_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && - HDstrcmp(env_h5_drvr, "multi") != 0); - if (have_alloc_vfd) { + suitable_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "stdio") != 0 && HDstrcmp(env_h5_drvr, "split") != 0 && + HDstrcmp(env_h5_drvr, "multi") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0); + if (suitable_vfd) { /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -6139,20 +6157,23 @@ test_mf_bug1(const char *env_h5_drvr, hid_t fapl) /* Add alignment to member files for split/multi driver */ if (split || multi) { - hid_t memb_fapl; + if (split) { + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; - /* Creat fapl */ - if ((memb_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) - TEST_ERROR + /* Get current multi settings */ + if (H5Pget_fapl_multi(copied_fapl, NULL, memb_fapl_arr, NULL, NULL, NULL) < 0) + TEST_ERROR - /* Set alignment. Note that it is the block size of the parent FAPL that - * is important here. */ - if (H5Pset_alignment(memb_fapl, 0, align) < 0) - TEST_ERROR + /* Set alignment. Note that it is the block size of the parent FAPL that + * is important here. */ + if (H5Pset_alignment(memb_fapl_arr[H5FD_MEM_SUPER], 0, align) < 0) + TEST_ERROR + if (H5Pset_alignment(memb_fapl_arr[H5FD_MEM_DRAW], 0, align) < 0) + TEST_ERROR - if (split) { /* Set split driver with new FAPLs */ - if (H5Pset_fapl_split(copied_fapl, "-m.h5", memb_fapl, "-r.h5", memb_fapl) < 0) + if (H5Pset_fapl_split(copied_fapl, "-m.h5", memb_fapl_arr[H5FD_MEM_SUPER], "-r.h5", + memb_fapl_arr[H5FD_MEM_DRAW]) < 0) TEST_ERROR } /* end if */ else { @@ -6165,14 +6186,17 @@ test_mf_bug1(const char *env_h5_drvr, hid_t fapl) /* Get current multi settings */ HDmemset(memb_name, 0, sizeof memb_name); - if (H5Pget_fapl_multi(copied_fapl, memb_map, NULL, memb_name, memb_addr, &relax) < 0) + if (H5Pget_fapl_multi(copied_fapl, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) TEST_ERROR /* Populate memb_fapl_arr, patch memb_addr so member file addresses * are aligned */ for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { - memb_fapl_arr[mt] = memb_fapl; - memb_addr[mt] = ((memb_addr[mt] + align - 1) / align) * align; + /* Set alignment. Note that it is the block size of the parent FAPL that + * is important here. */ + if (H5Pset_alignment(memb_fapl_arr[mt], 0, align) < 0) + TEST_ERROR + memb_addr[mt] = ((memb_addr[mt] + align - 1) / align) * align; } /* end for */ /* Set multi driver with new FAPLs */ @@ -6184,11 +6208,7 @@ test_mf_bug1(const char *env_h5_drvr, hid_t fapl) for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) HDfree(memb_name[mt]); } /* end else */ - - /* Close memb_fapl */ - if (H5Pclose(memb_fapl) < 0) - TEST_ERROR - } /* end if */ + } /* end if */ /* Reopen the file with alignment */ if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, copied_fapl)) < 0) @@ -6253,6 +6273,9 @@ test_mf_bug1(const char *env_h5_drvr, hid_t fapl) PASSED(); + if (H5Pclose(copied_fapl) < 0) + TEST_ERROR + /* Close file */ if (H5Fclose(file) < 0) TEST_ERROR @@ -7105,7 +7128,7 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) else TESTING("File's free-space is going away with old library format") - /* Current VFD that does not support contigous address space */ + /* Current VFD that does not support contiguous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); if (contig_addr_vfd) { @@ -7296,7 +7319,7 @@ test_mf_strat_thres_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_for else TESTING("File space strategy/persisting/threshold with old library format") - /* Current VFD that does not support contigous address space */ + /* Current VFD that does not support contiguous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); /* Set the filename to use for this test (dependent on fapl) */ @@ -7466,7 +7489,7 @@ test_mf_strat_thres_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format else TESTING("File space merge/shrink for section size < threshold with old library format") - /* Current VFD that does not support contigous address space */ + /* Current VFD that does not support contiguous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); /* Set the filename to use for this test (dependent on fapl) */ @@ -7900,6 +7923,8 @@ test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl) TEST_ERROR if (fs_persist) { + haddr_t prv_tag = HADDR_UNDEF; + /* Re-open the file */ if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) TEST_ERROR @@ -7908,6 +7933,9 @@ test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl) if (NULL == (f = (H5F_t *)H5VL_object(fid))) TEST_ERROR + /* Set the freespace tag for the metadata cache */ + H5AC_tag(H5AC__FREESPACE_TAG, &prv_tag); + /* Verify that the large generic manager is there */ H5MF__alloc_to_fs_type(f->shared, H5FD_MEM_DRAW, TBLOCK_SIZE5000, (H5F_mem_page_t *)&fs_type); if (!H5F_addr_defined(f->shared->fs_addr[fs_type])) @@ -7963,6 +7991,9 @@ test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl) if (found_addr != gaddr1) TEST_ERROR + /* Reset the previous tag */ + H5AC_tag(prv_tag, NULL); + /* Close file */ if (H5Fclose(fid) < 0) TEST_ERROR @@ -8016,7 +8047,7 @@ test_page_try_shrink(const char *env_h5_drvr, hid_t fapl) haddr_t addr1; /* Address for small metadata block */ haddr_t saddr1; /* Address for small raw data block */ haddr_t gaddr1; /* Address for large data block */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ htri_t status; /* status from shrinking */ h5_stat_size_t file_size; /* File size */ char filename[FILENAME_LEN]; /* Filename to use */ @@ -8146,7 +8177,7 @@ test_page_small_try_extend(const char *env_h5_drvr, hid_t fapl) H5F_t * f = NULL; /* Internal file object pointer */ haddr_t addr1, addr2, addr3; /* Addresses for small metadata blocks */ haddr_t saddr1; /* Address for small raw data block */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ htri_t was_extended; /* Whether the block can be extended or not */ char filename[FILENAME_LEN]; /* Filename to use */ @@ -8325,7 +8356,7 @@ test_page_large_try_extend(const char *env_h5_drvr, hid_t fapl) hid_t fcpl = -1; /* File creation property list */ H5F_t * f = NULL; /* Internal file object pointer */ haddr_t gaddr1, gaddr2, gaddr3, gaddr4; /* Addresses for large data blocks */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ htri_t was_extended; /* Whether the block can be extended or not */ char filename[FILENAME_LEN]; /* Filename to use */ @@ -8488,7 +8519,7 @@ test_page_large(const char *env_h5_drvr, hid_t fapl) hid_t fcpl = -1; /* File creation property list ID */ H5F_t * f = NULL; /* Internal file object pointer */ haddr_t gaddr1, gaddr2, gaddr3, gaddr4; /* Addresses for blocks */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ H5FS_stat_t fs_stat; /* Information for free-space manager */ h5_stat_size_t file_size; /* File size */ char filename[FILENAME_LEN]; /* Filename to use */ @@ -9120,7 +9151,7 @@ main(void) hbool_t api_ctx_pushed = FALSE; /* Whether API context pushed */ /* Get the VFD to use */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; diff --git a/test/mirror_vfd.c b/test/mirror_vfd.c index 3fa2cdc..d741519 100644 --- a/test/mirror_vfd.c +++ b/test/mirror_vfd.c @@ -130,7 +130,7 @@ mybzero(void *dest, size_t size) * Function: _populate_filepath * * Purpose: Given a directory name and a base name, concatenate the two and - * run h5fixname() to get the "actual" path to the intented target. + * run h5fixname() to get the "actual" path to the intended target. * `h5suffix' should be FALSE to keep the base name unaltered; * TRUE will append the '.h5' h5suffix to the basename... * FALSE -> h5fixname_no_suffix(), TRUE -> h5fixname() diff --git a/test/mount.c b/test/mount.c index c5ac9b3..b23de30 100644 --- a/test/mount.c +++ b/test/mount.c @@ -753,7 +753,7 @@ test_move(hid_t fapl) H5E_END_TRY; if (status >= 0) { H5_FAILED(); - HDputs(" Moving an object across files should't have been possible"); + HDputs(" Moving an object across files shouldn't have been possible"); TEST_ERROR } /* end if */ @@ -1835,7 +1835,7 @@ test_mount_after_unmount(hid_t fapl) if (HDstrcmp(objname, "/X/M/Y") != 0) TEST_ERROR - /* Rename object in file #3 that is "disconnected" from name hiearchy */ + /* Rename object in file #3 that is "disconnected" from name hierarchy */ /* (It is "disconnected" because it's parent file has been unmounted) */ if (H5Lmove(gidAMX, "M/Y", gidAMX, "M/Z", H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR diff --git a/test/mtime.c b/test/mtime.c index 43dc502..8e89d58 100644 --- a/test/mtime.c +++ b/test/mtime.c @@ -60,6 +60,7 @@ main(void) signed char buf1[32], buf2[32]; char filename[1024]; int token_cmp; + hbool_t driver_uses_modified_filename = h5_driver_uses_modified_filename(); h5_reset(); fapl = h5_fileaccess(); @@ -116,7 +117,7 @@ main(void) if (0 == oi1.ctime) { SKIPPED(); HDputs(" The modification time could not be decoded on this OS."); - HDputs(" Modification times will be mantained in the file but"); + HDputs(" Modification times will be maintained in the file but"); HDputs(" cannot be queried on this system. See H5O_mtime_decode()."); return 0; } @@ -131,63 +132,67 @@ main(void) } PASSED(); - /* Check opening existing file with old-style modification time information - * and make certain that the time is correct - */ - TESTING("accessing old modification time messages"); - - { - const char *testfile = H5_get_srcdir_filename(TESTFILE1); /* Corrected test file name */ - - file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); - if (file >= 0) { - if (H5Oget_info_by_name3(file, "/Dataset1", &oi1, H5O_INFO_TIME, H5P_DEFAULT) < 0) - TEST_ERROR; - if (oi1.ctime != MTIME1) { + if (!driver_uses_modified_filename) { + /* Check opening existing file with old-style modification time information + * and make certain that the time is correct + */ + TESTING("accessing old modification time messages"); + + { + const char *testfile = H5_get_srcdir_filename(TESTFILE1); /* Corrected test file name */ + + file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file >= 0) { + if (H5Oget_info_by_name3(file, "/Dataset1", &oi1, H5O_INFO_TIME, H5P_DEFAULT) < 0) + TEST_ERROR; + if (oi1.ctime != MTIME1) { + H5_FAILED(); + /* If this fails, examine H5Omtime.c. Modification time is very + * system dependent (e.g., on Windows DST must be hardcoded). */ + HDputs(" Old modification time incorrect"); + goto error; + } + if (H5Fclose(file) < 0) + TEST_ERROR; + } + else { H5_FAILED(); - /* If this fails, examine H5Omtime.c. Modification time is very - * system dependent (e.g., on Windows DST must be hardcoded). */ - HDputs(" Old modification time incorrect"); + HDprintf("***cannot open the pre-created old modification test file (%s)\n", testfile); goto error; - } - if (H5Fclose(file) < 0) - TEST_ERROR; + } /* end else */ } - else { - H5_FAILED(); - HDprintf("***cannot open the pre-created old modification test file (%s)\n", testfile); - goto error; - } /* end else */ + PASSED(); } - PASSED(); - /* Check opening existing file with new-style modification time information - * and make certain that the time is correct - */ - TESTING("accessing new modification time messages"); - - { - const char *testfile = H5_get_srcdir_filename(TESTFILE2); /* Corrected test file name */ - - file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); - if (file >= 0) { - if (H5Oget_info_by_name3(file, "/Dataset1", &oi2, H5O_INFO_TIME, H5P_DEFAULT) < 0) - TEST_ERROR; - if (oi2.ctime != MTIME2) { + if (!driver_uses_modified_filename) { + /* Check opening existing file with new-style modification time information + * and make certain that the time is correct + */ + TESTING("accessing new modification time messages"); + + { + const char *testfile = H5_get_srcdir_filename(TESTFILE2); /* Corrected test file name */ + + file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file >= 0) { + if (H5Oget_info_by_name3(file, "/Dataset1", &oi2, H5O_INFO_TIME, H5P_DEFAULT) < 0) + TEST_ERROR; + if (oi2.ctime != MTIME2) { + H5_FAILED(); + HDputs(" Modification time incorrect."); + goto error; + } + if (H5Fclose(file) < 0) + TEST_ERROR; + } + else { H5_FAILED(); - HDputs(" Modification time incorrect."); + HDprintf("***cannot open the pre-created old modification test file (%s)\n", testfile); goto error; - } - if (H5Fclose(file) < 0) - TEST_ERROR; + } /* end else */ } - else { - H5_FAILED(); - HDprintf("***cannot open the pre-created old modification test file (%s)\n", testfile); - goto error; - } /* end else */ + PASSED(); } - PASSED(); /* Verify symbol table messages are cached */ if (h5_verify_cached_stabs(FILENAME, fapl) < 0) diff --git a/test/ntypes.c b/test/ntypes.c index 13cc72a..37a63a4 100644 --- a/test/ntypes.c +++ b/test/ntypes.c @@ -3131,11 +3131,19 @@ main(void) nerrors += test_enum_dtype(file) < 0 ? 1 : 0; nerrors += test_array_dtype(file) < 0 ? 1 : 0; nerrors += test_array_dtype2(file) < 0 ? 1 : 0; - nerrors += test_vl_dtype(file) < 0 ? 1 : 0; - nerrors += test_vlstr_dtype(file) < 0 ? 1 : 0; + + if (!h5_using_parallel_driver(NULL)) { + nerrors += test_vl_dtype(file) < 0 ? 1 : 0; + nerrors += test_vlstr_dtype(file) < 0 ? 1 : 0; + } + nerrors += test_str_dtype(file) < 0 ? 1 : 0; - nerrors += test_refer_dtype(file) < 0 ? 1 : 0; - nerrors += test_refer_dtype2(file) < 0 ? 1 : 0; + + if (!h5_using_parallel_driver(NULL)) { + nerrors += test_refer_dtype(file) < 0 ? 1 : 0; + nerrors += test_refer_dtype2(file) < 0 ? 1 : 0; + } + nerrors += test_opaque_dtype(file) < 0 ? 1 : 0; nerrors += test_bitfield_dtype(file) < 0 ? 1 : 0; nerrors += test_ninteger() < 0 ? 1 : 0; diff --git a/test/null_vfd_plugin.c b/test/null_vfd_plugin.c new file mode 100644 index 0000000..f41da00 --- /dev/null +++ b/test/null_vfd_plugin.c @@ -0,0 +1,162 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Purpose: A simple Virtual File Driver (VFD) with almost no functionality + * that is used for testing basic VFD plugin handling (registration, etc.). + */ + +/* Public HDF5 headers */ +#include "hdf5.h" + +/* For HDF5 plugin functionality */ +#include "H5PLextern.h" + +/* This driver's header */ +#include "null_vfd_plugin.h" + +/* Prototypes */ +static H5FD_t *H5FD_null_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD_null_close(H5FD_t *_file); +static herr_t H5FD_null_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, + void *buf); +static herr_t H5FD_null_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, + const void *buf); +static haddr_t H5FD_null_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_null_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD_null_get_eof(const H5FD_t *_file, H5FD_mem_t type); + +static const H5FD_class_t H5FD_null_g = { + NULL_VFD_VALUE, /* value */ + NULL_VFD_NAME, /* name */ + 1, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + NULL, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + 0, /* fapl_size */ + NULL, /* fapl_get */ + NULL, /* fapl_copy */ + NULL, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_null_open, /* open */ + H5FD_null_close, /* close */ + NULL, /* cmp */ + NULL, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD_null_get_eoa, /* get_eoa */ + H5FD_null_set_eoa, /* set_eoa */ + H5FD_null_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD_null_read, /* read */ + H5FD_null_write, /* write */ + NULL, /* flush */ + NULL, /* truncate */ + NULL, /* lock */ + NULL, /* unlock */ + NULL, /* del */ + NULL, /* ctl */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +static H5FD_t * +H5FD_null_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) +{ + (void)name; + (void)flags; + (void)fapl_id; + (void)maxaddr; + + return NULL; +} + +static herr_t +H5FD_null_close(H5FD_t *_file) +{ + (void)_file; + + return 0; +} + +static herr_t +H5FD_null_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, void *buf) +{ + (void)_file; + (void)type; + (void)dxpl_id; + (void)addr; + (void)size; + (void)buf; + + return 0; +} + +static herr_t +H5FD_null_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf) +{ + (void)_file; + (void)type; + (void)dxpl_id; + (void)addr; + (void)size; + (void)buf; + + return 0; +} + +static haddr_t +H5FD_null_get_eoa(const H5FD_t *_file, H5FD_mem_t type) +{ + (void)_file; + (void)type; + + return HADDR_UNDEF; +} + +static herr_t +H5FD_null_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) +{ + (void)_file; + (void)type; + (void)addr; + + return 0; +} + +static haddr_t +H5FD_null_get_eof(const H5FD_t *_file, H5FD_mem_t type) +{ + (void)_file; + (void)type; + + return HADDR_UNDEF; +} + +/* These two functions are necessary to load this plugin using + * the HDF5 library. + */ + +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_VFD; +} + +const void * +H5PLget_plugin_info(void) +{ + return &H5FD_null_g; +} diff --git a/test/null_vfd_plugin.h b/test/null_vfd_plugin.h new file mode 100644 index 0000000..7b7671f --- /dev/null +++ b/test/null_vfd_plugin.h @@ -0,0 +1,33 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: The public header file for the null testing VFD plugin. + */ +#ifndef H5FDnull_H +#define H5FDnull_H + +#define H5FD_NULL (H5FD_null_init()) +#define NULL_VFD_NAME "null_vfd_plugin" +#define NULL_VFD_VALUE ((H5FD_class_value_t)200) + +#ifdef __cplusplus +extern "C" { +#endif + +H5_DLL hid_t H5FD_null_init(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/objcopy.c b/test/objcopy.c index 9aae621..e23ee8e 100644 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -2789,7 +2789,7 @@ test_copy_dataset_simple_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, h char dst_filename[NAME_BUF_SIZE]; if (test_open) { - TESTING("H5Ocopy(): empty and openend contiguous dataset"); + TESTING("H5Ocopy(): empty and opened contiguous dataset"); } else { TESTING("H5Ocopy(): empty contiguous dataset"); @@ -8679,7 +8679,7 @@ test_copy_soft_link(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t dst_fa char dst_filename[NAME_BUF_SIZE]; if (test_open) { - TESTING("H5Ocopy(): openend object through soft link"); + TESTING("H5Ocopy(): opened object through soft link"); } else { TESTING("H5Ocopy(): object through soft link"); @@ -9674,7 +9674,7 @@ test_copy_dataset_contig_named_vl(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl char src_filename[NAME_BUF_SIZE]; char dst_filename[NAME_BUF_SIZE]; - TESTING("H5Ocopy(): contigous dataset with named VLEN datatype"); + TESTING("H5Ocopy(): contiguous dataset with named VLEN datatype"); /* set initial data values */ for (i = 0; i < DIM_SIZE_1; i++) { @@ -10434,10 +10434,10 @@ test_copy_dataset_contig_vl_vl(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, h char dst_filename[NAME_BUF_SIZE]; if (test_open) { - TESTING("H5Ocopy(): contigous and opened dataset with nested VLEN datatype"); + TESTING("H5Ocopy(): contiguous and opened dataset with nested VLEN datatype"); } else { - TESTING("H5Ocopy(): contigous dataset with nested VLEN datatype"); + TESTING("H5Ocopy(): contiguous dataset with nested VLEN datatype"); } /* set initial data values */ @@ -13763,7 +13763,7 @@ test_copy_cdt_hier_merge(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t d TEST_ERROR /* - * Test 3: copy datsets in /g0 at SRC to DST group /uncopied + * Test 3: copy datasets in /g0 at SRC to DST group /uncopied */ if (H5Ocopy(fid_src, NAME_GROUP_TOP "/" SRC_DSET1, fid_dst, NAME_GROUP_UNCOPIED "/" SRC_DSET1, ocpypl_id, H5P_DEFAULT) < 0) @@ -17305,13 +17305,18 @@ error: int main(void) { - int nerrors = 0; - hid_t fapl, fapl2; - hid_t fcpl_shared, ocpl; - unsigned max_compact, min_dense; - int configuration; /* Configuration of tests. */ - int ExpressMode; - hbool_t same_file; /* Whether to run tests that only use one file */ + int nerrors = 0; + hid_t fapl, fapl2; + hid_t fcpl_shared, ocpl; + unsigned max_compact, min_dense; + int configuration; /* Configuration of tests. */ + int ExpressMode; + const char *env_h5_drvr; /* File Driver value from environment */ + hbool_t same_file; /* Whether to run tests that only use one file */ + + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; /* Setup */ h5_reset(); @@ -17468,9 +17473,14 @@ main(void) FALSE, "H5Ocopy(): expand soft link"); nerrors += test_copy_option(fcpl_src, fcpl_dst, src_fapl, dst_fapl, H5O_COPY_EXPAND_EXT_LINK_FLAG, FALSE, "H5Ocopy(): expand external link"); - nerrors += test_copy_option(fcpl_src, fcpl_dst, src_fapl, dst_fapl, - H5O_COPY_EXPAND_SOFT_LINK_FLAG | H5O_COPY_EXPAND_EXT_LINK_FLAG, FALSE, - "H5Ocopy(): expand soft and external links"); + + /* Splitter VFD currently has external link-related bugs */ + if (HDstrcmp(env_h5_drvr, "splitter")) { + nerrors += test_copy_option(fcpl_src, fcpl_dst, src_fapl, dst_fapl, + H5O_COPY_EXPAND_SOFT_LINK_FLAG | H5O_COPY_EXPAND_EXT_LINK_FLAG, FALSE, + "H5Ocopy(): expand soft and external links"); + } + nerrors += test_copy_option(fcpl_src, fcpl_dst, src_fapl, dst_fapl, H5O_COPY_SHALLOW_HIERARCHY_FLAG, FALSE, "H5Ocopy(): shallow group copy"); nerrors += test_copy_option(fcpl_src, fcpl_dst, src_fapl, dst_fapl, H5O_COPY_EXPAND_REFERENCE_FLAG, @@ -17548,9 +17558,14 @@ main(void) nerrors += test_copy_same_file_named_datatype(fcpl_src, src_fapl); - /* Test with dataset opened in the file or not */ - nerrors += test_copy_old_layout(fcpl_dst, dst_fapl, FALSE); - nerrors += test_copy_old_layout(fcpl_dst, dst_fapl, TRUE); + /* Check if current driver might modify the filename. Skip these tests + * if so, since the file is pre-generated. + */ + if (!h5_driver_uses_modified_filename()) { + /* Test with dataset opened in the file or not */ + nerrors += test_copy_old_layout(fcpl_dst, dst_fapl, FALSE); + nerrors += test_copy_old_layout(fcpl_dst, dst_fapl, TRUE); + } /* Test with dataset opened in the file or not */ nerrors += test_copy_null_ref(fcpl_src, fcpl_dst, src_fapl, dst_fapl); diff --git a/test/ohdr.c b/test/ohdr.c index d28b11e..c7a13fa 100644 --- a/test/ohdr.c +++ b/test/ohdr.c @@ -34,6 +34,13 @@ #define H5G_FRIEND /* suppress error about including H5Gpkg */ #include "H5Gpkg.h" +/* + * This file needs to access private routines from the H5FD package. + */ +#define H5FD_FRIEND /* suppress error about including H5FDpkg */ +#define H5FD_TESTING +#include "H5FDpkg.h" + const char *FILENAME[] = {"ohdr", "ohdr_min_a", "ohdr_min_b", NULL}; /* used for object header size comparison */ @@ -584,7 +591,7 @@ test_unknown(unsigned bogus_id, char *filename, hid_t fapl) TESTING("object with unknown header message & 'shareable' flag set"); - /* Open the dataset with the unknown header message, adn "shareable" flag */ + /* Open the dataset with the unknown header message, and "shareable" flag */ if ((did = H5Dopen2(loc_bogus, "Dataset5", H5P_DEFAULT)) < 0) FAIL_STACK_ERROR if (H5Dclose(did) < 0) @@ -1089,13 +1096,13 @@ test_minimized_dset_ohdr_size_comparisons(hid_t fapl_id) /* IDs for non-minimized file open */ hid_t file_f_id = -1; /* lower 'f' for standard file setting */ hid_t dset_f_x_id = -1; /* 'x' for default */ - hid_t dset_f_N_id = -1; /* 'N' for explcit non-minimized dset */ + hid_t dset_f_N_id = -1; /* 'N' for explicit non-minimized dset */ hid_t dset_f_Y_id = -1; /* 'Y' for minimized dset */ /* IDs for minimized file open */ hid_t file_F_id = -1; /* upper 'F' for minimized file setting */ hid_t dset_F_x_id = -1; /* 'x' for default */ - hid_t dset_F_N_id = -1; /* 'N' for explcit non-minimized dset */ + hid_t dset_F_N_id = -1; /* 'N' for explicit non-minimized dset */ hid_t dset_F_Y_id = -1; /* 'Y' for minimized dset */ char filename_a[512] = ""; @@ -1813,13 +1820,12 @@ main(void) herr_t ret; /* Generic return value */ /* Get the VFD to use */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; /* Check for VFD which stores data in multiple files */ - single_file_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && - HDstrcmp(env_h5_drvr, "family") != 0); + single_file_vfd = !h5_driver_uses_multiple_files(env_h5_drvr, 0); /* Reset library */ h5_reset(); @@ -2107,11 +2113,15 @@ main(void) if (h5_verify_cached_stabs(FILENAME, fapl) < 0) TEST_ERROR - /* A test to exercise the re-read of the object header for SWMR access */ - if (test_ohdr_swmr(TRUE) < 0) - TEST_ERROR - if (test_ohdr_swmr(FALSE) < 0) - TEST_ERROR + if (H5FD__supports_swmr_test(env_h5_drvr)) { + /* A test to exercise the re-read of the object header for SWMR access */ + if (test_ohdr_swmr(TRUE) < 0) + TEST_ERROR + if (test_ohdr_swmr(FALSE) < 0) + TEST_ERROR + } + else + HDputs("Skipped SWMR tests for SWMR-incompatible VFD"); /* Pop API context */ if (api_ctx_pushed && H5CX_pop(FALSE) < 0) diff --git a/test/page_buffer.c b/test/page_buffer.c index c7bb965..c3555aa 100644 --- a/test/page_buffer.c +++ b/test/page_buffer.c @@ -186,7 +186,7 @@ error: * Any data mis-matches or failures reported by the HDF5 * library result in test failure. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: unknown @@ -326,7 +326,7 @@ error: * Any data mis-matches or failures reported by the HDF5 * library result in test failure. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: unknown @@ -486,7 +486,7 @@ error: * Any data mis-matches or unexpected failures or successes * reported by the HDF5 library result in test failure. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: unknown @@ -666,7 +666,7 @@ error: * check that it appears *immediately* after max_lag + 1 * ticks, since the LRU list does not hold onto MPMDEs. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: David Young @@ -811,7 +811,7 @@ error: * XXX reduce duplication with test_spmde_delay_basic! * XXX * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: David Young @@ -989,7 +989,7 @@ error: * XXX reduce duplication with test_mpmde_delay_basic! * XXX * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: David Young @@ -1118,7 +1118,7 @@ error: * Any data mis-matches or failures reported by the HDF5 * library result in test failure. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: David Young @@ -1453,7 +1453,7 @@ error: * Any data mis-matches or failures reported by the HDF5 * library result in test failure. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: unknown @@ -1706,7 +1706,7 @@ error: * Any data mis-matches or failures reported by the HDF5 * library result in test failure. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: unknown @@ -1725,7 +1725,7 @@ error: * * JRM -- 2/23/17 * - * Reworked test for new implementatin of the page buffer. + * Reworked test for new implementation of the page buffer. * The major change was adapting the test for the new * page buffers refusal to buffer any raw data when * min_md_pages == max_pages, or any metadata pages wwhen @@ -2126,7 +2126,7 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if (f->shared->page_buf->curr_rd_pages != 2) TEST_ERROR; - /* adding more meta entires should replace meta entries since raw data + /* adding more meta entries should replace meta entries since raw data * is at its minimum */ if (H5F_block_write(f, H5FD_MEM_SUPER, meta_addr + (sizeof(int) * 600), sizeof(int) * 100, data) < 0) @@ -2141,7 +2141,7 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if (f->shared->page_buf->curr_rd_pages != 2) TEST_ERROR; - /* bring existing raw entires up the LRU */ + /* bring existing raw entries up the LRU */ if (H5F_block_read(f, H5FD_MEM_DRAW, raw_addr + (sizeof(int) * 750), sizeof(int) * 100, data) < 0) FAIL_STACK_ERROR; @@ -2180,7 +2180,7 @@ test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) if (f->shared->page_buf->curr_rd_pages != 3) TEST_ERROR; - /* adding 2 meta entries should replace 2 entires at the bottom + /* adding 2 meta entries should replace 2 entries at the bottom * of the LRU */ if (H5F_block_read(f, H5FD_MEM_SUPER, meta_addr + (sizeof(int) * 98), sizeof(int) * 100, data) < 0) @@ -2368,7 +2368,7 @@ error: * Any data mis-matches or failures reported by the HDF5 * library result in test failure. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: unknown @@ -2715,7 +2715,7 @@ error: * At present, page buffering should be disabled in parallel * builds. Verify this. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: John Mainzer @@ -2883,7 +2883,7 @@ error: * though it is not generating the I/O requests in this * test. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: John Mainzer @@ -2956,8 +2956,8 @@ md_entry_splitting_smoke_check(hid_t orig_fapl, const char *env_h5_drvr, bool vf * from the free space manager, and perform several reads and writes to * verify expected behaviour. * - * The synthetic set of metadata entries are constucted of integers - * so as to allow easy assignement of unique values. It is constructed + * The synthetic set of metadata entries are constructed of integers + * so as to allow easy assignment of unique values. It is constructed * as follows: * * size values: addr: @@ -3178,7 +3178,7 @@ error: * split correctly under a variety of conditions that * are unlikely unless the user chooses at odd page size. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: John Mainzer @@ -3251,7 +3251,7 @@ md_entry_splitting_boundary_test(hid_t orig_fapl, const char *env_h5_drvr, bool if (base_page_cnt != 1) TEST_ERROR; - /* Test the folowing cases: + /* Test the following cases: * * 1) splittable md entry that is page aligned and exactly one * page long. @@ -3324,7 +3324,7 @@ md_entry_splitting_boundary_test(hid_t orig_fapl, const char *env_h5_drvr, bool * should log 1 metadata insertions * * Note that exposes an inefficiency in the page buffer, as page - * aligned I/O requests of exactly oen page in length really should + * aligned I/O requests of exactly one page in length really should * bypass the page buffer. * * This should be fixed, but I am bypassing it for now. @@ -4096,7 +4096,7 @@ error: * * Purpose: Main function for the page buffer tests. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: unknown diff --git a/test/reserved.c b/test/reserved.c index 6e41a16..08e747e 100644 --- a/test/reserved.c +++ b/test/reserved.c @@ -71,7 +71,7 @@ rsrv_heap(void) } H5E_END_TRY - HDsprintf(dset_name, "Dset %d", i); + HDsnprintf(dset_name, sizeof(dset_name), "Dset %d", i); H5E_BEGIN_TRY { @@ -115,7 +115,7 @@ rsrv_heap(void) if (H5open() < 0) TEST_ERROR; - HDsprintf(dset_name, "Dset %d", i - 2); + HDsnprintf(dset_name, sizeof(dset_name), "Dset %d", i - 2); file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); if (file_id < 0) @@ -218,7 +218,7 @@ rsrv_ohdr(void) } /* end for */ for (i = 0; i < 2000; i++) { - HDsprintf(attrname, "attr %d", i); + HDsnprintf(attrname, sizeof(attrname), "attr %d", i); H5E_BEGIN_TRY { aid = H5Screate_simple(2, dims, NULL); @@ -489,7 +489,7 @@ main(void) hid_t fapl; const char *envval = NULL; - envval = HDgetenv("HDF5_DRIVER"); + envval = HDgetenv(HDF5_DRIVER); if (envval == NULL) envval = "nomatch"; /* QAK: should be able to use the core driver? */ diff --git a/test/ros3.c b/test/ros3.c index b7b27ed..a340985 100644 --- a/test/ros3.c +++ b/test/ros3.c @@ -1763,7 +1763,7 @@ main(void) bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL"); if (bucket_url_env == NULL || bucket_url_env[0] == '\0') { - HDprintf("WARNING: S3 bucket url is not defined in enviornment " + HDprintf("WARNING: S3 bucket url is not defined in environment " "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n"); } else { diff --git a/test/s3comms.c b/test/s3comms.c index cec7a27..45415cb 100644 --- a/test/s3comms.c +++ b/test/s3comms.c @@ -244,7 +244,7 @@ jserr_str(const char *expected, const char *actual, const char *reason) } #ifdef JSVERIFY_EXP_ACT -/* VERIFY rountines with paramter order (, [, ]) +/* VERIFY routines with parameter order (, [, ]) */ /*---------------------------------------------------------------------------- @@ -435,7 +435,7 @@ error: * * Purpose: * - * Demonstrate the construction of a Canoncial Request (and Signed Headers) + * Demonstrate the construction of a Canonical Request (and Signed Headers) * * Elided / not yet implemented: * Query strings @@ -730,7 +730,7 @@ test_hrb_init_request(void) FALSE, }, { - "null verb substitues to GET", + "null verb substitutes to GET", NULL, "/MYPATH/MYFILE.tiff", "/MYPATH/MYFILE.tiff", @@ -1143,10 +1143,10 @@ test_hrb_node_set(void) *********/ for (mock_i = 0; test->given[mock_i] != NULL; mock_i += 2) { - const char *name = test->given[mock_i]; - const char *valu = test->given[mock_i + 1]; + const char *name = test->given[mock_i]; + const char *value = test->given[mock_i + 1]; - FAIL_IF(SUCCEED != H5FD_s3comms_hrb_node_set(&list, name, valu)) + FAIL_IF(SUCCEED != H5FD_s3comms_hrb_node_set(&list, name, value)) } /******** * TEST * @@ -1162,11 +1162,11 @@ test_hrb_node_set(void) node = list; mock_i = 0; while (test->expected[mock_i] != NULL && node != NULL) { - const char *name = test->expected[mock_i]; - const char *valu = test->expected[mock_i + 1]; + const char *name = test->expected[mock_i]; + const char *value = test->expected[mock_i + 1]; JSVERIFY_STR(name, node->name, NULL) - JSVERIFY_STR(valu, node->value, NULL) + JSVERIFY_STR(value, node->value, NULL) mock_i += 2; node = node->next; @@ -1559,7 +1559,7 @@ test_parse_url(void) NULL, "a=b&d=b", }, - "QUERY with implict PATH", + "QUERY with implicit PATH", }, { "http://[5]/path?a=b&d=b", @@ -2014,7 +2014,7 @@ error: * H5FD_s3comms_s3r_read << called by getsize(), multiple times working * H5FD_s3comms_s3r_close * - * Shows most basic curl interation. + * Shows most basic curl iteration. * * Programmer: Jacob Smith * 2017-10-06 @@ -2610,7 +2610,7 @@ main(void) bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL"); if (bucket_url_env == NULL || bucket_url_env[0] == '\0') { - HDprintf("WARNING: S3 bucket url is not defined in enviornment " + HDprintf("WARNING: S3 bucket url is not defined in environment " "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n"); } else { @@ -2618,7 +2618,7 @@ main(void) s3_test_bucket_defined = TRUE; } - /* tests ordered rougly by dependence */ + /* tests ordered roughly by dependence */ nerrors += test_macro_format_credential() < 0 ? 1 : 0; nerrors += test_trim() < 0 ? 1 : 0; nerrors += test_nlowercase() < 0 ? 1 : 0; diff --git a/test/set_extent.c b/test/set_extent.c index 3078c2d..4437219 100644 --- a/test/set_extent.c +++ b/test/set_extent.c @@ -115,12 +115,12 @@ main(void) unsigned chunk_cache; /* Whether to enable chunk caching */ int nerrors = 0; const char *env_h5_drvr; /* File Driver value from environment */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; - /* Current VFD that does not support contigous address space */ + /* Current VFD that does not support contiguous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); /* Initialize random number seed */ @@ -343,35 +343,42 @@ do_ranks(hid_t fapl, hbool_t new_format) goto error; } /* end if */ - /* VL test */ - if (test_random_rank4_vl(fapl, dcpl, do_fillvalue, disable_edge_filters, FALSE, index_type) < 0) { - DO_RANKS_PRINT_CONFIG("Randomized rank 4 variable length") - HDprintf(" Index: %s\n", index_type == RANK4_INDEX_BTREE - ? "btree" - : (index_type == RANK4_INDEX_FARRAY ? "farray" : "earray")); - goto error; - } /* end if */ - - /* Sparse allocation test (regular and VL) */ - if (!(config & CONFIG_EARLY_ALLOC)) { - if (test_random_rank4(fapl, dcpl, do_fillvalue, disable_edge_filters, TRUE, index_type) < 0) { - DO_RANKS_PRINT_CONFIG("Randomized rank 4 with sparse allocation") + if (!h5_using_parallel_driver(NULL)) { + /* VL test */ + if (test_random_rank4_vl(fapl, dcpl, do_fillvalue, disable_edge_filters, FALSE, index_type) < + 0) { + DO_RANKS_PRINT_CONFIG("Randomized rank 4 variable length") HDprintf(" Index: %s\n", index_type == RANK4_INDEX_BTREE ? "btree" : (index_type == RANK4_INDEX_FARRAY ? "farray" : "earray")); goto error; } /* end if */ - if (test_random_rank4_vl(fapl, dcpl, do_fillvalue, disable_edge_filters, TRUE, index_type) < - 0) { - DO_RANKS_PRINT_CONFIG("Randomized rank 4 variable length with sparse allocation") + } + + /* Sparse allocation test (regular and VL) */ + if (!(config & CONFIG_EARLY_ALLOC)) { + if (test_random_rank4(fapl, dcpl, do_fillvalue, disable_edge_filters, TRUE, index_type) < 0) { + DO_RANKS_PRINT_CONFIG("Randomized rank 4 with sparse allocation") HDprintf(" Index: %s\n", index_type == RANK4_INDEX_BTREE ? "btree" : (index_type == RANK4_INDEX_FARRAY ? "farray" : "earray")); goto error; } /* end if */ - } /* end if */ + + if (!h5_using_parallel_driver(NULL)) { + if (test_random_rank4_vl(fapl, dcpl, do_fillvalue, disable_edge_filters, TRUE, + index_type) < 0) { + DO_RANKS_PRINT_CONFIG("Randomized rank 4 variable length with sparse allocation") + HDprintf(" Index: %s\n", + index_type == RANK4_INDEX_BTREE + ? "btree" + : (index_type == RANK4_INDEX_FARRAY ? "farray" : "earray")); + goto error; + } /* end if */ + } + } /* end if */ /* Break out if using the old format */ if (!new_format) @@ -649,7 +656,7 @@ test_rank1(hid_t fapl, hid_t dcpl, hbool_t do_fill_value, hbool_t disable_edge_f HDprintf("\n"); #endif - /* compare the read array with the shrinked array */ + /* compare the read array with the shrunk array */ for (i = 0; i < (int)dims_r[0]; i++) if (buf_s[i] != buf_o[i]) { HDprintf("buf_s[%d] = %d\n", i, buf_s[i]); @@ -1055,7 +1062,7 @@ test_rank2(hid_t fapl, hid_t dcpl, hbool_t do_fill_value, hbool_t disable_edge_f } #endif - /* compare the read array with the shrinked array */ + /* compare the read array with the shrunk array */ for (i = 0; i < (int)dims_r[0]; i++) { for (j = 0; j < (int)dims_r[1]; j++) { if (buf_s[i][j] != buf_o[i][j]) { @@ -1536,7 +1543,7 @@ test_rank3(hid_t fapl, hid_t dcpl, hbool_t do_fill_value, hbool_t disable_edge_f HDprintf("\n"); #endif - /* compare the read array with the shrinked array */ + /* compare the read array with the shrunk array */ for (i = 0; i < (int)dims_r[0]; i++) { for (j = 0; j < (int)dims_r[1]; j++) { for (k = 0; k < (int)dims_r[2]; k++) { @@ -1734,7 +1741,7 @@ test_external(hid_t fapl) hsize_t dims_r[RANK2]; /* read dimensions */ hsize_t maxdims[RANK2] = {DIME0, DIM1}; /* only the first dimension can be extendible */ int buf_o[DIM0][DIM1]; /* original buffer, for writing */ - int buf_s[DIMS0][DIMS1]; /* shrinked buffer, for reading */ + int buf_s[DIMS0][DIMS1]; /* shrunk buffer, for reading */ int buf_e[DIME0][DIM1]; /* extended buffer, for writing, dimension 1 is the original */ int buf_ro[DIM0][DIM1]; /* original buffer for reading */ int i, j; @@ -1946,7 +1953,7 @@ test_external(hid_t fapl) } #endif - /* compare the read array with the shrinked array */ + /* compare the read array with the shrunk array */ for (i = 0; i < (int)dims_r[0]; i++) { for (j = 0; j < (int)dims_r[1]; j++) { if (buf_s[i][j] != buf_o[i][j]) { diff --git a/test/stab.c b/test/stab.c index 798619d..b3db76e 100644 --- a/test/stab.c +++ b/test/stab.c @@ -125,7 +125,7 @@ test_misc(hid_t fcpl, hid_t fapl, hbool_t new_format) if (H5Gclose(g3) < 0) TEST_ERROR - /* Open all groups with absolute names to check for exsistence */ + /* Open all groups with absolute names to check for existence */ if ((g1 = H5Gopen2(fid, "/test_1a", H5P_DEFAULT)) < 0) TEST_ERROR if ((g2 = H5Gopen2(fid, "/test_1a/sub_1", H5P_DEFAULT)) < 0) @@ -1416,15 +1416,16 @@ main(void) hid_t fcpl, fcpl2; /* File creation property list ID */ unsigned new_format; /* Whether to use the new format or not */ const char *env_h5_drvr; /* File Driver value from environment */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ - int nerrors = 0; + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ + hbool_t driver_uses_modified_filename = h5_driver_uses_modified_filename(); + int nerrors = 0; /* Get the VFD to use */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; - /* VFD that does not support contigous address space */ + /* VFD that does not support contiguous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0); /* Reset library */ @@ -1473,14 +1474,21 @@ main(void) if (contig_addr_vfd) { nerrors += lifecycle(fcpl2, fapl2); nerrors += long_compact(fcpl2, fapl2); - nerrors += read_old(); + + if (!driver_uses_modified_filename) { + nerrors += read_old(); + } + nerrors += no_compact(fcpl2, fapl2); nerrors += gcpl_on_root(fapl2); } /* Old group API specific tests */ nerrors += old_api(fapl); - nerrors += corrupt_stab_msg(); + + if (!driver_uses_modified_filename) { + nerrors += corrupt_stab_msg(); + } /* Close 2nd FAPL */ H5Pclose(fapl2); diff --git a/test/swmr.c b/test/swmr.c index 9091fe2..dccd945 100644 --- a/test/swmr.c +++ b/test/swmr.c @@ -2395,7 +2395,7 @@ error: * (5) Parent: open a file with write access; enable SWMR writing mode * Child: concurrent open of the file with write and SWMR write access (fail) */ -#ifndef H5_HAVE_UNISTD_H +#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) static int test_start_swmr_write_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t new_format) @@ -2408,11 +2408,11 @@ test_start_swmr_write_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t new_format) } SKIPPED(); - HDputs(" Test skipped due to a lack of unistd.h functionality."); + HDputs(" Test skipped due to fork or waitpid not defined."); return 0; } /* test_start_swmr_write_concur() */ -#else /* H5_HAVE_UNISTD_H */ +#else /* !defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID) */ static int test_start_swmr_write_concur(hid_t in_fapl, hbool_t new_format) @@ -3014,7 +3014,7 @@ error: return -1; } /* test_start_swmr_write_concur() */ -#endif /* H5_HAVE_UNISTD_H */ +#endif /* !defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID) */ /* * test_start_swmr_write_stress_ohdr(): @@ -3666,7 +3666,7 @@ test_append_flush_dataset_chunked(hid_t in_fapl) { hid_t fid = -1; /* file ID */ hid_t fapl = -1; /* A copy of file access property */ - hid_t did1 = -1, did2 = -1; /* The datset ID */ + hid_t did1 = -1, did2 = -1; /* The dataset ID */ hid_t sid = -1; /* The dataspace ID */ hid_t dcpl = -1; /* A copy of dataset creation property */ hid_t dapl = -1; /* A copy of dataset access property */ @@ -3889,7 +3889,7 @@ test_append_flush_dataset_fixed(hid_t in_fapl) { hid_t fid = -1; /* file ID */ hid_t fapl = -1; /* A copy of file access property */ - hid_t did1 = -1, did2 = -1; /* The datset ID */ + hid_t did1 = -1, did2 = -1; /* The dataset ID */ hid_t sid = -1; /* The dataspace ID */ hid_t dapl = -1; /* A copy of dataset access property */ hid_t ddapl = -1; /* The dataset access property of the opened dataset */ @@ -4107,7 +4107,7 @@ test_append_flush_dataset_multiple(hid_t in_fapl) { hid_t fid = -1; /* file ID */ hid_t fapl = -1; /* A copy of file access property */ - hid_t did1 = -1, did2 = -1; /* The datset ID */ + hid_t did1 = -1, did2 = -1; /* The dataset ID */ hid_t sid = -1; /* The dataspace ID */ hid_t dcpl = -1; /* A copy of dataset creation property */ hid_t dapl1 = -1; /* A copy of dataset access property */ @@ -4809,7 +4809,7 @@ error: ** This is for concurrent access. ** *****************************************************************/ -#ifndef H5_HAVE_UNISTD_H +#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK)) static int test_file_lock_concur(hid_t H5_ATTR_UNUSED in_fapl) @@ -4817,12 +4817,12 @@ test_file_lock_concur(hid_t H5_ATTR_UNUSED in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--concurrent access"); SKIPPED(); - HDputs(" Test skipped due to a lack of unistd.h functionality."); + HDputs(" Test skipped due to fork or waitpid not defined."); return 0; } /* end test_file_lock_concur() */ -#else /* H5_HAVE_UNISTD_H */ +#else /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK)) */ static int test_file_lock_concur(hid_t in_fapl) @@ -5192,7 +5192,7 @@ error: } /* end test_file_lock_concur() */ -#endif /* H5_HAVE_UNISTD_H */ +#endif /* #if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK)) */ /**************************************************************** ** @@ -5202,7 +5202,7 @@ error: ** This is for concurrent access. ** *****************************************************************/ -#ifndef H5_HAVE_UNISTD_H +#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) static int test_file_lock_swmr_concur(hid_t H5_ATTR_UNUSED in_fapl) @@ -5210,12 +5210,12 @@ test_file_lock_swmr_concur(hid_t H5_ATTR_UNUSED in_fapl) /* Output message about test being performed */ TESTING("File open with different combintations of flags + SWMR flags--concurrent access"); SKIPPED(); - HDputs(" Test skipped due to a lack of unistd.h functionality."); + HDputs(" Test skipped due to fork or waitpid not defined."); return 0; } /* end test_file_lock_swmr_concur() */ -#else /* H5_HAVE_UNISTD_H */ +#else /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ static int test_file_lock_swmr_concur(hid_t in_fapl) @@ -6215,7 +6215,7 @@ error: } /* end test_file_lock_swmr_concur() */ -#endif /* H5_HAVE_UNISTD_H */ +#endif /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ /**************************************************************** ** @@ -6227,7 +6227,7 @@ error: static int test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override) { -#ifndef H5_HAVE_UNISTD_H +#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) if (turn_locking_on && env_var_override) TESTING("File locking: ON w/ env var override") else if (turn_locking_on && !env_var_override) @@ -6237,9 +6237,9 @@ test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_overri else TESTING("File locking: OFF") SKIPPED(); - HDputs(" Test skipped due to a lack of unistd.h functionality."); + HDputs(" Test skipped due to fork or waitpid not defined."); return 0; -#else /* H5_HAVE_UNISTD_H */ +#else /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ hid_t fid = -1; /* File ID */ hid_t fapl = -1; /* File access property list */ char filename[NAME_BUF_SIZE]; /* file name */ @@ -6270,13 +6270,13 @@ test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_overri /* If requested, set the environment variable */ if (env_var_override) { - if (HDsetenv("HDF5_USE_FILE_LOCKING", turn_locking_on ? "FALSE" : "TRUE", TRUE) < 0) + if (HDsetenv(HDF5_USE_FILE_LOCKING, turn_locking_on ? "FALSE" : "TRUE", TRUE) < 0) TEST_ERROR if (H5F__reparse_file_lock_variable_test() < 0) TEST_ERROR } else { - if (HDsetenv("HDF5_USE_FILE_LOCKING", "", TRUE) < 0) + if (HDsetenv(HDF5_USE_FILE_LOCKING, "", TRUE) < 0) TEST_ERROR if (H5F__reparse_file_lock_variable_test() < 0) TEST_ERROR @@ -6405,7 +6405,7 @@ error: return -1; -#endif /* H5_HAVE_UNISTD_H */ +#endif /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ } /* end test_file_locking() */ @@ -6699,7 +6699,7 @@ error: * (7) Refresh the dataset * (8) Verify the dataset's dimension and data are correct */ -#ifndef H5_HAVE_UNISTD_H +#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) static int test_refresh_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t new_format) @@ -6712,11 +6712,11 @@ test_refresh_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t new_format) } SKIPPED(); - HDputs(" Test skipped due to a lack of unistd.h functionality."); + HDputs(" Test skipped due to fork or waitpid not defined."); return 0; } /* test_refresh_concur() */ -#else /* H5_HAVE_UNISTD_H */ +#else /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ static int test_refresh_concur(hid_t in_fapl, hbool_t new_format) @@ -7016,7 +7016,7 @@ error: return -1; } /* test_refresh_concur() */ -#endif /* H5_HAVE_UNISTD_H */ +#endif /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ /* * test_multiple_same(): @@ -7367,7 +7367,7 @@ main(void) /* Skip this test if SWMR I/O is not supported for the VFD specified * by the environment variable. */ - driver = HDgetenv("HDF5_DRIVER"); + driver = HDgetenv(HDF5_DRIVER); if (!H5FD__supports_swmr_test(driver)) { HDprintf("This VFD does not support SWMR I/O\n"); return EXIT_SUCCESS; @@ -7377,7 +7377,7 @@ main(void) * about file locking. File locking should be used unless explicitly * disabled. */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + lock_env_var = HDgetenv(HDF5_USE_FILE_LOCKING); if (lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) use_file_locking = FALSE; else @@ -7412,7 +7412,7 @@ main(void) /* * Modify the following routines to test for files: * H5Fcreate(write, latest format) or H5Fcreate(SWMR write, non-latest-format) - * --both result in v3 superblock and latest version suppport + * --both result in v3 superblock and latest version support */ nerrors += test_start_swmr_write(fapl, TRUE); nerrors += test_start_swmr_write(fapl, FALSE); @@ -7446,7 +7446,7 @@ main(void) * * Modify the following 2 routines to test for files: * H5Fcreate(write, latest format) or H5Fcreate(SWMR write, non-latest-format) - * --both result in v3 superblock and latest version suppport + * --both result in v3 superblock and latest version support */ nerrors += test_file_lock_swmr_same(fapl); nerrors += test_file_lock_swmr_concur(fapl); diff --git a/test/swmr_addrem_writer.c b/test/swmr_addrem_writer.c index 5f0bba0..faf03dc 100644 --- a/test/swmr_addrem_writer.c +++ b/test/swmr_addrem_writer.c @@ -283,7 +283,7 @@ usage(void) } int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid; /* File ID for file opened */ long nops = 0; /* # of times to grow or shrink the dataset */ diff --git a/test/swmr_check_compat_vfd.c b/test/swmr_check_compat_vfd.c deleted file mode 100644 index 4133696..0000000 --- a/test/swmr_check_compat_vfd.c +++ /dev/null @@ -1,53 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* Purpose: This is a small program that checks if the HDF5_DRIVER - * environment variable is set to a value that supports SWMR. - * - * It is intended for use in shell scripts. - */ - -#include "h5test.h" - -/* This file needs to access the file driver testing code */ -#define H5FD_FRIEND /*suppress error about including H5FDpkg */ -#define H5FD_TESTING -#include "H5FDpkg.h" /* File drivers */ - -/*------------------------------------------------------------------------- - * Function: main - * - * Purpose: Inspects the HDF5_DRIVER environment variable, which - * determines the VFD that the test harness will use with - * the majority of the tests. - * - * Return: VFD supports SWMR: EXIT_SUCCESS - * - * VFD does not support SWMR - * or failure: EXIT_FAILURE - * - *------------------------------------------------------------------------- - */ -int -main(void) -{ - char *driver = NULL; - - driver = HDgetenv("HDF5_DRIVER"); - - if (H5FD__supports_swmr_test(driver)) - return EXIT_SUCCESS; - else - return EXIT_FAILURE; - -} /* end main() */ diff --git a/test/swmr_generator.c b/test/swmr_generator.c index 2bff12c..93cfa0f 100644 --- a/test/swmr_generator.c +++ b/test/swmr_generator.c @@ -221,7 +221,7 @@ gen_skeleton(const char *filename, hbool_t verbose, hbool_t swmr_write, int comp if (verbose) HDfprintf(stderr, "Closing objects\n"); - /* Close everythign */ + /* Close everything */ if (H5Pclose(dcpl) < 0) return -1; if (H5Sclose(sid) < 0) @@ -258,7 +258,7 @@ usage(void) } /* end usage() */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { int comp_level = -1; /* Compression level (-1 is no compression) */ hbool_t verbose = TRUE; /* Whether to emit some informational messages */ diff --git a/test/swmr_reader.c b/test/swmr_reader.c index cc16b92..a931cf7 100644 --- a/test/swmr_reader.c +++ b/test/swmr_reader.c @@ -387,7 +387,7 @@ usage(void) } int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { long nseconds = 0; /* # of seconds to test */ int poll_time = 1; /* # of seconds between polling */ diff --git a/test/swmr_remove_reader.c b/test/swmr_remove_reader.c index 0850b0b..9be602e 100644 --- a/test/swmr_remove_reader.c +++ b/test/swmr_remove_reader.c @@ -371,7 +371,7 @@ usage(void) } int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { long nseconds = 0; /* # of seconds to test */ int poll_time = 1; /* # of seconds between polling */ diff --git a/test/swmr_remove_writer.c b/test/swmr_remove_writer.c index e209285..4b4bb7b 100644 --- a/test/swmr_remove_writer.c +++ b/test/swmr_remove_writer.c @@ -217,7 +217,7 @@ usage(void) } int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid; /* File ID for file opened */ long nshrinks = 0; /* # of times to shrink the dataset */ diff --git a/test/swmr_sparse_reader.c b/test/swmr_sparse_reader.c index 6c1bd19..4b8a36a 100644 --- a/test/swmr_sparse_reader.c +++ b/test/swmr_sparse_reader.c @@ -29,7 +29,6 @@ /* Headers */ /***********/ -#include /* for PRIu64 */ #include "h5test.h" #include "swmr_common.h" @@ -343,7 +342,7 @@ usage(void) } /* end usage() */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { long nrecords = 0; /* # of records to read */ int poll_time = 1; /* # of seconds to sleep when waiting for writer */ diff --git a/test/swmr_sparse_writer.c b/test/swmr_sparse_writer.c index 2b46bb2..a7904ef 100644 --- a/test/swmr_sparse_writer.c +++ b/test/swmr_sparse_writer.c @@ -318,7 +318,7 @@ usage(void) } int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid; /* File ID for file opened */ long nrecords = 0; /* # of records to append */ diff --git a/test/swmr_start_write.c b/test/swmr_start_write.c index 9c51402..dc12b18 100644 --- a/test/swmr_start_write.c +++ b/test/swmr_start_write.c @@ -125,7 +125,7 @@ create_file(const char *filename, hbool_t verbose, FILE *verbose_file, unsigned * * Parameters: * fid: file ID for the SWMR test file - * comp_level: the compresssion level + * comp_level: the compression level * index_type: The chunk index type (b1 | b2 | ea | fa) * verbose: whether verbose console output is desired. * verbose_file: file pointer for verbose output @@ -352,7 +352,7 @@ usage(void) *H5Fstart_swmr_write(), add_records(), H5Fclose(). */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid; /* File ID for file opened */ long nrecords = 0; /* # of records to append */ @@ -385,7 +385,7 @@ main(int argc, const char *argv[]) /* Chunk index type */ case 'i': index_type = argv[u + 1]; - if (HDstrcmp(index_type, "ea") && HDstrcmp(index_type, "b2")) + if (HDstrcmp(index_type, "ea") != 0 && HDstrcmp(index_type, "b2") != 0) usage(); u += 2; break; diff --git a/test/swmr_writer.c b/test/swmr_writer.c index 6743f45..c44343d 100644 --- a/test/swmr_writer.c +++ b/test/swmr_writer.c @@ -275,7 +275,7 @@ usage(void) } int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid; /* File ID for file opened */ long nrecords = 0; /* # of records to append */ diff --git a/test/tarray.c b/test/tarray.c index af167a7..3b17239 100644 --- a/test/tarray.c +++ b/test/tarray.c @@ -2217,8 +2217,10 @@ test_array(void) test_array_bkg(); /* Read compound datatype with array fields and background fields read */ - /* This test uses a custom file */ - test_compat(); /* Test compatibility changes for compound datatype fields */ + if (!h5_driver_uses_modified_filename()) { + /* This test uses a custom file */ + test_compat(); /* Test compatibility changes for compound datatype fields */ + } } /* end test_array() */ diff --git a/test/tattr.c b/test/tattr.c index 756e139..2859f4c 100644 --- a/test/tattr.c +++ b/test/tattr.c @@ -2094,7 +2094,7 @@ test_attr_dense_verify(hid_t loc_id, unsigned max_attr) /* Re-open all the attributes by name and verify the data */ for (u = 0; u < max_attr; u++) { /* Open attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Aopen(loc_id, attrname, H5P_DEFAULT); CHECK(attr, FAIL, "H5Aopen"); @@ -2119,7 +2119,7 @@ test_attr_dense_verify(hid_t loc_id, unsigned max_attr) CHECK(attr, FAIL, "H5Aopen_by_idx"); /* Verify Name */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); name_len = H5Aget_name(attr, (size_t)ATTR_NAME_LEN, check_name); VERIFY(name_len, HDstrlen(attrname), "H5Aget_name"); if (HDstrcmp(check_name, attrname) != 0) @@ -2219,7 +2219,7 @@ test_attr_dense_create(hid_t fcpl, hid_t fapl) /* Add attributes, until just before converting to dense storage */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -2238,7 +2238,7 @@ test_attr_dense_create(hid_t fcpl, hid_t fapl) /* Add one more attribute, to push into "dense" storage */ /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -2274,9 +2274,11 @@ test_attr_dense_create(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_dense_create() */ /**************************************************************** @@ -2359,7 +2361,7 @@ test_attr_dense_open(hid_t fcpl, hid_t fapl) /* Add attributes, until just before converting to dense storage */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -2382,7 +2384,7 @@ test_attr_dense_open(hid_t fcpl, hid_t fapl) /* Add one more attribute, to push into "dense" storage */ /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -2418,9 +2420,11 @@ test_attr_dense_open(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_dense_open() */ /**************************************************************** @@ -2448,6 +2452,10 @@ test_attr_dense_delete(hid_t fcpl, hid_t fapl) int use_min_dset_oh = (dcpl_g != H5P_DEFAULT); herr_t ret; /* Generic return value */ + /* Only run this test for sec2/default driver */ + if (!h5_using_default_driver(NULL)) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing Deleting Attributes in Dense Storage\n")); @@ -2515,7 +2523,7 @@ test_attr_dense_delete(hid_t fcpl, hid_t fapl) /* Add attributes, until well into dense storage */ for (u = 0; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -2560,7 +2568,7 @@ test_attr_dense_delete(hid_t fcpl, hid_t fapl) /* Delete attributes until the attributes revert to compact storage again */ for (u--; u >= min_dense; u--) { /* Delete attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -2574,7 +2582,7 @@ test_attr_dense_delete(hid_t fcpl, hid_t fapl) VERIFY(is_dense, TRUE, "H5O__is_attr_dense_test"); /* Delete one more attribute, which should cause reversion to compact storage */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -2587,7 +2595,7 @@ test_attr_dense_delete(hid_t fcpl, hid_t fapl) CHECK(ret, FAIL, "test_attr_dense_verify"); /* Delete another attribute, to verify deletion in compact storage */ - HDsprintf(attrname, "attr %02u", (u - 1)); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", (u - 1)); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -2611,9 +2619,11 @@ test_attr_dense_delete(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_dense_delete() */ /**************************************************************** @@ -2643,6 +2653,10 @@ test_attr_dense_rename(hid_t fcpl, hid_t fapl) unsigned use_corder; /* Track creation order or not */ herr_t ret; /* Generic return value */ + /* Only run this test for sec2/default driver */ + if (!h5_using_default_driver(NULL)) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing Renaming Attributes in Dense Storage\n")); @@ -2710,7 +2724,7 @@ test_attr_dense_rename(hid_t fcpl, hid_t fapl) /* Add attributes, until well into dense storage */ for (u = 0; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, H5I_INVALID_HID, "H5Acreate2"); @@ -2776,7 +2790,7 @@ test_attr_dense_rename(hid_t fcpl, hid_t fapl) unsigned value; /* Attribute value */ /* Open attribute */ - HDsprintf(attrname, "new attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "new attr %02u", u); attr = H5Aopen(dataset, attrname, H5P_DEFAULT); CHECK(attr, H5I_INVALID_HID, "H5Aopen"); @@ -2802,9 +2816,11 @@ test_attr_dense_rename(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_dense_rename() */ /**************************************************************** @@ -2833,6 +2849,10 @@ test_attr_dense_unlink(hid_t fcpl, hid_t fapl) int use_min_dset_oh = (dcpl_g != H5P_DEFAULT); herr_t ret; /* Generic return value */ + /* Only run this test for sec2/default driver */ + if (!h5_using_default_driver(NULL)) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing Unlinking Object with Attributes in Dense Storage\n")); @@ -2894,7 +2914,7 @@ test_attr_dense_unlink(hid_t fcpl, hid_t fapl) /* Add attributes, until well into dense storage */ for (u = 0; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -2945,9 +2965,11 @@ test_attr_dense_unlink(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_dense_unlink() */ /**************************************************************** @@ -3035,7 +3057,7 @@ test_attr_dense_limits(hid_t fcpl, hid_t fapl) /* Create attribute */ u = 0; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3055,7 +3077,7 @@ test_attr_dense_limits(hid_t fcpl, hid_t fapl) /* Create attribute */ u = 1; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3085,7 +3107,7 @@ test_attr_dense_limits(hid_t fcpl, hid_t fapl) /* Delete attribute */ u = 0; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -3109,9 +3131,11 @@ test_attr_dense_limits(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_dense_limits() */ /**************************************************************** @@ -3192,7 +3216,7 @@ test_attr_dense_dup_ids(hid_t fcpl, hid_t fapl) /* Add attributes, until just before converting to dense storage */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3215,7 +3239,7 @@ test_attr_dense_dup_ids(hid_t fcpl, hid_t fapl) CHECK(sid2, FAIL, "H5Screate_simple"); /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_INT, sid2, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3498,7 +3522,7 @@ test_attr_dense_dup_ids(hid_t fcpl, hid_t fapl) /* Delete a few attributes until the storage switches to compact */ for (u = min_dense - 1; u <= max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); add_attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(add_attr, FAIL, "H5Acreate2"); @@ -3566,7 +3590,7 @@ test_attr_dense_dup_ids(hid_t fcpl, hid_t fapl) /* Add attributes, until just before converting to dense storage */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(gid1, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3732,7 +3756,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Create attribute */ u = 0; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3750,7 +3774,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Create attribute */ u = 1; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3768,7 +3792,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Create attribute */ u = 2; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, big_sid, H5P_DEFAULT, H5P_DEFAULT); if (low == H5F_LIBVER_LATEST) { CHECK(attr, FAIL, "H5Acreate2"); @@ -3791,7 +3815,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Create attribute */ u = 3; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(dataset, attrname, H5T_NATIVE_UINT, big_sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -3813,7 +3837,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Delete attribute */ u = 1; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -3827,7 +3851,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Delete attribute */ u = 3; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -3841,7 +3865,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Delete attribute */ u = 2; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -3855,7 +3879,7 @@ test_attr_big(hid_t fcpl, hid_t fapl) /* Delete attribute */ u = 0; - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -3896,9 +3920,11 @@ test_attr_big(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_big() */ /**************************************************************** @@ -4105,9 +4131,11 @@ test_attr_null_space(hid_t fcpl, hid_t fapl) ret = H5Sclose(null_sid); CHECK(ret, FAIL, "H5Sclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* test_attr_null_space() */ /**************************************************************** @@ -4250,7 +4278,7 @@ test_attr_many(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create many attributes */ for (u = 0; u < nattr; u++) { - HDsprintf(attrname, "a-%06u", u); + HDsnprintf(attrname, sizeof(attrname), "a-%06u", u); exists = H5Aexists(gid, attrname); VERIFY(exists, FALSE, "H5Aexists"); @@ -4302,7 +4330,7 @@ test_attr_many(hbool_t new_format, hid_t fcpl, hid_t fapl) for (u = 0; u < nattr; u++) { unsigned value; /* Attribute value */ - HDsprintf(attrname, "a-%06u", u); + HDsnprintf(attrname, sizeof(attrname), "a-%06u", u); exists = H5Aexists(gid, attrname); VERIFY(exists, TRUE, "H5Aexists"); @@ -4550,7 +4578,7 @@ test_attr_corder_create_compact(hid_t fcpl, hid_t fapl) /* Create several attributes, but keep storage in compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -4639,7 +4667,7 @@ test_attr_corder_create_compact(hid_t fcpl, hid_t fapl) H5A_info_t ainfo; /* Attribute information */ /* Retrieve information for attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Aget_info_by_name(my_dataset, ".", attrname, &ainfo, H5P_DEFAULT); CHECK(ret, FAIL, "H5Aget_info_by_name"); @@ -4754,7 +4782,7 @@ test_attr_corder_create_dense(hid_t fcpl, hid_t fapl) /* Create several attributes, but keep storage in compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -4777,7 +4805,7 @@ test_attr_corder_create_dense(hid_t fcpl, hid_t fapl) } /* end for */ /* Create another attribute, to push into dense storage */ - HDsprintf(attrname, "attr %02u", max_compact); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", max_compact); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -4870,7 +4898,7 @@ test_attr_corder_create_dense(hid_t fcpl, hid_t fapl) H5A_info_t ainfo; /* Attribute information */ /* Retrieve information for attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Aget_info_by_name(my_dataset, ".", attrname, &ainfo, H5P_DEFAULT); CHECK(ret, FAIL, "H5Aget_info_by_name"); @@ -5143,7 +5171,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) /* Create several attributes, but keep storage in compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -5166,7 +5194,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) } /* end for */ /* Create another attribute, to push into dense storage */ - HDsprintf(attrname, "attr %02u", max_compact); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", max_compact); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -5194,7 +5222,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) /* Delete several attributes from object, until attribute storage resumes compact form */ for (u = max_compact; u >= min_dense; u--) { - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(my_dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -5214,7 +5242,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) } /* end for */ /* Delete another attribute, to push attribute storage into compact form */ - HDsprintf(attrname, "attr %02u", (min_dense - 1)); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", (min_dense - 1)); ret = H5Adelete(my_dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -5230,7 +5258,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) /* Re-add attributes to get back into dense form */ for (u = (min_dense - 1); u < (max_compact + 1); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -5317,7 +5345,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) /* Delete several attributes from object, until attribute storage resumes compact form */ for (u = max_compact; u >= min_dense; u--) { - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(my_dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -5337,7 +5365,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) } /* end for */ /* Delete another attribute, to push attribute storage into compact form */ - HDsprintf(attrname, "attr %02u", (min_dense - 1)); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", (min_dense - 1)); ret = H5Adelete(my_dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); @@ -5353,7 +5381,7 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) /* Re-add attributes to get back into dense form */ for (u = (min_dense - 1); u < (max_compact + 1); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -5382,11 +5410,11 @@ test_attr_corder_transition(hid_t fcpl, hid_t fapl) /* Delete all attributes */ for (u = max_compact; u > 0; u--) { - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); ret = H5Adelete(my_dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); } /* end for */ - HDsprintf(attrname, "attr %02u", 0); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", 0); ret = H5Adelete(my_dataset, attrname); CHECK(ret, FAIL, "H5Adelete"); } /* end for */ @@ -5526,7 +5554,7 @@ test_attr_corder_delete(hid_t fcpl, hid_t fapl) /* Create attributes, until attribute storage is in dense form */ for (u = 0; u < max_compact * 2; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -5858,7 +5886,7 @@ test_attr_info_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) is_dense = H5O__is_attr_dense_test(my_dataset); VERIFY(is_dense, FALSE, "H5O__is_attr_dense_test"); - /* Check for query on non-existant attribute */ + /* Check for query on non-existent attribute */ ret = H5Aget_info_by_idx(my_dataset, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t)0, &ainfo, H5P_DEFAULT); VERIFY(ret, FAIL, "H5Aget_info_by_idx"); @@ -5869,7 +5897,7 @@ test_attr_info_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create attributes, up to limit of compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -5909,7 +5937,7 @@ test_attr_info_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create more attributes, to push into dense form */ for (; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -6346,14 +6374,14 @@ test_attr_delete_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) is_dense = H5O__is_attr_dense_test(my_dataset); VERIFY(is_dense, FALSE, "H5O__is_attr_dense_test"); - /* Check for deleting non-existant attribute */ + /* Check for deleting non-existent attribute */ ret = H5Adelete_by_idx(my_dataset, ".", idx_type, order, (hsize_t)0, H5P_DEFAULT); VERIFY(ret, FAIL, "H5Adelete_by_idx"); /* Create attributes, up to limit of compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -6428,9 +6456,9 @@ test_attr_delete_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) ret = (herr_t)H5Aget_name_by_idx(my_dataset, ".", idx_type, order, (hsize_t)0, tmpname, (size_t)NAME_BUF_SIZE, H5P_DEFAULT); if (order == H5_ITER_INC) - HDsprintf(attrname, "attr %02u", (u + 1)); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", (u + 1)); else - HDsprintf(attrname, "attr %02u", (max_compact - (u + 2))); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", (max_compact - (u + 2))); ret = HDstrcmp(attrname, tmpname); VERIFY(ret, 0, "H5Aget_name_by_idx"); } /* end for */ @@ -6466,7 +6494,7 @@ test_attr_delete_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create more attributes, to push into dense form */ for (u = 0; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -6556,9 +6584,10 @@ test_attr_delete_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) ret = (herr_t)H5Aget_name_by_idx(my_dataset, ".", idx_type, order, (hsize_t)0, tmpname, (size_t)NAME_BUF_SIZE, H5P_DEFAULT); if (order == H5_ITER_INC) - HDsprintf(attrname, "attr %02u", (u + 1)); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", (u + 1)); else - HDsprintf(attrname, "attr %02u", ((max_compact * 2) - (u + 2))); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", + ((max_compact * 2) - (u + 2))); ret = HDstrcmp(attrname, tmpname); VERIFY(ret, 0, "H5Aget_name_by_idx"); } /* end for */ @@ -6600,7 +6629,7 @@ test_attr_delete_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create attributes, to push into dense form */ for (u = 0; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -6669,9 +6698,10 @@ test_attr_delete_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) ret = (herr_t)H5Aget_name_by_idx(my_dataset, ".", idx_type, order, (hsize_t)u, tmpname, (size_t)NAME_BUF_SIZE, H5P_DEFAULT); if (order == H5_ITER_INC) - HDsprintf(attrname, "attr %02u", ((u * 2) + 1)); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", ((u * 2) + 1)); else - HDsprintf(attrname, "attr %02u", ((max_compact * 2) - ((u * 2) + 2))); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", + ((max_compact * 2) - ((u * 2) + 2))); ret = HDstrcmp(attrname, tmpname); VERIFY(ret, 0, "H5Aget_name_by_idx"); } /* end for */ @@ -6721,9 +6751,10 @@ test_attr_delete_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) ret = (herr_t)H5Aget_name_by_idx(my_dataset, ".", idx_type, order, (hsize_t)0, tmpname, (size_t)NAME_BUF_SIZE, H5P_DEFAULT); if (order == H5_ITER_INC) - HDsprintf(attrname, "attr %02u", ((u * 2) + 3)); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", ((u * 2) + 3)); else - HDsprintf(attrname, "attr %02u", ((max_compact * 2) - ((u * 2) + 4))); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", + ((max_compact * 2) - ((u * 2) + 4))); ret = HDstrcmp(attrname, tmpname); VERIFY(ret, 0, "H5Aget_name_by_idx"); } /* end for */ @@ -6804,7 +6835,7 @@ attr_iterate2_cb(hid_t loc_id, const char *attr_name, const H5A_info_t *info, vo } /* end if */ /* Verify name of link */ - HDsprintf(attrname, "attr %02u", (unsigned)my_info.corder); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", (unsigned)my_info.corder); if (HDstrcmp(attr_name, attrname) != 0) return (H5_ITER_ERROR); @@ -7314,7 +7345,7 @@ test_attr_iterate2(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create attributes, up to limit of compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -7386,7 +7417,7 @@ test_attr_iterate2(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create more attributes, to push into dense form */ for (u = max_compact; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -7678,7 +7709,7 @@ test_attr_open_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create attributes, up to limit of compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -7737,7 +7768,7 @@ test_attr_open_by_idx(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create more attributes, to push into dense form */ for (u = max_compact; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -7842,7 +7873,7 @@ attr_open_check(hid_t fid, const char *dsetname, hid_t obj_id, unsigned max_attr /* Open each attribute on object by index and check that it's the correct one */ for (u = 0; u < max_attrs; u++) { /* Open the attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr_id = H5Aopen(obj_id, attrname, H5P_DEFAULT); CHECK(attr_id, FAIL, "H5Aopen"); @@ -7998,7 +8029,7 @@ test_attr_open_by_name(hbool_t new_format, hid_t fcpl, hid_t fapl) is_dense = H5O__is_attr_dense_test(my_dataset); VERIFY(is_dense, FALSE, "H5O__is_attr_dense_test"); - /* Check for opening a non-existant attribute on an object with no attributes */ + /* Check for opening a non-existent attribute on an object with no attributes */ ret_id = H5Aopen(my_dataset, "foo", H5P_DEFAULT); VERIFY(ret_id, FAIL, "H5Aopen"); @@ -8011,7 +8042,7 @@ test_attr_open_by_name(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create attributes, up to limit of compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -8037,7 +8068,7 @@ test_attr_open_by_name(hbool_t new_format, hid_t fcpl, hid_t fapl) is_dense = H5O__is_attr_dense_test(my_dataset); VERIFY(is_dense, FALSE, "H5O__is_attr_dense_test"); - /* Check for opening a non-existant attribute on an object with compact attribute storage */ + /* Check for opening a non-existent attribute on an object with compact attribute storage */ ret_id = H5Aopen(my_dataset, "foo", H5P_DEFAULT); VERIFY(ret_id, FAIL, "H5Aopen"); @@ -8077,7 +8108,7 @@ test_attr_open_by_name(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create more attributes, to push into dense form */ for (u = max_compact; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate2(my_dataset, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate2"); @@ -8118,7 +8149,7 @@ test_attr_open_by_name(hbool_t new_format, hid_t fcpl, hid_t fapl) VERIFY(name_count, (max_compact * 2), "H5O__attr_dense_info_test"); } /* end if */ - /* Check for opening a non-existant attribute on an object with dense attribute storage */ + /* Check for opening a non-existent attribute on an object with dense attribute storage */ ret_id = H5Aopen(my_dataset, "foo", H5P_DEFAULT); VERIFY(ret_id, FAIL, "H5Aopen"); @@ -8260,7 +8291,7 @@ test_attr_create_by_name(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create attributes, up to limit of compact form */ for (u = 0; u < max_compact; u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate_by_name(fid, dsetname, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate_by_name"); @@ -8317,7 +8348,7 @@ test_attr_create_by_name(hbool_t new_format, hid_t fcpl, hid_t fapl) /* Create more attributes, to push into dense form */ for (u = max_compact; u < (max_compact * 2); u++) { /* Create attribute */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); attr = H5Acreate_by_name(fid, dsetname, attrname, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(attr, FAIL, "H5Acreate_by_name"); @@ -8542,7 +8573,7 @@ test_attr_shared_write(hid_t fcpl, hid_t fapl) /* Add attributes to each dataset, until after converting to dense storage */ for (u = 0; u < max_compact * 2; u++) { /* Create attribute name */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); /* Alternate between creating "small" & "big" attributes */ if (u % 2) { @@ -8705,9 +8736,11 @@ test_attr_shared_write(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* end for */ /* Close dataspaces */ @@ -8877,7 +8910,7 @@ test_attr_shared_rename(hid_t fcpl, hid_t fapl) /* Add attributes to each dataset, until after converting to dense storage */ for (u = 0; u < max_compact * 2; u++) { /* Create attribute name */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); /* Alternate between creating "small" & "big" attributes */ if (u % 2) { @@ -9151,9 +9184,11 @@ test_attr_shared_rename(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* end for */ /* Close dataspaces */ @@ -9322,7 +9357,7 @@ test_attr_shared_delete(hid_t fcpl, hid_t fapl) /* Add attributes to each dataset, until after converting to dense storage */ for (u = 0; u < max_compact * 2; u++) { /* Create attribute name */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); /* Alternate between creating "small" & "big" attributes */ if (u % 2) { @@ -9430,7 +9465,7 @@ test_attr_shared_delete(hid_t fcpl, hid_t fapl) /* Delete attributes from second dataset */ for (u = 0; u < max_compact * 2; u++) { /* Create attribute name */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); /* Delete second dataset's attribute */ ret = H5Adelete_by_name(fid, DSET2_NAME, attrname, H5P_DEFAULT); @@ -9521,9 +9556,11 @@ test_attr_shared_delete(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* end for */ /* Close dataspaces */ @@ -9692,7 +9729,7 @@ test_attr_shared_unlink(hid_t fcpl, hid_t fapl) /* Add attributes to each dataset, until after converting to dense storage */ for (u = 0; u < max_compact * 2; u++) { /* Create attribute name */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); /* Alternate between creating "small" & "big" attributes */ if (u % 2) { @@ -9816,7 +9853,7 @@ test_attr_shared_unlink(hid_t fcpl, hid_t fapl) /* Check ref count on attributes of first dataset */ for (u = 0; u < max_compact * 2; u++) { /* Create attribute name */ - HDsprintf(attrname, "attr %02u", u); + HDsnprintf(attrname, sizeof(attrname), "attr %02u", u); /* Open attribute on first dataset */ attr = H5Aopen(dataset, attrname, H5P_DEFAULT); @@ -9878,9 +9915,11 @@ test_attr_shared_unlink(hid_t fcpl, hid_t fapl) ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); - /* Check size of file */ - filesize = h5_get_file_size(FILENAME, fapl); - VERIFY(filesize, empty_filesize, "h5_get_file_size"); + if (h5_using_default_driver(NULL)) { + /* Check size of file */ + filesize = h5_get_file_size(FILENAME, fapl); + VERIFY(filesize, empty_filesize, "h5_get_file_size"); + } } /* end for */ /* Close dataspaces */ @@ -10986,7 +11025,7 @@ test_attr_bug8(hid_t fcpl, hid_t fapl) ** chunk. Do this twice: once with only attributes in the ** object header chunk and once with a (small) soft link in ** the chunk as well. In both cases, the shrunk chunk will -** initally be too small and a new NULL message must be +** initially be too small and a new NULL message must be ** created. ** ****************************************************************/ diff --git a/test/tcoords.c b/test/tcoords.c index fef8689..b2d68d3 100644 --- a/test/tcoords.c +++ b/test/tcoords.c @@ -17,7 +17,7 @@ * * Test the element coordinates for dataspace selection. For * chunked dataset, when the hyperslab selection of some - * dimensions is full, the library optimize it by "flattenning" + * dimensions is full, the library optimize it by "flattening" * the fully selected dimensions. This program tests if the * coordinates of selected elements are correctly calculated. * diff --git a/test/test_usecases.sh.in b/test/test_usecases.sh.in index eaa875e..fd30afd 100644 --- a/test/test_usecases.sh.in +++ b/test/test_usecases.sh.in @@ -21,17 +21,26 @@ # For now, it shows how to run the test cases programs. It only verifies the # exit codes are okay (0). -srcdir=@srcdir@ -bindir=@bindir@ +############################################################################### +## test variables +############################################################################### -# If the bindir directory is not set just use current (.). -if test -z "$bindir"; then - bindir=. -fi +# Number of errors encountered during test run. +nerrors=0 + +# Define variables +verbose=yes + +############################################################################### +## Main +############################################################################### +srcdir=@srcdir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -$bindir/swmr_check_compat_vfd +$utils_testdir/swmr_check_compat_vfd rc=$? if [[ $rc != 0 ]] ; then echo @@ -53,10 +62,6 @@ RESULT_SKIP="-SKIP-" USECASES_PROGRAMS="use_append_chunk use_append_mchunks" TESTNAME="Use Case" -# Define variables -nerrors=0 -verbose=yes - # Source in the output filter function definitions. . $srcdir/../bin/output_filter.sh @@ -91,7 +96,7 @@ TOOLTEST() { # Run test. TESTING $program $@ ( - $RUNSERIAL $bindir/$program "$@" + $RUNSERIAL $testdir/$program "$@" ) >$actual 2>$actual_err exit_code=$? @@ -123,7 +128,7 @@ TOOLTEST() { # the test. Running each of these tests in its own directory should eliminate # the problem. mkdir usecases_test -cp twriteorder usecases_test +cp $testdir/twriteorder usecases_test for FILE in use_*; do case "$FILE" in *.o) continue ;; ## don't copy the .o files diff --git a/test/testcheck_version.sh.in b/test/testcheck_version.sh.in index 6378ee5..273702e 100644 --- a/test/testcheck_version.sh.in +++ b/test/testcheck_version.sh.in @@ -36,7 +36,7 @@ RM='rm -f' PURPOSE() { echo "Tests for the H5check_version function." echo "Note that abort messages may appear due to the expected termination" - echo "of the program when it is tested with mis-matched version numnbers." + echo "of the program when it is tested with mis-matched version numbers." } # Print a line-line message left justified in a field of 70 characters. @@ -232,7 +232,7 @@ esac # RUNSERIAL is used. Check if it can return exit code from executalbe correctly. if [ -n "$RUNSERIAL_NOEXITCODE" ]; then - echo "***Warning*** Serial Exit Code is not passed back to shell corretly." + echo "***Warning*** Serial Exit Code is not passed back to shell correctly." echo "***Warning*** Exit code checking is skipped." h5haveexitcode=no fi diff --git a/test/testflushrefresh.sh.in b/test/testflushrefresh.sh.in index 83685e8..209d370 100644 --- a/test/testflushrefresh.sh.in +++ b/test/testflushrefresh.sh.in @@ -58,14 +58,13 @@ fi ############################################################################### ## Main ############################################################################### -# The build (current) directory might be different than the source directory. -if test -z "$srcdir"; then - srcdir=. -fi +srcdir=@srcdir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -./swmr_check_compat_vfd +$utils_testdir/swmr_check_compat_vfd rc=$? if [ $rc -ne 0 ] ; then echo @@ -83,7 +82,7 @@ fi # the test. Running each of these tests in its own directory should eliminate # the problem. mkdir -p flushrefresh_test -cp flushrefresh flushrefresh_test +cp $testdir/flushrefresh flushrefresh_test # With the --disable-shared option, flushrefresh is built in the test directory, # otherwise it is in test/.libs with a wrapper script named flushrefresh in @@ -118,7 +117,7 @@ fi # ======================== # Launch the Test Program. # ======================== -./flushrefresh & +$testdir/flushrefresh & pid_main=$! # ======================================= @@ -155,7 +154,7 @@ until [ $verification_done -eq 1 ]; do verification_done=1 echo "all flush verification complete" > $endsignal else - ./flushrefresh $param1 $param2 + $testdir/flushrefresh $param1 $param2 # Check for core dump if [ $? -gt 0 ]; then @@ -200,7 +199,7 @@ if [ $timedout -eq 0 ]; then verification_done=2 echo "all refresh verification complete" > $endsignal else - ./flushrefresh $param1 + $testdir/flushrefresh $param1 # Check for core dump if [ $? -gt 0 ]; then diff --git a/test/testhdf5.c b/test/testhdf5.c index 45c0f9f..dc5f0aa 100644 --- a/test/testhdf5.c +++ b/test/testhdf5.c @@ -81,7 +81,7 @@ main(int argc, char *argv[]) TestSummary(); /* Clean up test files, if allowed */ - if (GetTestCleanup() && !HDgetenv("HDF5_NOCLEANUP")) + if (GetTestCleanup() && !HDgetenv(HDF5_NOCLEANUP)) TestCleanup(); /* Release test infrastructure */ diff --git a/test/testlibinfo.sh.in b/test/testlibinfo.sh.in index 2a9a05c..d9fee48 100644 --- a/test/testlibinfo.sh.in +++ b/test/testlibinfo.sh.in @@ -19,7 +19,7 @@ # libhdf5.settings file. # Part 2: # If executable is linked with the static hdf5 library (how to determine?), -# verify an executable indeed contains an exact copy of hte content of the +# verify an executable indeed contains an exact copy of the content of the # libhdf5.settings file. # # Programmer: Albert Cheng diff --git a/test/testmeta.c b/test/testmeta.c index 7cb3000..a3b5160 100644 --- a/test/testmeta.c +++ b/test/testmeta.c @@ -15,7 +15,7 @@ * This program illustrates assertion errors when linked * to HDF5 1.4.1-post2 or 1.4.2-pre3 debug library. * - * If the assertion errors are ignored, the program eventially causes + * If the assertion errors are ignored, the program eventually causes * an error in H5Gcreate2 when writing object 83381. * * When writing in single file mode, the assertion errors still occur diff --git a/test/testswmr.pwsh.in b/test/testswmr.pwsh.in new file mode 100644 index 0000000..de7a57a --- /dev/null +++ b/test/testswmr.pwsh.in @@ -0,0 +1,519 @@ +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the swmr feature. + +$srcdir = '@srcdir@' +$utils_testdir=@H5_UTILS_TEST_BUILDDIR@ +$testdir=@H5_TEST_BUILDDIR@ + +############################################################################### +## test parameters +############################################################################### + +$Nreaders = 5 # number of readers to launch +$Nrdrs_spa = 3 # number of sparse readers to launch +$Nrecords = 200000 # number of records to write +$Nrecs_rem = 40000 # number of times to shrink +$Nrecs_spa = 20000 # number of records to write in the sparse test +$Nsecs_add = 5 # number of seconds per read interval +$Nsecs_rem = 3 # number of seconds per read interval +$Nsecs_addrem = 8 # number of seconds per read interval +$nerrors = 0 + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE' # The message file created by writer that the open is complete + # This should be the same as the define in "test/swmr_common.h" +$MESSAGE_TIMEOUT = 300 # Message timeout length in secs + # This should be the same as the define in "test/h5test.h" + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +function Test-WithSpaces { + $SPACES=' ' + #Write-Output "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $Message is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +function Wait-Message { + [CmdletBinding()] + param ( + [Parameter(Mandatory, ValueFromPipeline)] + [string]$TestPath, # Get the path of the message file to wait for + [Parameter(Mandatory, ValueFromPipeline)] + [string]$Message # Get the name of the message file to wait for + ) + + PROCESS { + $message = Join-Path -Path $TestPath -ChildPath $Message + $t0 = Get-Date # Get current time + $t1 = $t0 + $difft = New-Timespan -Start $t0 -End $t1 # Initialize the time difference + $mexist = 0 # Indicate whether the message file is found + while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) { # Loop till message times out + $t1 = Get-Date # Get current time in seconds + $difft = New-Timespan -Start $t0 -End $t1 # Calculate the time difference + #Write-Output "Check for $message : time=$difft" + if ([System.IO.File]::Exists($message)) { # If message file is found: + $mexist = 1 # indicate the message file is found + Remove-Item $message # remove the message file + break # get out of the while loop + } + } + if ($mexist -eq 0) { + # Issue warning that the writer message file is not found, continue with launching the reader(s) + Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds" + } + else { + Write-Output "$WRITER_MESSAGE is found" + } + } +} + +############################################################################### +## Main +## +## Modifications: +## Vailin Choi; July 2013 +## Add waiting of message file before launching the reader(s). +## Due to the implementation of file locking, coordination +## is needed in file opening for the writer/reader tests +## to proceed as expected. +## +############################################################################### + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe +$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow +if ($rp.ExitCode -ne 0) { + Write-Output "" + Write-Output "The VFD specified by the HDF5_DRIVER environment variable" + Write-Output "does not support SWMR." + Write-Output "" + Write-Output "SWMR acceptance tests skipped" + Write-Output "" + exit 0 +} + +Set-Location -Path swmr_test +$testdir = Join-Path -Path $testdir -ChildPath swmr_test + +# Loop over index types +$IndexTypeLoop = '-i ea', '-i b2' +foreach ($index_type in $IndexTypeLoop) { + # Try with and without compression + $CompressLoop = '', '-c 5' + foreach ($compress in $CompressLoop) { + Write-Output "" + Write-Output "*******************************************************************************" + Write-Output "** Loop testing parameters: $index_type $compress" + Write-Output "*******************************************************************************" + Write-Output "" + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Generator test" + Write-Output "###############################################################################" + # Launch the Generator without SWMR_WRITE + Write-Output "launch the swmr_generator" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "$compress $index_type" -PassThru -Wait + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Launch the Generator with SWMR_WRITE + Write-Output "launch the swmr_generator with SWMR_WRITE" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "-s $compress $index_type" -PassThru -Wait + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + exit 1 + } + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Use H5Fstart_swmr_write() to enable SWMR writing mode" + Write-Output "###############################################################################" + + # Remove any possible writer message file before launching writer + if ([System.IO.File]::Exists($WRITER_MESSAGE)) { # If message file is found + Remove-Item $WRITER_MESSAGE + } + # + # Launch the Writer + Write-Output "launch the swmr_start_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_start_write.exe -NoNewWindow -PassThru -ArgumentList "$compress $index_type $Nrecords $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + + # + # Launch the Readers + #declare -a seeds = ( ... ) + Write-Output "launch $Nreaders swmr_readers" + $pid_readers = @() + $n = 0 + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 | tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Writer test - test expanding the dataset" + Write-Output "###############################################################################" + + # Launch the Generator + Write-Output "launch the swmr_generator" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "-s $compress $index_type" + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # + # Launch the Writer + Write-Output "launch the swmr_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecords $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Readers + #declare -a seeds = ( ... ) + Write-Output "launch $Nreaders swmr_readers" + $pid_readers = @() + $n = 0 + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Remove test - test shrinking the dataset" + Write-Output "###############################################################################" + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # Launch the Remove Writer + Write-Output "launch the swmr_remove_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_remove_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecs_rem $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Remove Readers + #declare -a seeds = ( ... ) + $n = 0 + $pid_readers = @() + Write-Output "launch $Nreaders swmr_remove_readers" + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_rem $seed" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Add/remove test - randomly grow or shrink the dataset" + Write-Output "###############################################################################" + + # Launch the Generator + Write-Output "launch the swmr_generator" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow-PassThru -Wait -ArgumentList "$compress $index_type" + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Launch the Writer (not in parallel - just to rebuild the datasets) + Write-Output "launch the swmr_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -Wait -ArgumentList "$Nrecords $seed" + if ($rp.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # + # Launch the Add/Remove Writer + Write-Output "launch the swmr_addrem_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_addrem_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecords $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Add/Remove Readers + #declare -a seeds = ( ... ) + $n = 0 + $pid_readers = @() + Write-Output "launch $Nreaders swmr_remove_readers" + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_addrem $seed" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Sparse writer test - test writing to random locations in the dataset" + Write-Output "###############################################################################" + + # Launch the Generator + # NOTE: Random seed is shared between readers and writers and is + # created by the generator. + Write-Output "launch the swmr_generator" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "$compress $index_type $seed" + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # Launch the Sparse writer + Write-Output "launch the swmr_sparse_writer" + $rp = Start-Process -FilePath $testdir/swmr_sparse_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecs_spa" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.Id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Sparse readers + $n = 0 + $pid_readers = @() + Write-Output "launch $Nrdrs_spa swmr_sparse_readers" + while ($n -lt $Nrdrs_spa) { + # The sparse reader spits out a LOT of data so it's set to 'quiet' + $rp = Start-Process -FilePath $testdir/swmr_sparse_reader.exe -NoNewWindow -PassThru -ArgumentList "-q $Nrecs_spa" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Collect exit code of the readers + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + } +} + +############################################################################### +## Report and exit +############################################################################### +cd .. +#Write-Output "nerrors=$nerrors" +if ($nerrors -eq 0) { + Write-Output "SWMR tests passed." +# if test -z "$HDF5_NOCLEANUP"; then +# # delete the test directory +# Remove-Item swmr_test -Recurse +# fi + exit 0 +} +else { + Write-Warning "SWMR tests failed with $nerrors errors." + exit 1 +} + diff --git a/test/testswmr.sh.in b/test/testswmr.sh.in index 37e8f9c..771fe4a 100644 --- a/test/testswmr.sh.in +++ b/test/testswmr.sh.in @@ -17,7 +17,8 @@ # Albert Cheng, 2009/07/22 srcdir=@srcdir@ -bindir=@bindir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ ############################################################################### ## test parameters @@ -37,9 +38,9 @@ nerrors=0 ## definitions for message file to coordinate test runs ############################################################################### WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete - # This should be the same as the define in "./swmr_common.h" + # This should be the same as the define in "test/swmr_common.h" MESSAGE_TIMEOUT=300 # Message timeout length in secs - # This should be the same as the define in "./h5test.h" + # This should be the same as the define in "test/h5test.h" ############################################################################### ## short hands and function definitions @@ -93,19 +94,10 @@ WAIT_MESSAGE() { ## to proceed as expected. ## ############################################################################### -# The build (current) directory might be different than the source directory. -if test -z "$srcdir"; then - srcdir=. -fi - -# If the bindir directory is not set just use current (.). -if test -z "$bindir"; then - bindir=. -fi # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -$bindir/swmr_check_compat_vfd +$utils_testdir/swmr_check_compat_vfd rc=$? if [ $rc -ne 0 ] ; then echo @@ -178,7 +170,7 @@ do echo "###############################################################################" # Launch the Generator without SWMR_WRITE echo launch the swmr_generator - $bindir/swmr_generator $compress $index_type + $testdir/swmr_generator $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -186,13 +178,13 @@ do # Launch the Generator with SWMR_WRITE echo launch the swmr_generator with SWMR_WRITE - $bindir/swmr_generator -s $compress $index_type + $testdir/swmr_generator -s $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` fi - # Check for error and exit if one occured + # Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "SWMR tests failed with $nerrors errors." @@ -210,7 +202,7 @@ do # Launch the Writer echo launch the swmr_start_writer seed="" # Put -r command here - $bindir/swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out & + $testdir/swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -226,7 +218,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - $bindir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -252,7 +244,7 @@ do nerrors=`expr $nerrors + 1` fi - # Check for error and exit if one occured + # Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "SWMR tests failed with $nerrors errors." @@ -271,7 +263,7 @@ do # Launch the Generator echo launch the swmr_generator - $bindir/swmr_generator -s $compress $index_type + $testdir/swmr_generator -s $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -283,7 +275,7 @@ do # Launch the Writer echo launch the swmr_writer seed="" # Put -r command here - $bindir/swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out & + $testdir/swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -298,7 +290,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - $bindir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -324,7 +316,7 @@ do nerrors=`expr $nerrors + 1` fi - # Check for error and exit if one occured + # Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "SWMR tests failed with $nerrors errors." @@ -346,7 +338,7 @@ do # Launch the Remove Writer echo launch the swmr_remove_writer seed="" # Put -r command here - $bindir/swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out & + $testdir/swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -361,7 +353,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - $bindir/swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n & + $testdir/swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -387,7 +379,7 @@ do nerrors=`expr $nerrors + 1` fi - # Check for error and exit if one occured + # Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "SWMR tests failed with $nerrors errors." @@ -406,7 +398,7 @@ do # Launch the Generator echo launch the swmr_generator - $bindir/swmr_generator $compress $index_type + $testdir/swmr_generator $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -415,7 +407,7 @@ do # Launch the Writer (not in parallel - just to rebuild the datasets) echo launch the swmr_writer seed="" # Put -r command here - $bindir/swmr_writer $Nrecords $seed + $testdir/swmr_writer $Nrecords $seed if test $? -ne 0; then echo writer had error nerrors=`expr $nerrors + 1` @@ -427,7 +419,7 @@ do # Launch the Add/Remove Writer echo launch the swmr_addrem_writer seed="" # Put -r command here - $bindir/swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out & + $testdir/swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -442,7 +434,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - $bindir/swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n & + $testdir/swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -468,7 +460,7 @@ do nerrors=`expr $nerrors + 1` fi - # Check for error and exit if one occured + # Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "SWMR tests failed with $nerrors errors." @@ -490,7 +482,7 @@ do # created by the generator. echo launch the swmr_generator seed="" # Put -r command here - $bindir/swmr_generator $compress $index_type $seed + $testdir/swmr_generator $compress $index_type $seed if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -500,7 +492,7 @@ do rm -f $WRITER_MESSAGE # Launch the Sparse writer echo launch the swmr_sparse_writer - nice -n 20 $bindir/swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out & + nice -n 20 $testdir/swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -513,7 +505,7 @@ do echo launch $Nrdrs_spa swmr_sparse_readers while [ $n -lt $Nrdrs_spa ]; do # The sparse reader spits out a LOT of data so it's set to 'quiet' - $bindir/swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n & + $testdir/swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -538,7 +530,7 @@ do fi done - # Check for error and exit if one occured + # Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "SWMR tests failed with $nerrors errors." diff --git a/test/testvdsswmr.pwsh.in b/test/testvdsswmr.pwsh.in new file mode 100644 index 0000000..bf5aabb --- /dev/null +++ b/test/testvdsswmr.pwsh.in @@ -0,0 +1,190 @@ +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the swmr feature using virtual datasets. + +$srcdir = '@srcdir@' +$utils_testdir=@H5_UTILS_TEST_BUILDDIR@ +$testdir=@H5_TEST_BUILDDIR@ + +############################################################################### +## test parameters +############################################################################### + +$Nwriters = 6 # number of writers (1 per source dataset) +$Nreaders = 5 # number of readers to launch +$nerrors = 0 + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE' # The message file created by writer that the open is complete + # This should be the same as the define in "test/swmr_common.h" +$MESSAGE_TIMEOUT = 300 # Message timeout length in secs + # This should be the same as the define in "test/h5test.h" + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +function Test-WithSpaces { + $SPACES=' ' + #Write-Output "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $Message is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +function Wait-Message { + [CmdletBinding()] + param ( + [Parameter(Mandatory, ValueFromPipeline)] + [string]$TestPath, # Get the path of the message file to wait for + [Parameter(Mandatory, ValueFromPipeline)] + [string]$Message # Get the name of the message file to wait for + ) + + PROCESS { + $message = Join-Path -Path $TestPath -ChildPath $Message + $t0 = Get-Date # Get current time + $t1 = $t0 + $difft = New-Timespan -Start $t0 -End $t1 # Initialize the time difference + $mexist = 0 # Indicate whether the message file is found + while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) { # Loop till message times out + $t1 = Get-Date # Get current time in seconds + $difft = New-Timespan -Start $t0 -End $t1 # Calculate the time difference + #Write-Output "Check for $message : time=$difft" + if ([System.IO.File]::Exists($message)) { # If message file is found: + $mexist = 1 # indicate the message file is found + Remove-Item $message # remove the message file + break # get out of the while loop + } + } + if ($mexist -eq 0) { + # Issue warning that the writer message file is not found, continue with launching the reader(s) + Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds" + } + else { + Write-Output "$WRITER_MESSAGE is found" + } + } +} + +############################################################################### +## Main +############################################################################### + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe +$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow +if ($rp.ExitCode -ne 0) { + Write-Output "" + Write-Output "The VFD specified by the HDF5_DRIVER environment variable" + Write-Output "does not support SWMR." + Write-Output "" + Write-Output "SWMR acceptance tests skipped" + Write-Output "" + exit 0 +} + +Set-Location -Path vds_swmr_test +$testdir = Join-Path -Path $testdir -ChildPath vds_swmr_test + +Write-Output "" +Write-Output "###############################################################################" +Write-Output "## Basic VDS SWMR test - writing to a tiled plane" +Write-Output "###############################################################################" + +# Launch the file generator +Write-Output "launch the swmr_generator" +$rp = Start-Process -FilePath $testdir/vds_swmr_gen.exe -NoNewWindow -PassThru -Wait +if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 +} + +# Check for error and exit if one occurred +#Write-Output "nerrors=$nerrors" +if ($nerrors -ne 0) { + Write-Warning "VDS SWMR tests failed with $nerrors errors." + exit 1 +} + +# Launch the writers +Write-Output "launch the $Nwriters SWMR VDS writers (1 per source)" +$pid_writers = @() +$n = 0 +while ($n -lt $Nwriters) { + $rp = Start-Process -FilePath $testdir/vds_swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "$n" 2>&1 | tee swmr_writer.out.$n + $pid_writers += $rp.id + $n += 1 +} +#Write-Output "pid_writers=$pid_writers" + +# Sleep to ensure that the writers have started +Start-Sleep -Seconds 3 + +# Launch the readers +Write-Output "launch $Nreaders SWMR readers" +$pid_readers = @() +$n = 0 +while ($n -lt $Nreaders) { + $rp = Start-Process -FilePath $testdir/vds_swmr_reader.exe -NoNewWindow -PassThru 2>&1 | tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 +} +#Write-Output "pid_readers=$pid_readers" + +# Collect exit code of the writers +foreach ($xpid in $pid_writers) { + #Write-Output "checked writer $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } +} + +# Collect exit code of the readers +# (they usually finish after the writers) +foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } +} + +# Check for error and exit if one occurred +#Write-Output "nerrors=$nerrors" +if ($nerrors -ne 0) { + Write-Warning "VDS SWMR tests failed with $nerrors errors." + exit 1 +} + +############################################################################### +## Report and exit +############################################################################### +cd .. +#Write-Output "nerrors=$nerrors" +if ($nerrors -eq 0) { + Write-Output "VDS SWMR tests passed." +# if test -z "$HDF5_NOCLEANUP"; then +# # delete the test directory +# Remove-Item vds_swmr_test -Recurse +# fi + exit 0 +} +else { + Write-Warning "VDS SWMR tests failed with $nerrors errors." + exit 1 +} + diff --git a/test/testvdsswmr.sh.in b/test/testvdsswmr.sh.in index 5399903..399fdef 100644 --- a/test/testvdsswmr.sh.in +++ b/test/testvdsswmr.sh.in @@ -17,7 +17,8 @@ # Dana Robinson, November 2015 srcdir=@srcdir@ -bindir=@bindir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ ############################################################################### ## test parameters @@ -31,9 +32,9 @@ nerrors=0 ## definitions for message file to coordinate test runs ############################################################################### WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete - # This should be the same as the define in "$bindir/swmr_common.h" + # This should be the same as the define in "test/swmr_common.h" MESSAGE_TIMEOUT=300 # Message timeout length in secs - # This should be the same as the define in "./h5test.h" + # This should be the same as the define in "test/h5test.h" ############################################################################### ## short hands and function definitions @@ -79,19 +80,10 @@ WAIT_MESSAGE() { ############################################################################### ## Main ############################################################################### -# The build (current) directory might be different than the source directory. -if test -z "$srcdir"; then - srcdir=. -fi - -# If the bindir directory is not set just use current (.). -if test -z "$bindir"; then - bindir=. -fi # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -$bindir/swmr_check_compat_vfd +$utils_testdir/swmr_check_compat_vfd rc=$? if [ $rc -ne 0 ] ; then echo @@ -154,13 +146,13 @@ echo "########################################################################## # Launch the file generator echo launch the generator -$bindir/vds_swmr_gen +$testdir/vds_swmr_gen if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` fi -# Check for error and exit if one occured +# Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "VDS SWMR tests failed with $nerrors errors." @@ -172,7 +164,7 @@ echo "launch the $Nwriters SWMR VDS writers (1 per source)" pid_writers="" n=0 while [ $n -lt $Nwriters ]; do - $bindir/vds_swmr_writer $n & + $testdir/vds_swmr_writer $n & pid_writers="$pid_writers $!" n=`expr $n + 1` done @@ -187,7 +179,7 @@ echo launch $Nreaders SWMR readers pid_readers="" n=0 while [ $n -lt $Nreaders ]; do - $bindir/vds_swmr_reader & + $testdir/vds_swmr_reader & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -215,7 +207,7 @@ for xpid in $pid_readers; do fi done -# Check for error and exit if one occured +# Check for error and exit if one occurred $DPRINT nerrors=$nerrors if test $nerrors -ne 0 ; then echo "VDS SWMR tests failed with $nerrors errors." diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index a56bf99..88a4ec1 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -21,6 +21,8 @@ srcdir=@srcdir@ . ${srcdir}/supervise.subr +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ + ############################################################################### ## test parameters ############################################################################### @@ -80,6 +82,12 @@ GFAIL_FIFO_WRITER_TO_READER=fifo_group_writer_to_reader GFAIL_FIFO_READER_TO_WRITER=fifo_group_reader_to_writer ############################################################################### +## Was the aux process built? +############################################################################### +# +USING_AUX_PROCESS=@AUX_PROCESS@ + +############################################################################### ## short hands and function definitions ############################################################################### # @@ -131,17 +139,15 @@ if test -z "$srcdir"; then fi # Check to see if the VFD specified by the HDF5_DRIVER environment variable -# supports SWMR and/or if we are using parallel HDF5, which does not -# currently support SWMR. -./vfd_swmr_check_compat +# supports SWMR. +$utils_testdir/swmr_check_compat_vfd rc=$? if [ $rc -ne 0 ] ; then echo - echo "Either the VFD specified by the HDF5_DRIVER environment variable" - echo "does not support VFD SWMR or parallel HDF5 was configured, which" - echo "currently does not support SWMR." + echo "The VFD specified by the HDF5_DRIVER environment variable" + echo "does not support SWMR." echo - echo "VFD SWMR acceptance tests skipped" + echo "SWMR acceptance tests skipped" echo exit 0 fi @@ -1075,62 +1081,68 @@ fi # # for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t -F" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do - if [ ${do_many_small:-no} = no ]; then - continue - fi - # - # Test many small datasets of two or three dimensions. - # - # Perform 25 iterations on 100 extensible datasets configured with - # 2D 16x16 chunks or 3D 1x16x16 chunks of 32-bit unsigned integer elements, - # expanding each dataset by a chunk in one dimension (up to 25x1 - # 16x16 chunks) on each iteration. - # - # Perform the test again, extending each dataset - # in *two* dimensions (up to 25x25 16x16 chunks). - # - # If testing 3D datasets (-t option), extending each dataset along the - # first dimension (up to 25 1x16x16) - # + if [ ${do_many_small:-no} = no ]; then + continue + fi + # + # Test many small datasets of two or three dimensions. + # + # Perform 25 iterations on 100 extensible datasets configured with + # 2D 16x16 chunks or 3D 1x16x16 chunks of 32-bit unsigned integer elements, + # expanding each dataset by a chunk in one dimension (up to 25x1 + # 16x16 chunks) on each iteration. + # + # Perform the test again, extending each dataset + # in *two* dimensions (up to 25x25 16x16 chunks). + # + # If testing 3D datasets (-t option), extending each dataset along the + # first dimension (up to 25 1x16x16) + # - # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. - # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of - # VDS across multiple files (-M option), the program also skips for future support. + # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. + # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of + # VDS across multiple files (-M option), the program also skips for future support. + if test "X$USING_AUX_PROCESS" = "Xyes"; then if [[ $options == *"-M"* ]]; then - catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & else - catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & fi pid_aux_proc=$! + fi - echo launch vfd_swmr_bigset_writer many small, options $options - catch_out_err_and_rc vfd_swmr_bigset_writer \ - ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q & - pid_writer=$! + echo launch vfd_swmr_bigset_writer many small, options $options + catch_out_err_and_rc vfd_swmr_bigset_writer \ + ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q & + pid_writer=$! - catch_out_err_and_rc vfd_swmr_bigset_reader \ - ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q & - pid_reader=$! + catch_out_err_and_rc vfd_swmr_bigset_reader \ + ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q & + pid_reader=$! - # Wait for the reader to finish before signalling the - # writer to quit: the writer holds the file open so that the - # reader will find the shadow file when it opens - # the .h5 file. Also wait for the auxiliary process to finish. + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. Also wait for the auxiliary process to finish. + if test "X$USING_AUX_PROCESS" = "Xyes"; then wait $pid_aux_proc - wait $pid_reader - wait $pid_writer + fi + wait $pid_reader + wait $pid_writer - # Collect exit code of the auxiliary process - if [ $(cat aux_process.rc) -ne 0 ]; then - echo the auxiliary process had error - nerrors=$((nerrors + 1)) - fi + # Collect exit code of the auxiliary process + if test "X$USING_AUX_PROCESS" = "Xyes"; then + if [ $(cat aux_process.rc) -ne 0 ]; then + echo the auxiliary process had error + nerrors=$((nerrors + 1)) + fi + fi - # Collect exit code of the reader - if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then - echo reader had error - nerrors=$((nerrors + 1)) - fi + # Collect exit code of the reader + if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi # Collect exit code of the writer if [ $(cat vfd_swmr_bigset_writer.rc) -ne 0 ]; then @@ -1139,83 +1151,89 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t fi # Clean up output files - rm -f vfd_swmr_bigset_writer.{out,rc} - rm -f vfd_swmr_bigset_reader.*.{out,rc} - rm -f aux_process.{out,rc} - rm -f mdfile bigset_updater.* bigset-shadow-* + rm -f vfd_swmr_bigset_writer.{out,rc} + rm -f vfd_swmr_bigset_reader.*.{out,rc} + rm -f aux_process.{out,rc} + rm -f mdfile bigset_updater.* bigset-shadow-* done # bigset test for bigger chunks for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d 1 -t -F -l 10" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do - # - # Test a few big datasets of two or three dimensions. - # - # Perform 25 iterations on 10 extensible datasets configured with - # 2D 256x256 chunks or 3D 8x256x256 of 32-bit unsigned integer elements, - # expanding each dataset by a chunk in one dimension (up to 25x1 - # 256x256 chunks) on each iteration. - # - # Perform the test again, extending each dataset - # in *two* dimensions (up to 25x25 256x256 chunks). - # - # If testing 3D datasets (-t option), extending each dataset along the - # first dimension (up to 25 8x256x256) - # + # + # Test a few big datasets of two or three dimensions. + # + # Perform 25 iterations on 10 extensible datasets configured with + # 2D 256x256 chunks or 3D 8x256x256 of 32-bit unsigned integer elements, + # expanding each dataset by a chunk in one dimension (up to 25x1 + # 256x256 chunks) on each iteration. + # + # Perform the test again, extending each dataset + # in *two* dimensions (up to 25x25 256x256 chunks). + # + # If testing 3D datasets (-t option), extending each dataset along the + # first dimension (up to 25 8x256x256) + # - if [ ${do_few_big:-no} = no ]; then - continue - fi - echo launch vfd_swmr_bigset_writer few big, options $options ......may take some time...... + if [ ${do_few_big:-no} = no ]; then + continue + fi + echo launch vfd_swmr_bigset_writer few big, options $options ......may take some time...... - # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. - # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of - # VDS across multiple files (-M option), the program also skips for future support. + # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. + # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of + # VDS across multiple files (-M option), the program also skips for future support. + if test "X$USING_AUX_PROCESS" = "Xyes"; then if [[ $options == *"-M"* ]]; then - catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & else - catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & fi pid_aux_proc=$! + fi - catch_out_err_and_rc vfd_swmr_bigset_writer \ - ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q & - pid_writer=$! + catch_out_err_and_rc vfd_swmr_bigset_writer \ + ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q & + pid_writer=$! - catch_out_err_and_rc vfd_swmr_bigset_reader \ - ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q & - pid_reader=$! + catch_out_err_and_rc vfd_swmr_bigset_reader \ + ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q & + pid_reader=$! - # Wait for the reader to finish before signalling the - # writer to quit: the writer holds the file open so that the - # reader will find the shadow file when it opens - # the .h5 file. Also wait for the auxiliary process to finish. + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. Also wait for the auxiliary process to finish. + if test "X$USING_AUX_PROCESS" = "Xyes"; then wait $pid_aux_proc - wait $pid_reader - wait $pid_writer + fi + wait $pid_reader + wait $pid_writer # Collect exit code of the auxiliary process - if [ $(cat aux_process.rc) -ne 0 ]; then - echo the auxiliary process had error - nerrors=$((nerrors + 1)) - fi + if test "X$USING_AUX_PROCESS" = "Xyes"; then + if [ $(cat aux_process.rc) -ne 0 ]; then + echo the auxiliary process had error + nerrors=$((nerrors + 1)) + fi + fi - # Collect exit code of the reader - if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then - echo reader had error - nerrors=$((nerrors + 1)) - fi + # Collect exit code of the reader + if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi - # Collect exit code of the writer - if [ $(cat vfd_swmr_bigset_writer.rc) -ne 0 ]; then - echo writer had error - nerrors=$((nerrors + 1)) - fi + # Collect exit code of the writer + if [ $(cat vfd_swmr_bigset_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi - # Clean up output files - rm -f vfd_swmr_bigset_writer.{out,rc} - rm -f vfd_swmr_bigset_reader.*.{out,rc} - rm -f aux_process.{out,rc} - rm -f mdfile bigset_updater.* bigset-shadow-* + # Clean up output files + rm -f vfd_swmr_bigset_writer.{out,rc} + rm -f vfd_swmr_bigset_reader.*.{out,rc} + rm -f aux_process.{out,rc} + rm -f mdfile bigset_updater.* bigset-shadow-* done ############################################################################### @@ -1563,7 +1581,7 @@ fi # Only for exhaustive run # # Verify that meaningful message about increasing max_lag is printed when -# incorrect metadata checksum error is enountered when loading the cache entry. +# incorrect metadata checksum error is encountered when loading the cache entry. # # Note that there will be messages from the error stack printed out for this test. # This is expected. diff --git a/test/tfile.c b/test/tfile.c index 656e956..31a9581 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -35,6 +35,10 @@ #define H5F_TESTING #include "H5Fpkg.h" /* File access */ +#define H5FD_FRIEND /*suppress error about including H5FDpkg.h */ +#define H5FD_TESTING +#include "H5FDpkg.h" + #define H5D_FRIEND /*suppress error about including H5Dpkg */ #include "H5Dpkg.h" /* Dataset access */ @@ -176,8 +180,8 @@ const char *FILESPACE_NAME[] = {"tfilespace", NULL}; #define DSET_DS1 "DS1" /* Local test function declarations for version bounds */ -static void test_libver_bounds_low_high(void); -static void test_libver_bounds_super(hid_t fapl); +static void test_libver_bounds_low_high(const char *env_h5_drvr); +static void test_libver_bounds_super(hid_t fapl, const char *env_h5_drvr); static void test_libver_bounds_super_create(hid_t fapl, hid_t fcpl, htri_t is_swmr, htri_t non_def_fsm); static void test_libver_bounds_super_open(hid_t fapl, hid_t fcpl, htri_t is_swmr, htri_t non_def_fsm); static void test_libver_bounds_obj(hid_t fapl); @@ -210,11 +214,11 @@ static void test_rw_noupdate(void); static void test_file_create(void) { - hid_t fid1, fid2, fid3; /* HDF5 File IDs */ - hid_t tmpl1, tmpl2; /*file creation templates */ - hsize_t ublock; /*sizeof userblock */ - size_t parm; /*file-creation parameters */ - size_t parm2; /*file-creation parameters */ + hid_t fid1, fid2, fid3; /* HDF5 File IDs */ + hid_t tmpl1, tmpl2; /* file creation templates */ + hsize_t ublock; /* sizeof userblock */ + size_t parm; /* file-creation parameters */ + size_t parm2; /* file-creation parameters */ unsigned iparm; unsigned iparm2; herr_t ret; /*generic return value */ @@ -225,7 +229,7 @@ test_file_create(void) /* First ensure the file does not exist */ HDremove(FILE1); - /* Try opening a non-existant file */ + /* Try opening a non-existent file */ fid1 = H5Fopen(FILE1, H5F_ACC_RDWR, H5P_DEFAULT); VERIFY(fid1, FAIL, "H5Fopen"); @@ -304,186 +308,189 @@ test_file_create(void) VERIFY(ret, FAIL, "H5Pclose"); #endif - /* Create a new file with a non-standard file-creation template */ - tmpl1 = H5Pcreate(H5P_FILE_CREATE); - CHECK(tmpl1, FAIL, "H5Pcreate"); + if (h5_using_default_driver(NULL)) { - /* Try setting some bad userblock sizes */ - H5E_BEGIN_TRY - { - ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE1); - } - H5E_END_TRY; - VERIFY(ret, FAIL, "H5Pset_userblock"); - H5E_BEGIN_TRY - { - ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE2); - } - H5E_END_TRY; - VERIFY(ret, FAIL, "H5Pset_userblock"); - H5E_BEGIN_TRY - { - ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE3); - } - H5E_END_TRY; - VERIFY(ret, FAIL, "H5Pset_userblock"); - H5E_BEGIN_TRY - { - ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE4); - } - H5E_END_TRY; - VERIFY(ret, FAIL, "H5Pset_userblock"); - H5E_BEGIN_TRY - { - ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE5); - } - H5E_END_TRY; - VERIFY(ret, FAIL, "H5Pset_userblock"); - H5E_BEGIN_TRY - { - ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE6); - } - H5E_END_TRY; - VERIFY(ret, FAIL, "H5Pset_userblock"); - H5E_BEGIN_TRY - { - ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE7); - } - H5E_END_TRY; - VERIFY(ret, FAIL, "H5Pset_userblock"); - - /* Set the new file-creation parameters */ - ret = H5Pset_userblock(tmpl1, F2_USERBLOCK_SIZE); - CHECK(ret, FAIL, "H5Pset_userblock"); + /* Create a new file with a non-standard file-creation template */ + tmpl1 = H5Pcreate(H5P_FILE_CREATE); + CHECK(tmpl1, FAIL, "H5Pcreate"); - ret = H5Pset_sizes(tmpl1, (size_t)F2_OFFSET_SIZE, (size_t)F2_LENGTH_SIZE); - CHECK(ret, FAIL, "H5Pset_sizes"); - - ret = H5Pset_sym_k(tmpl1, F2_SYM_INTERN_K, F2_SYM_LEAF_K); - CHECK(ret, FAIL, "H5Pset_sym_k"); + /* Try setting some bad userblock sizes */ + H5E_BEGIN_TRY + { + ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE1); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_userblock"); + H5E_BEGIN_TRY + { + ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE2); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_userblock"); + H5E_BEGIN_TRY + { + ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE3); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_userblock"); + H5E_BEGIN_TRY + { + ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE4); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_userblock"); + H5E_BEGIN_TRY + { + ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE5); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_userblock"); + H5E_BEGIN_TRY + { + ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE6); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_userblock"); + H5E_BEGIN_TRY + { + ret = H5Pset_userblock(tmpl1, BAD_USERBLOCK_SIZE7); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_userblock"); - /* - * Try to create second file, with non-standard file-creation template - * params. - */ - fid2 = H5Fcreate(FILE2, H5F_ACC_TRUNC, tmpl1, H5P_DEFAULT); - CHECK(fid2, FAIL, "H5Fcreate"); + /* Set the new file-creation parameters */ + ret = H5Pset_userblock(tmpl1, F2_USERBLOCK_SIZE); + CHECK(ret, FAIL, "H5Pset_userblock"); - /* Release file-creation template */ - ret = H5Pclose(tmpl1); - CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pset_sizes(tmpl1, (size_t)F2_OFFSET_SIZE, (size_t)F2_LENGTH_SIZE); + CHECK(ret, FAIL, "H5Pset_sizes"); - /* Make certain we can create a dataset properly in the file with the userblock */ - { - hid_t dataset_id, dataspace_id; /* identifiers */ - hsize_t dims[F2_RANK]; - unsigned data[F2_DIM0][F2_DIM1]; - unsigned i, j; + ret = H5Pset_sym_k(tmpl1, F2_SYM_INTERN_K, F2_SYM_LEAF_K); + CHECK(ret, FAIL, "H5Pset_sym_k"); - /* Create the data space for the dataset. */ - dims[0] = F2_DIM0; - dims[1] = F2_DIM1; - dataspace_id = H5Screate_simple(F2_RANK, dims, NULL); - CHECK(dataspace_id, FAIL, "H5Screate_simple"); + /* + * Try to create second file, with non-standard file-creation template + * params. + */ + fid2 = H5Fcreate(FILE2, H5F_ACC_TRUNC, tmpl1, H5P_DEFAULT); + CHECK(fid2, FAIL, "H5Fcreate"); - /* Create the dataset. */ - dataset_id = - H5Dcreate2(fid2, F2_DSET, H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - CHECK(dataset_id, FAIL, "H5Dcreate2"); + /* Release file-creation template */ + ret = H5Pclose(tmpl1); + CHECK(ret, FAIL, "H5Pclose"); - for (i = 0; i < F2_DIM0; i++) - for (j = 0; j < F2_DIM1; j++) - data[i][j] = i * 10 + j; + /* Make certain we can create a dataset properly in the file with the userblock */ + { + hid_t dataset_id, dataspace_id; /* identifiers */ + hsize_t dims[F2_RANK]; + unsigned data[F2_DIM0][F2_DIM1]; + unsigned i, j; + + /* Create the data space for the dataset. */ + dims[0] = F2_DIM0; + dims[1] = F2_DIM1; + dataspace_id = H5Screate_simple(F2_RANK, dims, NULL); + CHECK(dataspace_id, FAIL, "H5Screate_simple"); + + /* Create the dataset. */ + dataset_id = H5Dcreate2(fid2, F2_DSET, H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + CHECK(dataset_id, FAIL, "H5Dcreate2"); + + for (i = 0; i < F2_DIM0; i++) + for (j = 0; j < F2_DIM1; j++) + data[i][j] = i * 10 + j; + + /* Write data to the new dataset */ + ret = H5Dwrite(dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + CHECK(ret, FAIL, "H5Dwrite"); + + /* End access to the dataset and release resources used by it. */ + ret = H5Dclose(dataset_id); + CHECK(ret, FAIL, "H5Dclose"); - /* Write data to the new dataset */ - ret = H5Dwrite(dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - CHECK(ret, FAIL, "H5Dwrite"); + /* Terminate access to the data space. */ + ret = H5Sclose(dataspace_id); + CHECK(ret, FAIL, "H5Sclose"); + } - /* End access to the dataset and release resources used by it. */ - ret = H5Dclose(dataset_id); - CHECK(ret, FAIL, "H5Dclose"); + /* Get the file-creation template */ + tmpl1 = H5Fget_create_plist(fid2); + CHECK(tmpl1, FAIL, "H5Fget_create_plist"); - /* Terminate access to the data space. */ - ret = H5Sclose(dataspace_id); - CHECK(ret, FAIL, "H5Sclose"); - } + /* Get the file-creation parameters */ + ret = H5Pget_userblock(tmpl1, &ublock); + CHECK(ret, FAIL, "H5Pget_userblock"); + VERIFY(ublock, F2_USERBLOCK_SIZE, "H5Pget_userblock"); - /* Get the file-creation template */ - tmpl1 = H5Fget_create_plist(fid2); - CHECK(tmpl1, FAIL, "H5Fget_create_plist"); + ret = H5Pget_sizes(tmpl1, &parm, &parm2); + CHECK(ret, FAIL, "H5Pget_sizes"); + VERIFY(parm, F2_OFFSET_SIZE, "H5Pget_sizes"); + VERIFY(parm2, F2_LENGTH_SIZE, "H5Pget_sizes"); - /* Get the file-creation parameters */ - ret = H5Pget_userblock(tmpl1, &ublock); - CHECK(ret, FAIL, "H5Pget_userblock"); - VERIFY(ublock, F2_USERBLOCK_SIZE, "H5Pget_userblock"); + ret = H5Pget_sym_k(tmpl1, &iparm, &iparm2); + CHECK(ret, FAIL, "H5Pget_sym_k"); + VERIFY(iparm, F2_SYM_INTERN_K, "H5Pget_sym_k"); + VERIFY(iparm2, F2_SYM_LEAF_K, "H5Pget_sym_k"); - ret = H5Pget_sizes(tmpl1, &parm, &parm2); - CHECK(ret, FAIL, "H5Pget_sizes"); - VERIFY(parm, F2_OFFSET_SIZE, "H5Pget_sizes"); - VERIFY(parm2, F2_LENGTH_SIZE, "H5Pget_sizes"); + /* Clone the file-creation template */ + tmpl2 = H5Pcopy(tmpl1); + CHECK(tmpl2, FAIL, "H5Pcopy"); - ret = H5Pget_sym_k(tmpl1, &iparm, &iparm2); - CHECK(ret, FAIL, "H5Pget_sym_k"); - VERIFY(iparm, F2_SYM_INTERN_K, "H5Pget_sym_k"); - VERIFY(iparm2, F2_SYM_LEAF_K, "H5Pget_sym_k"); + /* Release file-creation template */ + ret = H5Pclose(tmpl1); + CHECK(ret, FAIL, "H5Pclose"); - /* Clone the file-creation template */ - tmpl2 = H5Pcopy(tmpl1); - CHECK(tmpl2, FAIL, "H5Pcopy"); + /* Set the new file-creation parameter */ + ret = H5Pset_userblock(tmpl2, F3_USERBLOCK_SIZE); + CHECK(ret, FAIL, "H5Pset_userblock"); - /* Release file-creation template */ - ret = H5Pclose(tmpl1); - CHECK(ret, FAIL, "H5Pclose"); + /* + * Try to create second file, with non-standard file-creation template + * params + */ + fid3 = H5Fcreate(FILE3, H5F_ACC_TRUNC, tmpl2, H5P_DEFAULT); + CHECK(fid3, FAIL, "H5Fcreate"); - /* Set the new file-creation parameter */ - ret = H5Pset_userblock(tmpl2, F3_USERBLOCK_SIZE); - CHECK(ret, FAIL, "H5Pset_userblock"); + /* Release file-creation template */ + ret = H5Pclose(tmpl2); + CHECK(ret, FAIL, "H5Pclose"); - /* - * Try to create second file, with non-standard file-creation template - * params - */ - fid3 = H5Fcreate(FILE3, H5F_ACC_TRUNC, tmpl2, H5P_DEFAULT); - CHECK(fid3, FAIL, "H5Fcreate"); + /* Get the file-creation template */ + tmpl1 = H5Fget_create_plist(fid3); + CHECK(tmpl1, FAIL, "H5Fget_create_plist"); - /* Release file-creation template */ - ret = H5Pclose(tmpl2); - CHECK(ret, FAIL, "H5Pclose"); + /* Get the file-creation parameters */ + ret = H5Pget_userblock(tmpl1, &ublock); + CHECK(ret, FAIL, "H5Pget_userblock"); + VERIFY(ublock, F3_USERBLOCK_SIZE, "H5Pget_userblock"); - /* Get the file-creation template */ - tmpl1 = H5Fget_create_plist(fid3); - CHECK(tmpl1, FAIL, "H5Fget_create_plist"); + ret = H5Pget_sizes(tmpl1, &parm, &parm2); + CHECK(ret, FAIL, "H5Pget_sizes"); + VERIFY(parm, F3_OFFSET_SIZE, "H5Pget_sizes"); + VERIFY(parm2, F3_LENGTH_SIZE, "H5Pget_sizes"); - /* Get the file-creation parameters */ - ret = H5Pget_userblock(tmpl1, &ublock); - CHECK(ret, FAIL, "H5Pget_userblock"); - VERIFY(ublock, F3_USERBLOCK_SIZE, "H5Pget_userblock"); + ret = H5Pget_sym_k(tmpl1, &iparm, &iparm2); + CHECK(ret, FAIL, "H5Pget_sym_k"); + VERIFY(iparm, F3_SYM_INTERN_K, "H5Pget_sym_k"); + VERIFY(iparm2, F3_SYM_LEAF_K, "H5Pget_sym_k"); - ret = H5Pget_sizes(tmpl1, &parm, &parm2); - CHECK(ret, FAIL, "H5Pget_sizes"); - VERIFY(parm, F3_OFFSET_SIZE, "H5Pget_sizes"); - VERIFY(parm2, F3_LENGTH_SIZE, "H5Pget_sizes"); + /* Release file-creation template */ + ret = H5Pclose(tmpl1); + CHECK(ret, FAIL, "H5Pclose"); - ret = H5Pget_sym_k(tmpl1, &iparm, &iparm2); - CHECK(ret, FAIL, "H5Pget_sym_k"); - VERIFY(iparm, F3_SYM_INTERN_K, "H5Pget_sym_k"); - VERIFY(iparm2, F3_SYM_LEAF_K, "H5Pget_sym_k"); + /* Close second file */ + ret = H5Fclose(fid2); + CHECK(ret, FAIL, "H5Fclose"); - /* Release file-creation template */ - ret = H5Pclose(tmpl1); - CHECK(ret, FAIL, "H5Pclose"); + /* Close third file */ + ret = H5Fclose(fid3); + CHECK(ret, FAIL, "H5Fclose"); + } /* Close first file */ ret = H5Fclose(fid1); CHECK(ret, FAIL, "H5Fclose"); - - /* Close second file */ - ret = H5Fclose(fid2); - CHECK(ret, FAIL, "H5Fclose"); - - /* Close third file */ - ret = H5Fclose(fid3); - CHECK(ret, FAIL, "H5Fclose"); } /* test_file_create() */ /**************************************************************** @@ -492,7 +499,7 @@ test_file_create(void) ** ****************************************************************/ static void -test_file_open(void) +test_file_open(const char *env_h5_drvr) { hid_t fid1, fid2; /*HDF5 File IDs */ hid_t did; /*dataset ID */ @@ -510,6 +517,10 @@ test_file_open(void) * Test single file open */ + /* Only run this test with sec2/default driver */ + if (!h5_using_default_driver(env_h5_drvr)) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing Low-Level File Opening I/O\n")); @@ -1619,22 +1630,24 @@ test_file_perm2(void) ** H5Fis_accessible() API call. ** *****************************************************************/ -#define FILE_IS_ACCESSIBLE "tfile_is_accessible" +#define FILE_IS_ACCESSIBLE "tfile_is_accessible" +#define FILE_IS_ACCESSIBLE_NON_HDF5 "tfile_is_accessible_non_hdf5" static void test_file_is_accessible(const char *env_h5_drvr) { - hid_t fid = H5I_INVALID_HID; /* File opened with read-write permission */ - hid_t fcpl_id = H5I_INVALID_HID; /* File creation property list */ - hid_t fapl_id = H5I_INVALID_HID; /* File access property list */ - int fd; /* POSIX file descriptor */ - char filename[FILENAME_LEN]; /* Filename to use */ - char sb_filename[FILENAME_LEN]; /* Name of file w/ superblock */ - ssize_t nbytes; /* Number of bytes written */ - unsigned u; /* Local index variable */ - unsigned char buf[1024]; /* Buffer of data to write */ - htri_t is_hdf5; /* Whether a file is an HDF5 file */ - int posix_ret; /* Return value from POSIX calls */ - herr_t ret; /* Return value from HDF5 calls */ + hid_t fid = H5I_INVALID_HID; /* File opened with read-write permission */ + hid_t fcpl_id = H5I_INVALID_HID; /* File creation property list */ + hid_t fapl_id = H5I_INVALID_HID; /* File access property list */ + int fd; /* POSIX file descriptor */ + char filename[FILENAME_LEN]; /* Filename to use */ + char non_hdf5_filename[FILENAME_LEN]; /* Base name of non-hdf5 file */ + char non_hdf5_sb_filename[FILENAME_LEN]; /* Name of non-hdf5 superblock file */ + ssize_t nbytes; /* Number of bytes written */ + unsigned u; /* Local index variable */ + unsigned char buf[1024]; /* Buffer of data to write */ + htri_t is_hdf5; /* Whether a file is an HDF5 file */ + int posix_ret; /* Return value from POSIX calls */ + herr_t ret; /* Return value from HDF5 calls */ /* Output message about test being performed */ MESSAGE(5, ("Testing Detection of HDF5 Files\n")); @@ -1643,13 +1656,11 @@ test_file_is_accessible(const char *env_h5_drvr) fapl_id = h5_fileaccess(); CHECK(fapl_id, H5I_INVALID_HID, "H5Pcreate"); - /* Fix up filenames - * For VFDs that create multiple files, we also need the name - * of the file with the superblock. With single-file VFDs, this - * will be equal to the one from h5_fixname(). - */ + /* Fix up filenames */ h5_fixname(FILE_IS_ACCESSIBLE, fapl_id, filename, sizeof(filename)); - h5_fixname_superblock(FILE_IS_ACCESSIBLE, fapl_id, sb_filename, sizeof(filename)); + h5_fixname(FILE_IS_ACCESSIBLE_NON_HDF5, fapl_id, non_hdf5_filename, sizeof(non_hdf5_filename)); + h5_fixname_superblock(FILE_IS_ACCESSIBLE_NON_HDF5, fapl_id, non_hdf5_sb_filename, + sizeof(non_hdf5_sb_filename)); /****************/ /* Normal usage */ @@ -1727,48 +1738,54 @@ test_file_is_accessible(const char *env_h5_drvr) VERIFY(is_hdf5, TRUE, "H5Fis_accessible"); } /* end if */ - /***********************/ - /* EMPTY non-HDF5 file */ - /***********************/ + /* Don't run the below tests for drivers that use multiple + * logical files, like the splitter driver. + */ + if (!h5_driver_uses_multiple_files(env_h5_drvr, H5_EXCLUDE_MULTIPART_DRIVERS)) { + /***********************/ + /* EMPTY non-HDF5 file */ + /***********************/ - /* Create non-HDF5 file and check it */ - fd = HDopen(sb_filename, O_RDWR | O_CREAT | O_TRUNC, H5_POSIX_CREATE_MODE_RW); - CHECK(fd, (-1), "HDopen"); + /* Create non-HDF5 file and check it */ + fd = HDopen(non_hdf5_sb_filename, O_RDWR | O_CREAT | O_TRUNC, H5_POSIX_CREATE_MODE_RW); + CHECK(fd, (-1), "HDopen"); - /* Close the file */ - posix_ret = HDclose(fd); - CHECK(posix_ret, (-1), "HDclose"); + /* Close the file */ + posix_ret = HDclose(fd); + CHECK(posix_ret, (-1), "HDclose"); - /* Verify that the file is NOT an HDF5 file */ - is_hdf5 = H5Fis_accessible(filename, fapl_id); - VERIFY(is_hdf5, FALSE, "H5Fis_accessible (empty non-HDF5 file)"); + /* Verify that the file is NOT an HDF5 file using the base filename */ + is_hdf5 = H5Fis_accessible(non_hdf5_filename, fapl_id); + VERIFY(is_hdf5, FALSE, "H5Fis_accessible (empty non-HDF5 file)"); - /***************************/ - /* Non-empty non-HDF5 file */ - /***************************/ + /***************************/ + /* Non-empty non-HDF5 file */ + /***************************/ - /* Create non-HDF5 file and check it */ - fd = HDopen(sb_filename, O_RDWR | O_CREAT | O_TRUNC, H5_POSIX_CREATE_MODE_RW); - CHECK(fd, (-1), "HDopen"); + /* Create non-HDF5 file and check it */ + fd = HDopen(non_hdf5_sb_filename, O_RDWR | O_CREAT | O_TRUNC, H5_POSIX_CREATE_MODE_RW); + CHECK(fd, (-1), "HDopen"); - /* Initialize information to write */ - for (u = 0; u < 1024; u++) - buf[u] = (unsigned char)u; + /* Initialize information to write */ + for (u = 0; u < 1024; u++) + buf[u] = (unsigned char)u; - /* Write some information */ - nbytes = HDwrite(fd, buf, (size_t)1024); - VERIFY(nbytes, 1024, "HDwrite"); + /* Write some information */ + nbytes = HDwrite(fd, buf, (size_t)1024); + VERIFY(nbytes, 1024, "HDwrite"); - /* Close the file */ - posix_ret = HDclose(fd); - CHECK(posix_ret, (-1), "HDclose"); + /* Close the file */ + posix_ret = HDclose(fd); + CHECK(posix_ret, (-1), "HDclose"); - /* Verify that the file is not an HDF5 file */ - is_hdf5 = H5Fis_accessible(filename, fapl_id); - VERIFY(is_hdf5, FALSE, "H5Fis_accessible (non-HDF5 file)"); + /* Verify that the file is not an HDF5 file */ + is_hdf5 = H5Fis_accessible(non_hdf5_filename, fapl_id); + VERIFY(is_hdf5, FALSE, "H5Fis_accessible (non-HDF5 file)"); + } /* Clean up files */ h5_delete_test_file(filename, fapl_id); + h5_delete_test_file(non_hdf5_filename, fapl_id); /* Close property list */ ret = H5Pclose(fapl_id); @@ -1780,7 +1797,7 @@ test_file_is_accessible(const char *env_h5_drvr) ** ** test_file_ishdf5(): low-level file test routine. ** This test checks whether the H5Fis_hdf5() routine is working -** correctly in variuous situations. +** correctly in various situations. ** *****************************************************************/ #ifndef H5_NO_DEPRECATED_SYMBOLS @@ -1800,6 +1817,9 @@ test_file_ishdf5(const char *env_h5_drvr) int posix_ret; /* Return value from POSIX calls */ herr_t ret; /* Return value from HDF5 calls */ + if (!h5_using_default_driver(env_h5_drvr)) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing Detection of HDF5 Files (using deprecated H5Fis_hdf5() call)\n")); @@ -1835,33 +1855,28 @@ test_file_ishdf5(const char *env_h5_drvr) /* Non-default user block size */ /*******************************/ - /* This test is not currently working for the family VFD. - * There are failures when creating files with userblocks. - */ - if (0 != HDstrcmp(env_h5_drvr, "family")) { - /* Create a file creation property list with a non-default user block size */ - fcpl_id = H5Pcreate(H5P_FILE_CREATE); - CHECK(fcpl_id, H5I_INVALID_HID, "H5Pcreate"); + /* Create a file creation property list with a non-default user block size */ + fcpl_id = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl_id, H5I_INVALID_HID, "H5Pcreate"); - ret = H5Pset_userblock(fcpl_id, (hsize_t)2048); - CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_userblock(fcpl_id, (hsize_t)2048); + CHECK(ret, FAIL, "H5Pset_userblock"); - /* Create file with non-default user block */ - fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); - CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); + /* Create file with non-default user block */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); + CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); - /* Release file creation property list */ - ret = H5Pclose(fcpl_id); - CHECK(ret, FAIL, "H5Pclose"); + /* Release file creation property list */ + ret = H5Pclose(fcpl_id); + CHECK(ret, FAIL, "H5Pclose"); - /* Close file */ - ret = H5Fclose(fid); - CHECK(ret, FAIL, "H5Fclose"); + /* Close file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); - /* Verify that the file is an HDF5 file */ - is_hdf5 = H5Fis_hdf5(sb_filename); - VERIFY(is_hdf5, TRUE, "H5Fis_hdf5"); - } /* end if */ + /* Verify that the file is an HDF5 file */ + is_hdf5 = H5Fis_hdf5(sb_filename); + VERIFY(is_hdf5, TRUE, "H5Fis_hdf5"); /***************************/ /* Non-empty non-HDF5 file */ @@ -2001,7 +2016,7 @@ test_file_delete(hid_t fapl_id) ** ** test_file_open_dot(): low-level file test routine. ** This test checks whether opening objects with "." for a name -** works correctly in variuous situations. +** works correctly in various situations. ** *****************************************************************/ static void @@ -2383,7 +2398,7 @@ test_file_double_root_open(void) grp2_id = H5Gopen2(file2_id, "/", H5P_DEFAULT); CHECK(grp2_id, FAIL, "H5Gopen2"); - /* Note "assymetric" close order */ + /* Note "asymmetric" close order */ ret = H5Gclose(grp1_id); CHECK(ret, FAIL, "H5Gclose"); ret = H5Gclose(grp2_id); @@ -2422,7 +2437,7 @@ test_file_double_group_open(void) grp2_id = H5Gopen2(file2_id, GRP_NAME, H5P_DEFAULT); CHECK(grp2_id, FAIL, "H5Gopen2"); - /* Note "assymetric" close order */ + /* Note "asymmetric" close order */ ret = H5Gclose(grp1_id); CHECK(ret, FAIL, "H5Gclose"); ret = H5Gclose(grp2_id); @@ -2471,7 +2486,7 @@ test_file_double_dataset_open(void) ret = H5Sclose(space_id); CHECK(ret, FAIL, "H5Sclose"); - /* Note "assymetric" close order */ + /* Note "asymmetric" close order */ ret = H5Dclose(dset1_id); CHECK(ret, FAIL, "H5Dclose"); ret = H5Dclose(dset2_id); @@ -2872,7 +2887,7 @@ test_file_double_datatype_open(void) type2_id = H5Topen2(file2_id, TYPE_NAME, H5P_DEFAULT); CHECK(type2_id, FAIL, "H5Topen2"); - /* Note "assymetric" close order */ + /* Note "asymmetric" close order */ ret = H5Tclose(type1_id); CHECK(ret, FAIL, "H5Tclose"); ret = H5Tclose(type2_id); @@ -2896,7 +2911,7 @@ test_file_double_datatype_open(void) ** *****************************************************************/ static void -test_userblock_file_size(void) +test_userblock_file_size(const char *env_h5_drvr) { hid_t file1_id, file2_id; hid_t group1_id, group2_id; @@ -2908,6 +2923,11 @@ test_userblock_file_size(void) unsigned long fileno1, fileno2; /* File number */ herr_t ret; /* Generic return value */ + /* Don't run with multi/split, family or direct drivers */ + if (!HDstrcmp(env_h5_drvr, "multi") || !HDstrcmp(env_h5_drvr, "split") || + !HDstrcmp(env_h5_drvr, "family") || !HDstrcmp(env_h5_drvr, "direct")) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing file size with user block\n")); @@ -2917,7 +2937,7 @@ test_userblock_file_size(void) ret = H5Pset_userblock(fcpl2_id, USERBLOCK_SIZE); CHECK(ret, FAIL, "H5Pset_userblock"); - /* Create files. Onyl file2 with have a userblock. */ + /* Create files. Only file2 with have a userblock. */ file1_id = H5Fcreate(FILE1, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(file1_id, FAIL, "H5Fcreate"); file2_id = H5Fcreate(FILE2, H5F_ACC_TRUNC, fcpl2_id, H5P_DEFAULT); @@ -3294,13 +3314,17 @@ test_userblock_alignment_helper2(hid_t fapl, hbool_t open_rw) ** *****************************************************************/ static void -test_userblock_alignment(void) +test_userblock_alignment(const char *env_h5_drvr) { hid_t fid; /* File ID */ hid_t fcpl; /* File creation property list ID */ hid_t fapl; /* File access property list ID */ herr_t ret; /* Generic return value */ + /* Only run with sec2 driver */ + if (!h5_using_default_driver(env_h5_drvr)) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing that non-zero userblocks and object alignment interact correctly.\n")); @@ -3512,13 +3536,17 @@ test_userblock_alignment(void) ** *****************************************************************/ static void -test_userblock_alignment_paged(void) +test_userblock_alignment_paged(const char *env_h5_drvr) { hid_t fid; /* File ID */ hid_t fcpl; /* File creation property list ID */ hid_t fapl; /* File access property list ID */ herr_t ret; /* Generic return value */ + /* Only run with sec2 driver */ + if (!h5_using_default_driver(env_h5_drvr)) + return; + /* Output message about test being performed */ MESSAGE(5, ("Testing interaction between userblock and alignment (via paged aggregation and " "H5Pset_alignment)\n")); @@ -3838,7 +3866,7 @@ test_userblock_alignment_paged(void) * Strategy is H5F_FILE_SPACE_NONE; fsp_size = 1024 * H5Pset_alignment() is 16 * Outcome: - * Should succed: + * Should succeed: * userblock (512) is integral multiple of alignment (16) */ /* Create file creation property list with user block */ @@ -3877,7 +3905,7 @@ test_userblock_alignment_paged(void) * H5Pset_alignment() is 3 * Reopen the file; H5Pset_alignment() is 1024 * Outcome: - * Should succed: + * Should succeed: * Userblock (512) is the same as alignment (512); * The H5Pset_alignment() calls have no effect */ @@ -3946,7 +3974,7 @@ test_filespace_info(const char *env_h5_drvr) hsize_t fs_threshold; /* Free-space section threshold--iteration variable */ hsize_t fsp_size; /* File space page size */ char filename[FILENAME_LEN]; /* Filename to use */ - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contiguous address space */ herr_t ret; /* Return value */ /* Output message about test being performed */ @@ -5188,7 +5216,7 @@ test_filespace_1_10_0_compatible(void) ** Verify that the trunk can open, read and modify these files-- ** 1) They are initially created (via gen_filespace.c) in the trunk ** with combinations of file space strategies, default/non-default -** threshold, and file spacing paging enabled/disbled. +** threshold, and file spacing paging enabled/disabled. ** The library creates the file space info message with ** "mark if unknown" in these files. ** 2) They are copied to the 1.8 branch, and are opened/read/modified @@ -5575,7 +5603,10 @@ test_libver_bounds(void) test_libver_bounds_real(H5F_LIBVER_EARLIEST, 1, H5F_LIBVER_LATEST, 2); test_libver_bounds_real(H5F_LIBVER_LATEST, 2, H5F_LIBVER_EARLIEST, 2); test_libver_bounds_open(); - test_libver_bounds_copy(); + + if (!h5_driver_uses_modified_filename()) { + test_libver_bounds_copy(); + } } /* end test_libver_bounds() */ /************************************************************************************** @@ -5600,7 +5631,7 @@ test_libver_bounds(void) ** **************************************************************************************/ static void -test_libver_bounds_low_high(void) +test_libver_bounds_low_high(const char *env_h5_drvr) { hid_t fapl = H5I_INVALID_HID; /* File access property list */ H5F_libver_t low, high; /* Low and high bounds */ @@ -5640,7 +5671,7 @@ test_libver_bounds_low_high(void) VERIFY(ret, SUCCEED, "H5Pset_libver_bounds"); /* Tests to verify version bounds */ - test_libver_bounds_super(fapl); + test_libver_bounds_super(fapl, env_h5_drvr); test_libver_bounds_obj(fapl); test_libver_bounds_dataset(fapl); test_libver_bounds_dataspace(fapl); @@ -5673,7 +5704,7 @@ test_libver_bounds_low_high(void) ** *************************************************************************/ static void -test_libver_bounds_super(hid_t fapl) +test_libver_bounds_super(hid_t fapl, const char *env_h5_drvr) { hid_t fcpl = H5I_INVALID_HID; /* File creation property list */ herr_t ret; /* The return value */ @@ -5685,12 +5716,14 @@ test_libver_bounds_super(hid_t fapl) /* Verify superblock version when creating a file with input fapl, fcpl #A and with/without SWMR access */ - test_libver_bounds_super_create(fapl, fcpl, TRUE, FALSE); + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_create(fapl, fcpl, TRUE, FALSE); test_libver_bounds_super_create(fapl, fcpl, FALSE, FALSE); /* Verify superblock version when opening a file which is created with input fapl, fcpl #A and with/without SWMR access */ - test_libver_bounds_super_open(fapl, fcpl, TRUE, FALSE); + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_open(fapl, fcpl, TRUE, FALSE); test_libver_bounds_super_open(fapl, fcpl, FALSE, FALSE); /* Close the fcpl */ @@ -5706,12 +5739,14 @@ test_libver_bounds_super(hid_t fapl) /* Verify superblock version when creating a file with input fapl, fcpl #B and with/without SWMR access */ - test_libver_bounds_super_create(fapl, fcpl, TRUE, FALSE); + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_create(fapl, fcpl, TRUE, FALSE); test_libver_bounds_super_create(fapl, fcpl, FALSE, FALSE); /* Verify superblock version when opening a file which is created with input fapl, fcpl #B and with/without SWMR access */ - test_libver_bounds_super_open(fapl, fcpl, TRUE, FALSE); + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_open(fapl, fcpl, TRUE, FALSE); test_libver_bounds_super_open(fapl, fcpl, FALSE, FALSE); /* Close the fcpl */ @@ -5729,38 +5764,44 @@ test_libver_bounds_super(hid_t fapl) /* Verify superblock version when creating a file with input fapl, fcpl #C and with/without SWMR access */ - test_libver_bounds_super_create(fapl, fcpl, TRUE, FALSE); + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_create(fapl, fcpl, TRUE, FALSE); test_libver_bounds_super_create(fapl, fcpl, FALSE, FALSE); /* Verify superblock version when opening a file which is created with input fapl, fcpl #C and with/without SWMR access */ - test_libver_bounds_super_open(fapl, fcpl, TRUE, FALSE); + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_open(fapl, fcpl, TRUE, FALSE); test_libver_bounds_super_open(fapl, fcpl, FALSE, FALSE); /* Close the fcpl */ ret = H5Pclose(fcpl); CHECK(ret, FAIL, "H5Pclose"); - /* Create a fcpl with persistent free-space manager enabled: #D */ - /* This will result in superblock version 2 */ - fcpl = H5Pcreate(H5P_FILE_CREATE); - CHECK(fcpl, H5I_INVALID_HID, "H5Pcreate"); - ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, 1, (hsize_t)1); - CHECK(ret, FAIL, "H5Pset_file_space"); - - /* Verify superblock version when creating a file with input fapl, - fcpl #D and with/without SWMR access */ - test_libver_bounds_super_create(fapl, fcpl, TRUE, TRUE); - test_libver_bounds_super_create(fapl, fcpl, FALSE, TRUE); - - /* Verify superblock version when opening a file which is created - with input fapl, fcpl #D and with/without SWMR access */ - test_libver_bounds_super_open(fapl, fcpl, TRUE, TRUE); - test_libver_bounds_super_open(fapl, fcpl, FALSE, TRUE); - - /* Close the fcpl */ - ret = H5Pclose(fcpl); - CHECK(ret, FAIL, "H5Pclose"); + if (h5_using_default_driver(env_h5_drvr)) { + /* Create a fcpl with persistent free-space manager enabled: #D */ + /* This will result in superblock version 2 */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, H5I_INVALID_HID, "H5Pcreate"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, 1, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space"); + + /* Verify superblock version when creating a file with input fapl, + fcpl #D and with/without SWMR access */ + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_create(fapl, fcpl, TRUE, TRUE); + test_libver_bounds_super_create(fapl, fcpl, FALSE, TRUE); + + /* Verify superblock version when opening a file which is created + with input fapl, fcpl #D and with/without SWMR access */ + if (H5FD__supports_swmr_test(env_h5_drvr)) + test_libver_bounds_super_open(fapl, fcpl, TRUE, TRUE); + test_libver_bounds_super_open(fapl, fcpl, FALSE, TRUE); + + /* Close the fcpl */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + } } /* end test_libver_bounds_super() */ @@ -6349,7 +6390,7 @@ test_libver_bounds_dataset(hid_t fapl) VERIFY(dset->shared->dcpl_cache.fill.version, H5O_fill_ver_bounds[low], "H5O_fill_ver_bounds"); } - /* Verify filter pipleline message version */ + /* Verify filter pipeline message version */ VERIFY(dset->shared->dcpl_cache.pline.version, H5O_pline_ver_bounds[low], "H5O_pline_ver_bounds"); /* Close the dataset */ @@ -6402,7 +6443,7 @@ test_libver_bounds_dataset(hid_t fapl) ret = H5Sclose(sid); CHECK(ret, FAIL, "H5Sclose"); - /* Close the datset creation property list */ + /* Close the dataset creation property list */ ret = H5Pclose(dcpl); CHECK(ret, FAIL, "H5Pclose"); @@ -6422,7 +6463,7 @@ test_libver_bounds_dataset(hid_t fapl) /* Loop through all the combinations of low/high bounds in new_fapl */ /* Open the file with the fapl and create the chunked dataset */ - /* Verify the dataset's layout, fill value and filter pipleline message versions */ + /* Verify the dataset's layout, fill value and filter pipeline message versions */ for (low = H5F_LIBVER_EARLIEST; low < H5F_LIBVER_NBOUNDS; low++) { for (high = H5F_LIBVER_EARLIEST; high < H5F_LIBVER_NBOUNDS; high++) { H5E_BEGIN_TRY @@ -6527,7 +6568,7 @@ test_libver_bounds_dataset(hid_t fapl) ** new fapl: ** --Open the same file in (a) with the fapl ** --Create a chunked dataset, a compact dataset and -** a contigous dataset +** a contiguous dataset ** --Verify the dataspace message version for these ** three datasets ** --Delete the three datasets and the dataspaces @@ -6876,7 +6917,7 @@ static void test_libver_bounds_datatype_check(hid_t fapl, hid_t tid) { hid_t fid = H5I_INVALID_HID; /* File ID */ - hid_t new_fapl = H5I_INVALID_HID; /* File acess property list */ + hid_t new_fapl = H5I_INVALID_HID; /* File access property list */ hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ hid_t dtid = H5I_INVALID_HID; /* Datatype ID for the dataset */ hid_t str_tid = H5I_INVALID_HID; /* String datatype ID */ @@ -7136,7 +7177,7 @@ test_libver_bounds_attributes(hid_t fapl) fid = H5Fcreate(FILE8, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); - /* Integer datatpye */ + /* Integer datatype */ tid = H5Tcopy(H5T_NATIVE_INT); CHECK(tid, H5I_INVALID_HID, "H5Tcopy"); @@ -7245,7 +7286,7 @@ test_libver_bounds_attributes(hid_t fapl) fid = H5Fcreate(FILE8, H5F_ACC_TRUNC, fcpl, fapl); CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); - /* Create an integer datatye */ + /* Create an integer datatype */ tid = H5Tcopy(H5T_NATIVE_INT); CHECK(tid, H5I_INVALID_HID, "H5Tcopy"); @@ -7752,7 +7793,7 @@ test_min_dset_ohdr(void) ****************************************************************/ #ifndef H5_NO_DEPRECATED_SYMBOLS static void -test_deprec(void) +test_deprec(const char *env_h5_drvr) { hid_t file; /* File IDs for old & new files */ hid_t fcpl; /* File creation property list */ @@ -7805,166 +7846,169 @@ test_deprec(void) ret = H5Fclose(file); CHECK(ret, FAIL, "H5Fclose"); - /* Create a file creation property list */ - fcpl = H5Pcreate(H5P_FILE_CREATE); - CHECK(fcpl, FAIL, "H5Pcreate"); + /* Only run this part of the test with the sec2/default driver */ + if (h5_using_default_driver(env_h5_drvr)) { + /* Create a file creation property list */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); - /* Set a property in the FCPL that will push the superblock version up */ - ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0); - ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); - CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + /* Set a property in the FCPL that will push the superblock version up */ + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); - fapl = H5Pcreate(H5P_FILE_ACCESS); - ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1024); - CHECK(ret, FAIL, "H5Pset_alignment"); + fapl = H5Pcreate(H5P_FILE_ACCESS); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Creating a file with the non-default file creation property list should - * create a version 2 superblock - */ + /* Creating a file with the non-default file creation property list should + * create a version 2 superblock + */ - /* Create file with custom file creation property list */ - file = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); - CHECK(file, FAIL, "H5Fcreate"); + /* Create file with custom file creation property list */ + file = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + CHECK(file, FAIL, "H5Fcreate"); - new_fapl = H5Fget_access_plist(file); - H5Pget_alignment(new_fapl, NULL, &align); + new_fapl = H5Fget_access_plist(file); + H5Pget_alignment(new_fapl, NULL, &align); - /* Close FCPL */ - ret = H5Pclose(fcpl); - CHECK(ret, FAIL, "H5Pclose"); + /* Close FCPL */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Get the file's version information */ - ret = H5Fget_info1(file, &finfo); - CHECK(ret, FAIL, "H5Fget_info1"); - VERIFY(finfo.super_ext_size, 152, "H5Fget_info1"); - VERIFY(finfo.sohm.hdr_size, 0, "H5Fget_info1"); - VERIFY(finfo.sohm.msgs_info.index_size, 0, "H5Fget_info1"); - VERIFY(finfo.sohm.msgs_info.heap_size, 0, "H5Fget_info1"); + /* Get the file's version information */ + ret = H5Fget_info1(file, &finfo); + CHECK(ret, FAIL, "H5Fget_info1"); + VERIFY(finfo.super_ext_size, 152, "H5Fget_info1"); + VERIFY(finfo.sohm.hdr_size, 0, "H5Fget_info1"); + VERIFY(finfo.sohm.msgs_info.index_size, 0, "H5Fget_info1"); + VERIFY(finfo.sohm.msgs_info.heap_size, 0, "H5Fget_info1"); - /* Get the file's dataset creation property list */ - fcpl = H5Fget_create_plist(file); - CHECK(fcpl, FAIL, "H5Fget_create_plist"); + /* Get the file's dataset creation property list */ + fcpl = H5Fget_create_plist(file); + CHECK(fcpl, FAIL, "H5Fget_create_plist"); - /* Get the file's version information */ - ret = H5Pget_version(fcpl, &super, &freelist, &stab, &shhdr); - CHECK(ret, FAIL, "H5Pget_version"); - VERIFY(super, 2, "H5Pget_version"); - VERIFY(freelist, 0, "H5Pget_version"); - VERIFY(stab, 0, "H5Pget_version"); - VERIFY(shhdr, 0, "H5Pget_version"); + /* Get the file's version information */ + ret = H5Pget_version(fcpl, &super, &freelist, &stab, &shhdr); + CHECK(ret, FAIL, "H5Pget_version"); + VERIFY(super, 2, "H5Pget_version"); + VERIFY(freelist, 0, "H5Pget_version"); + VERIFY(stab, 0, "H5Pget_version"); + VERIFY(shhdr, 0, "H5Pget_version"); - /* Close FCPL */ - ret = H5Pclose(fcpl); - CHECK(ret, FAIL, "H5Pclose"); + /* Close FCPL */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); + /* Close file */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); - /* Re-open the file */ - file = H5Fopen(FILE1, H5F_ACC_RDONLY, H5P_DEFAULT); - CHECK(file, FAIL, "H5Fcreate"); + /* Re-open the file */ + file = H5Fopen(FILE1, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(file, FAIL, "H5Fcreate"); - /* Get the file's version information */ - ret = H5Fget_info1(file, &finfo); - CHECK(ret, FAIL, "H5Fget_info1"); - VERIFY(finfo.super_ext_size, 152, "H5Fget_info1"); - VERIFY(finfo.sohm.hdr_size, 0, "H5Fget_info1"); - VERIFY(finfo.sohm.msgs_info.index_size, 0, "H5Fget_info1"); - VERIFY(finfo.sohm.msgs_info.heap_size, 0, "H5Fget_info1"); + /* Get the file's version information */ + ret = H5Fget_info1(file, &finfo); + CHECK(ret, FAIL, "H5Fget_info1"); + VERIFY(finfo.super_ext_size, 152, "H5Fget_info1"); + VERIFY(finfo.sohm.hdr_size, 0, "H5Fget_info1"); + VERIFY(finfo.sohm.msgs_info.index_size, 0, "H5Fget_info1"); + VERIFY(finfo.sohm.msgs_info.heap_size, 0, "H5Fget_info1"); - /* Get the file's creation property list */ - fcpl = H5Fget_create_plist(file); - CHECK(fcpl, FAIL, "H5Fget_create_plist"); + /* Get the file's creation property list */ + fcpl = H5Fget_create_plist(file); + CHECK(fcpl, FAIL, "H5Fget_create_plist"); - /* Get the file's version information */ - ret = H5Pget_version(fcpl, &super, &freelist, &stab, &shhdr); - CHECK(ret, FAIL, "H5Pget_version"); - VERIFY(super, 2, "H5Pget_version"); - VERIFY(freelist, 0, "H5Pget_version"); - VERIFY(stab, 0, "H5Pget_version"); - VERIFY(shhdr, 0, "H5Pget_version"); + /* Get the file's version information */ + ret = H5Pget_version(fcpl, &super, &freelist, &stab, &shhdr); + CHECK(ret, FAIL, "H5Pget_version"); + VERIFY(super, 2, "H5Pget_version"); + VERIFY(freelist, 0, "H5Pget_version"); + VERIFY(stab, 0, "H5Pget_version"); + VERIFY(shhdr, 0, "H5Pget_version"); - /* Close FCPL */ - ret = H5Pclose(fcpl); - CHECK(ret, FAIL, "H5Pclose"); + /* Close FCPL */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); + /* Close file */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); - { /* Test deprecated H5Pget/set_file_space() */ + { /* Test deprecated H5Pget/set_file_space() */ - H5F_file_space_type_t old_strategy; - hsize_t old_threshold; - hid_t fid; - hid_t ffcpl; + H5F_file_space_type_t old_strategy; + hsize_t old_threshold; + hid_t fid; + hid_t ffcpl; - fcpl = H5Pcreate(H5P_FILE_CREATE); - CHECK(fcpl, FAIL, "H5Pcreate"); + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); - ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); - VERIFY(old_strategy, H5F_FILE_SPACE_ALL, "H5Pget_file_space"); - VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL, "H5Pget_file_space"); + VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); - /* Set file space strategy and free space section threshold */ - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Set file space strategy and free space section threshold */ + ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); + CHECK(ret, FAIL, "H5Pget_file_space"); - /* Get the file space info from the creation property */ - ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); - VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); - VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); + /* Get the file space info from the creation property */ + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_DEFAULT, (hsize_t)3); - CHECK(ret, FAIL, "H5Pget_file_space"); + ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_DEFAULT, (hsize_t)3); + CHECK(ret, FAIL, "H5Pget_file_space"); - ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); - VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); - VERIFY(old_threshold, 3, "H5Pget_file_space"); + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); - /* Create a file */ - fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT); - CHECK(file, FAIL, "H5Fcreate"); + /* Create a file */ + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT); + CHECK(file, FAIL, "H5Fcreate"); - old_strategy = H5F_FILE_SPACE_DEFAULT; - old_threshold = 0; - ffcpl = H5Fget_create_plist(fid); - ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); - VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); - VERIFY(old_threshold, 3, "H5Pget_file_space"); + old_strategy = H5F_FILE_SPACE_DEFAULT; + old_threshold = 0; + ffcpl = H5Fget_create_plist(fid); + ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); - /* Close file */ - ret = H5Fclose(fid); - CHECK(ret, FAIL, "H5Fclose"); + /* Close file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); - ret = H5Pclose(ffcpl); - CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(ffcpl); + CHECK(ret, FAIL, "H5Pclose"); - ret = H5Pclose(fcpl); - CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Reopen the file */ - fid = H5Fopen(FILE1, H5F_ACC_RDONLY, H5P_DEFAULT); - CHECK(fid, FAIL, "H5Fcreate"); + /* Reopen the file */ + fid = H5Fopen(FILE1, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fcreate"); - old_strategy = H5F_FILE_SPACE_DEFAULT; - old_threshold = 0; - ffcpl = H5Fget_create_plist(fid); - ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); - VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); - VERIFY(old_threshold, 3, "H5Pget_file_space"); + old_strategy = H5F_FILE_SPACE_DEFAULT; + old_threshold = 0; + ffcpl = H5Fget_create_plist(fid); + ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); - ret = H5Pclose(ffcpl); - CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(ffcpl); + CHECK(ret, FAIL, "H5Pclose"); - ret = H5Fclose(fid); - CHECK(ret, FAIL, "H5Fclose"); + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + } } } /* test_deprec */ @@ -7978,15 +8022,16 @@ test_deprec(void) void test_file(void) { - const char *env_h5_drvr; /* File Driver value from environment */ - hid_t fapl_id = H5I_INVALID_HID; /* VFD-dependent fapl ID */ + const char *env_h5_drvr; /* File Driver value from environment */ + hid_t fapl_id = H5I_INVALID_HID; /* VFD-dependent fapl ID */ + hbool_t driver_uses_modified_filename = h5_driver_uses_modified_filename(); herr_t ret; /* Output message about test being performed */ MESSAGE(5, ("Testing Low-Level File I/O\n")); /* Get the VFD to use */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; @@ -7995,7 +8040,7 @@ test_file(void) CHECK(fapl_id, H5I_INVALID_HID, "h5_fileaccess"); test_file_create(); /* Test file creation(also creation templates)*/ - test_file_open(); /* Test file opening */ + test_file_open(env_h5_drvr); /* Test file opening */ test_file_reopen(); /* Test file reopening */ test_file_close(); /* Test file close behavior */ test_get_file_id(); /* Test H5Iget_file_id */ @@ -8013,14 +8058,19 @@ test_file(void) test_file_double_datatype_open(); /* Test opening same named datatype from two files works properly */ test_file_double_file_dataset_open(TRUE); test_file_double_file_dataset_open(FALSE); - test_userblock_file_size(); /* Tests that files created with a userblock have the correct size */ - test_cached_stab_info(); /* Tests that files are created with cached stab info in the superblock */ - test_rw_noupdate(); /* Test to ensure that RW permissions don't write the file unless dirtied */ - test_userblock_alignment(); /* Tests that files created with a userblock and alignment interact properly - */ - test_userblock_alignment_paged(); /* Tests files created with a userblock and alignment (via paged - aggregation) interact properly */ - test_filespace_info(env_h5_drvr); /* Test file creation public routines: */ + test_userblock_file_size( + env_h5_drvr); /* Tests that files created with a userblock have the correct size */ + test_cached_stab_info(); /* Tests that files are created with cached stab info in the superblock */ + + if (!driver_uses_modified_filename) { + test_rw_noupdate(); /* Test to ensure that RW permissions don't write the file unless dirtied */ + } + + test_userblock_alignment( + env_h5_drvr); /* Tests that files created with a userblock and alignment interact properly */ + test_userblock_alignment_paged(env_h5_drvr); /* Tests files created with a userblock and alignment (via + paged aggregation) interact properly */ + test_filespace_info(env_h5_drvr); /* Test file creation public routines: */ /* H5Pget/set_file_space_strategy() & H5Pget/set_file_space_page_size() */ /* Skipped testing for multi/split drivers */ test_file_freespace(env_h5_drvr); /* Test file public routine H5Fget_freespace() */ @@ -8032,19 +8082,25 @@ test_file(void) /* Setup for multi/split drivers are there already */ test_sects_freespace(env_h5_drvr, FALSE); /* Test file public routine H5Fget_free_sections() */ /* Skipped testing for multi/split drivers */ - test_filespace_compatible(); /* Test compatibility for file space management */ - test_filespace_round_compatible(); /* Testing file space compatibility for files from trunk to 1_8 to - trunk */ - test_filespace_1_10_0_compatible(); /* Testing file space compatibility for files from release 1.10.0 */ - test_libver_bounds(); /* Test compatibility for file space management */ - test_libver_bounds_low_high(); + + if (!driver_uses_modified_filename) { + test_filespace_compatible(); /* Test compatibility for file space management */ + + test_filespace_round_compatible(); /* Testing file space compatibility for files from trunk to 1_8 to + trunk */ + test_filespace_1_10_0_compatible(); /* Testing file space compatibility for files from release 1.10.0 + */ + } + + test_libver_bounds(); /* Test compatibility for file space management */ + test_libver_bounds_low_high(env_h5_drvr); test_libver_macros(); /* Test the macros for library version comparison */ test_libver_macros2(); /* Test the macros for library version comparison */ test_incr_filesize(); /* Test H5Fincrement_filesize() and H5Fget_eoa() */ - test_min_dset_ohdr(); /* Test datset object header minimization */ + test_min_dset_ohdr(); /* Test dataset object header minimization */ #ifndef H5_NO_DEPRECATED_SYMBOLS test_file_ishdf5(env_h5_drvr); /* Test detecting HDF5 files correctly */ - test_deprec(); /* Test deprecated routines */ + test_deprec(env_h5_drvr); /* Test deprecated routines */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ ret = H5Pclose(fapl_id); @@ -8069,13 +8125,17 @@ test_file(void) void cleanup_file(void) { - HDremove(SFILE1); - HDremove(FILE1); - HDremove(FILE2); - HDremove(FILE3); - HDremove(FILE4); - HDremove(FILE5); - HDremove(FILE6); - HDremove(FILE7); - HDremove(DST_FILE); + H5E_BEGIN_TRY + { + H5Fdelete(SFILE1, H5P_DEFAULT); + H5Fdelete(FILE1, H5P_DEFAULT); + H5Fdelete(FILE2, H5P_DEFAULT); + H5Fdelete(FILE3, H5P_DEFAULT); + H5Fdelete(FILE4, H5P_DEFAULT); + H5Fdelete(FILE5, H5P_DEFAULT); + H5Fdelete(FILE6, H5P_DEFAULT); + H5Fdelete(FILE7, H5P_DEFAULT); + H5Fdelete(DST_FILE, H5P_DEFAULT); + } + H5E_END_TRY; } diff --git a/test/tgenprop.c b/test/tgenprop.c index 55e5a9b..f54fbbc 100644 --- a/test/tgenprop.c +++ b/test/tgenprop.c @@ -107,7 +107,7 @@ test_genprop_basic_class(void) ret = H5Pequal(cid2, H5P_ROOT); VERIFY(ret, 1, "H5Pequal"); - /* Make certain false postives aren't being returned */ + /* Make certain false positives aren't being returned */ ret = H5Pequal(cid2, H5P_FILE_CREATE); VERIFY(ret, 0, "H5Pequal"); @@ -185,7 +185,7 @@ test_genprop_basic_class_prop(void) CHECK_I(ret, "H5Pget_nprops"); VERIFY(nprops, 0, "H5Pget_nprops"); - /* Check the existance of the first property (should fail) */ + /* Check the existence of the first property (should fail) */ ret = H5Pexist(cid1, PROP1_NAME); VERIFY(ret, 0, "H5Pexist"); @@ -199,7 +199,7 @@ test_genprop_basic_class_prop(void) H5Pregister2(cid1, PROP1_NAME, PROP1_SIZE, PROP1_DEF_VALUE, NULL, NULL, NULL, NULL, NULL, NULL, NULL); VERIFY(ret, FAIL, "H5Pregister2"); - /* Check the existance of the first property */ + /* Check the existence of the first property */ ret = H5Pexist(cid1, PROP1_NAME); VERIFY(ret, 1, "H5Pexist"); @@ -223,7 +223,7 @@ test_genprop_basic_class_prop(void) H5Pregister2(cid1, PROP2_NAME, PROP2_SIZE, PROP2_DEF_VALUE, NULL, NULL, NULL, NULL, NULL, NULL, NULL); VERIFY(ret, FAIL, "H5Pregister2"); - /* Check the existance of the second property */ + /* Check the existence of the second property */ ret = H5Pexist(cid1, PROP2_NAME); VERIFY(ret, 1, "H5Pexist"); @@ -242,7 +242,7 @@ test_genprop_basic_class_prop(void) H5Pregister2(cid1, PROP3_NAME, PROP3_SIZE, PROP3_DEF_VALUE, NULL, NULL, NULL, NULL, NULL, NULL, NULL); CHECK_I(ret, "H5Pregister2"); - /* Check the existance of the third property */ + /* Check the existence of the third property */ ret = H5Pexist(cid1, PROP3_NAME); VERIFY(ret, 1, "H5Pexist"); @@ -1942,7 +1942,7 @@ test_genprop_deprec_class(void) CHECK_I(ret, "H5Pget_nprops"); VERIFY(nprops, 0, "H5Pget_nprops"); - /* Check the existance of the first property (should fail) */ + /* Check the existence of the first property (should fail) */ ret = H5Pexist(cid1, PROP1_NAME); VERIFY(ret, 0, "H5Pexist"); @@ -1954,7 +1954,7 @@ test_genprop_deprec_class(void) ret = H5Pregister1(cid1, PROP1_NAME, PROP1_SIZE, PROP1_DEF_VALUE, NULL, NULL, NULL, NULL, NULL, NULL); VERIFY(ret, FAIL, "H5Pregister1"); - /* Check the existance of the first property */ + /* Check the existence of the first property */ ret = H5Pexist(cid1, PROP1_NAME); VERIFY(ret, 1, "H5Pexist"); @@ -1976,7 +1976,7 @@ test_genprop_deprec_class(void) ret = H5Pregister1(cid1, PROP2_NAME, PROP2_SIZE, PROP2_DEF_VALUE, NULL, NULL, NULL, NULL, NULL, NULL); VERIFY(ret, FAIL, "H5Pregister1"); - /* Check the existance of the second property */ + /* Check the existence of the second property */ ret = H5Pexist(cid1, PROP2_NAME); VERIFY(ret, 1, "H5Pexist"); @@ -1994,7 +1994,7 @@ test_genprop_deprec_class(void) ret = H5Pregister1(cid1, PROP3_NAME, PROP3_SIZE, PROP3_DEF_VALUE, NULL, NULL, NULL, NULL, NULL, NULL); CHECK_I(ret, "H5Pregister1"); - /* Check the existance of the third property */ + /* Check the existence of the third property */ ret = H5Pexist(cid1, PROP3_NAME); VERIFY(ret, 1, "H5Pexist"); diff --git a/test/th5o.c b/test/th5o.c index ebb6c6e..9504f7a 100644 --- a/test/th5o.c +++ b/test/th5o.c @@ -43,14 +43,17 @@ test_h5o_open(void) { hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ + char filename[1024]; hsize_t dims[RANK]; H5I_type_t id_type; /* Type of IDs returned from H5Oopen */ H5G_info_t ginfo; /* Group info struct */ H5T_class_t type_class; /* Class of the datatype */ herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create a group, dataset, and committed datatype within the file */ @@ -155,11 +158,14 @@ test_h5o_close(void) { hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ + char filename[1024]; hsize_t dims[RANK]; herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create a group, dataset, and committed datatype within the file */ @@ -247,8 +253,9 @@ test_h5o_open_by_addr(void) { hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ - H5L_info2_t li; /* Buffer for H5Lget_info2 */ - haddr_t grp_addr; /* Addresses for objects */ + char filename[1024]; + H5L_info2_t li; /* Buffer for H5Lget_info2 */ + haddr_t grp_addr; /* Addresses for objects */ haddr_t dset_addr; haddr_t dtype_addr; hsize_t dims[RANK]; @@ -257,8 +264,10 @@ test_h5o_open_by_addr(void) H5T_class_t type_class; /* Class of the datatype */ herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create a group, dataset, and committed datatype within the file */ @@ -397,15 +406,18 @@ test_h5o_open_by_token(void) { hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ - H5L_info2_t li; /* Buffer for H5Lget_info */ + char filename[1024]; + H5L_info2_t li; /* Buffer for H5Lget_info */ hsize_t dims[RANK]; H5I_type_t id_type; /* Type of IDs returned from H5Oopen */ H5G_info_t ginfo; /* Group info struct */ H5T_class_t type_class; /* Class of the datatype */ herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create a group, dataset, and committed datatype within the file */ @@ -515,12 +527,15 @@ test_h5o_refcount(void) { hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ - H5O_info2_t oinfo; /* Object info struct */ + char filename[1024]; + H5O_info2_t oinfo; /* Object info struct */ hsize_t dims[RANK]; herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create a group, dataset, and committed datatype within the file */ @@ -594,7 +609,7 @@ test_h5o_refcount(void) CHECK(ret, FAIL, "H5Oget_info_by_name3"); VERIFY(oinfo.rc, 1, "reference count in H5Oget_info_by_name3"); - /* Increment the reference counts and then close the file to make sure the increment is permanant */ + /* Increment the reference counts and then close the file to make sure the increment is permanent */ ret = H5Oincr_refcount(grp); CHECK(ret, FAIL, "H5Oincr_refcount"); ret = H5Oincr_refcount(dtype); @@ -612,7 +627,7 @@ test_h5o_refcount(void) CHECK(ret, FAIL, "H5Fclose"); /* Re-open the file and check that the reference counts were really incremented */ - fid = H5Fopen(TEST_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); grp = H5Gopen2(fid, "group", H5P_DEFAULT); @@ -650,7 +665,7 @@ test_h5o_refcount(void) CHECK(ret, FAIL, "H5Fclose"); /* Re-open the file and check that the reference counts were really decremented */ - fid = H5Fopen(TEST_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); grp = H5Gopen2(fid, "group", H5P_DEFAULT); @@ -709,10 +724,11 @@ test_h5o_refcount(void) static void test_h5o_plist(void) { - hid_t fid; /* HDF5 File ID */ - hid_t grp, dset, dtype, dspace; /* Object identifiers */ - hid_t fapl; /* File access property list */ - hid_t gcpl, dcpl, tcpl; /* Object creation properties */ + hid_t fid; /* HDF5 File ID */ + hid_t grp, dset, dtype, dspace; /* Object identifiers */ + hid_t fapl; /* File access property list */ + hid_t gcpl, dcpl, tcpl; /* Object creation properties */ + char filename[1024]; unsigned def_max_compact, def_min_dense; /* Default phase change parameters */ unsigned max_compact, min_dense; /* Actual phase change parameters */ herr_t ret; /* Value returned from API calls */ @@ -725,8 +741,10 @@ test_h5o_plist(void) ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); CHECK(ret, FAIL, "H5Pset_libver_bounds"); + h5_fixname(TEST_FILENAME, fapl, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); CHECK(fid, FAIL, "H5Fcreate"); /* Create group, dataset & named datatype creation property lists */ @@ -840,7 +858,7 @@ test_h5o_plist(void) CHECK(ret, FAIL, "H5Fclose"); /* Re-open the file and check that the object creation properties persist */ - fid = H5Fopen(TEST_FILENAME, H5F_ACC_RDONLY, fapl); + fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl); CHECK(fid, FAIL, "H5Fopen"); /* Re-open objects */ @@ -909,7 +927,8 @@ test_h5o_link(void) hid_t type_id = -1; hid_t fapl_id = -1; hid_t lcpl_id = -1; - hsize_t dims[2] = {TEST6_DIM1, TEST6_DIM2}; + char filename[1024]; + hsize_t dims[2] = {TEST6_DIM1, TEST6_DIM2}; htri_t committed; /* Whether the named datatype is committed */ H5F_libver_t low, high; /* File format bounds */ int * wdata; @@ -917,6 +936,8 @@ test_h5o_link(void) int i, n; herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Allocate memory buffers */ /* (These are treated as 2-D buffers) */ wdata = (int *)HDmalloc((size_t)(TEST6_DIM1 * TEST6_DIM2) * sizeof(int)); @@ -957,7 +978,7 @@ test_h5o_link(void) continue; /* Create a new HDF5 file */ - file_id = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); CHECK(file_id, FAIL, "H5Fcreate"); /* Close the FAPL */ @@ -1060,6 +1081,7 @@ test_h5o_comment(void) hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ hid_t attr_space, attr_id; + char filename[1024]; hsize_t dims[RANK]; hsize_t attr_dims = 1; int attr_value = 5; @@ -1073,8 +1095,10 @@ test_h5o_comment(void) herr_t ret; /* Value returned from API calls */ int ret_value; + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create an attribute for the file */ @@ -1152,7 +1176,7 @@ test_h5o_comment(void) /* Now make sure that the comments are correct all 4 types of objects */ /* Open file */ - fid = H5Fopen(TEST_FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + fid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Getting the comment on the file and verify it */ @@ -1232,6 +1256,7 @@ test_h5o_comment_by_name(void) hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ hid_t attr_space, attr_id; + char filename[1024]; hsize_t dims[RANK]; hsize_t attr_dims = 1; int attr_value = 5; @@ -1245,8 +1270,10 @@ test_h5o_comment_by_name(void) herr_t ret; /* Value returned from API calls */ int ret_value; + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create an attribute for the file */ @@ -1324,7 +1351,7 @@ test_h5o_comment_by_name(void) /* Now make sure that the comments are correct all 4 types of objects */ /* Open file */ - fid = H5Fopen(TEST_FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + fid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Getting the comment on the file and verify it */ @@ -1390,13 +1417,16 @@ test_h5o_comment_by_name(void) static void test_h5o_getinfo_same_file(void) { - hid_t fid1, fid2; /* HDF5 File ID */ - hid_t gid1, gid2; /* Group IDs */ + hid_t fid1, fid2; /* HDF5 File ID */ + hid_t gid1, gid2; /* Group IDs */ + char filename[1024]; H5O_info2_t oinfo1, oinfo2; /* Object info structs */ herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid1 = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid1, FAIL, "H5Fcreate"); /* Create two groups in the file */ @@ -1438,9 +1468,9 @@ test_h5o_getinfo_same_file(void) CHECK(ret, FAIL, "H5Fclose"); /* Open file twice */ - fid1 = H5Fopen(TEST_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + fid1 = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid1, FAIL, "H5Fopen"); - fid2 = H5Fopen(TEST_FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + fid2 = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid2, FAIL, "H5Fopen"); /* Open the two groups in the file */ @@ -1497,8 +1527,9 @@ test_h5o_open_by_addr_deprec(void) { hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ - H5L_info1_t li; /* Buffer for H5Lget_info1 */ - haddr_t grp_addr; /* Addresses for objects */ + char filename[1024]; + H5L_info1_t li; /* Buffer for H5Lget_info1 */ + haddr_t grp_addr; /* Addresses for objects */ haddr_t dset_addr; haddr_t dtype_addr; hsize_t dims[RANK]; @@ -1507,8 +1538,10 @@ test_h5o_open_by_addr_deprec(void) H5T_class_t type_class; /* Class of the datatype */ herr_t ret; /* Value returned from API calls */ + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + /* Create a new HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create a group, dataset, and committed datatype within the file */ @@ -1682,13 +1715,16 @@ test_h5o_getinfo_visit(void) hid_t gid1 = -1, gid2 = -1; /* Group IDs */ hid_t sid = -1; /* Dataspace ID */ hid_t aid = -1; /* Attribute ID */ - H5O_info1_t oinfo1, oinfo2; /* Object info structs */ - char attrname[25]; /* Attribute name */ - int j; /* Local index variable */ - herr_t ret; /* Value returned from API calls */ + char filename[1024]; + H5O_info1_t oinfo1, oinfo2; /* Object info structs */ + char attrname[25]; /* Attribute name */ + int j; /* Local index variable */ + herr_t ret; /* Value returned from API calls */ + + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); /* Create an HDF5 file */ - fid = H5Fcreate(TEST_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create "group1" in the file */ @@ -1806,5 +1842,12 @@ test_h5o(void) void cleanup_h5o(void) { - HDremove(TEST_FILENAME); + char filename[1024]; + + H5E_BEGIN_TRY + { + h5_fixname(TEST_FILENAME, H5P_DEFAULT, filename, sizeof filename); + H5Fdelete(filename, H5P_DEFAULT); + } + H5E_END_TRY; } diff --git a/test/th5s.c b/test/th5s.c index 51216dc..1bc19d4 100644 --- a/test/th5s.c +++ b/test/th5s.c @@ -194,7 +194,7 @@ test_h5s_basic(void) * If this test fails and the H5S_MAX_RANK variable has changed, follow * the instructions in space_overflow.c for regenerating the th5s.h5 file. */ - { + if (!h5_driver_uses_modified_filename()) { const char *testfile = H5_get_srcdir_filename(TESTFILE); /* Corrected test file name */ fid1 = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); @@ -2022,7 +2022,7 @@ test_h5s_encode_points(H5F_libver_t low, H5F_libver_t high) { hid_t fapl = -1; /* File access property list ID */ hid_t sid; /* Dataspace ID */ - hsize_t numparticles = 8388608; /* Used to calculate dimenion size */ + hsize_t numparticles = 8388608; /* Used to calculate dimension size */ unsigned num_dsets = 513; /* used to calculate dimension size */ hsize_t total_particles = numparticles * num_dsets; hsize_t vdsdims[1] = {total_particles}; /* Dimension size */ diff --git a/test/tid.c b/test/tid.c index 0a45116..583004e 100644 --- a/test/tid.c +++ b/test/tid.c @@ -41,7 +41,7 @@ basic_id_test(void) int num_ref; hsize_t num_members; - /* Try to register an ID with ficticious types */ + /* Try to register an ID with fictitious types */ H5E_BEGIN_TRY arrayID = H5Iregister((H5I_type_t)420, testObj); H5E_END_TRY @@ -58,7 +58,7 @@ basic_id_test(void) if (arrayID != H5I_INVALID_HID) goto out; - /* Try to access IDs with ficticious types */ + /* Try to access IDs with fictitious types */ H5E_BEGIN_TRY testPtr = H5Iobject_verify((hid_t)100, (H5I_type_t)0); H5E_END_TRY diff --git a/test/titerate.c b/test/titerate.c index 10ed039..1d87343 100644 --- a/test/titerate.c +++ b/test/titerate.c @@ -1175,8 +1175,10 @@ test_iterate(void) #endif } /* end for */ - /* Test the fix for issue HDFFV-10588 */ - test_corrupted_attnamelen(); + if (!h5_driver_uses_modified_filename()) { + /* Test the fix for issue HDFFV-10588 */ + test_corrupted_attnamelen(); + } /* Close FAPLs */ ret = H5Pclose(fapl); diff --git a/test/tmisc.c b/test/tmisc.c index b267330..f8bf602 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -287,7 +287,7 @@ typedef struct { #define MISC25C_ATTRNAME2 "ab" /* Definitions for misc. test #26 */ -#define MISC26_FILE "dcpl_file" +#define MISC26_FILE "dcpl_file.h5" /* Definitions for misc. test #27 */ /* (Note that this test file is generated by the "gen_bad_ohdr.c" code) */ @@ -1083,7 +1083,7 @@ test_misc6(void) /* Loop through adding attributes to each dataset */ for (u = 0; u < MISC6_NUMATTR; u++) { /* Create name for attribute */ - HDsprintf(attr_name, "Attr#%u", u); + HDsnprintf(attr_name, sizeof(attr_name), "Attr#%u", u); /* Open the file */ loc_id = H5Fopen(MISC6_FILE, H5F_ACC_RDWR, H5P_DEFAULT); @@ -1280,7 +1280,7 @@ test_misc8(void) for (v = 0; v < MISC8_DIM1; v++) *tdata++ = (int)(((u * MISC8_DIM1) + v) % 13); - /* Create a file acccess property list */ + /* Create a file access property list */ fapl = H5Pcreate(H5P_FILE_ACCESS); CHECK(fapl, FAIL, "H5Pcreate"); @@ -3034,7 +3034,7 @@ test_misc18(void) /* Loop creating attributes on each dataset, flushing them to the file each time */ for (u = 0; u < 10; u++) { /* Set up attribute name */ - HDsprintf(attr_name, "Attr %u", u); + HDsnprintf(attr_name, sizeof(attr_name), "Attr %u", u); /* Create & close attribute on first dataset */ aid = H5Acreate2(did1, attr_name, H5T_STD_U32LE, sid, H5P_DEFAULT, H5P_DEFAULT); @@ -3892,7 +3892,7 @@ test_misc21(void) /**************************************************************** ** -** test_misc22(): Test SZIP bits-per-pixel paramter. +** test_misc22(): Test SZIP bits-per-pixel parameter. ** This should be set according to the datatype. ** Tests for precision and offset combo's. ** @@ -5504,7 +5504,7 @@ test_misc30(void) CHECK(ret, FAIL, "test_misc30_get_info"); } - HDsprintf(gname, "/g0/group%d", i); + HDsnprintf(gname, sizeof(gname), "/g0/group%d", i); gid = H5Gcreate2(fid, gname, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(gid, FAIL, "H5Gcreate2"); @@ -6026,29 +6026,51 @@ test_misc36(void) void test_misc(void) { + hbool_t driver_uses_modified_filename = h5_driver_uses_modified_filename(); + hbool_t default_driver = h5_using_default_driver(NULL); + /* Output message about test being performed */ MESSAGE(5, ("Testing Miscellaneous Routines\n")); - test_misc1(); /* Test unlinking a dataset & immediately re-using name */ - test_misc2(); /* Test storing a VL-derived datatype in two different files */ - test_misc3(); /* Test reading from chunked dataset with non-zero fill value */ - test_misc4(); /* Test retrieving the fileno for various objects with H5Oget_info() */ - test_misc5(); /* Test several level deep nested compound & VL datatypes */ - test_misc6(); /* Test object header continuation code */ - test_misc7(); /* Test for sensible datatypes stored on disk */ - test_misc8(); /* Test storage sizes of various types of dataset storage */ - test_misc9(); /* Test for opening (not creating) core files */ - test_misc10(); /* Test for using dataset creation property lists from old files */ - test_misc11(); /* Test for all properties of a file creation property list being stored */ + test_misc1(); /* Test unlinking a dataset & immediately re-using name */ + test_misc2(); /* Test storing a VL-derived datatype in two different files */ + test_misc3(); /* Test reading from chunked dataset with non-zero fill value */ + test_misc4(); /* Test retrieving the fileno for various objects with H5Oget_info() */ + test_misc5(); /* Test several level deep nested compound & VL datatypes */ + test_misc6(); /* Test object header continuation code */ + test_misc7(); /* Test for sensible datatypes stored on disk */ + test_misc8(); /* Test storage sizes of various types of dataset storage */ + test_misc9(); /* Test for opening (not creating) core files */ + + if (!driver_uses_modified_filename) { + test_misc10(); /* Test for using dataset creation property lists from old files */ + } + + if (default_driver) { + test_misc11(); /* Test for all properties of a file creation property list being stored */ + } + test_misc12(); /* Test VL-strings in chunked datasets operating correctly */ - test_misc13(); /* Test that a user block can be insert in front of file contents */ + + if (default_driver) { + test_misc13(); /* Test that a user block can be insert in front of file contents */ + } + test_misc14(); /* Test that deleted dataset's data is removed from sieve buffer correctly */ - test_misc15(); /* Test that checking a file's access property list more than once works */ + + if (!driver_uses_modified_filename) { + test_misc15(); /* Test that checking a file's access property list more than once works */ + } + test_misc16(); /* Test array of fixed-length string */ test_misc17(); /* Test array of ASCII character */ test_misc18(); /* Test new object header information in H5O_info_t struct */ test_misc19(); /* Test incrementing & decrementing ref count on IDs */ - test_misc20(); /* Test problems with truncated dimensions in version 2 of storage layout message */ + + if (!driver_uses_modified_filename) { + test_misc20(); /* Test problems with truncated dimensions in version 2 of storage layout message */ + } + #ifdef H5_HAVE_FILTER_SZIP test_misc21(); /* Test that "late" allocation time is treated the same as "incremental", for chunked datasets w/a filters */ @@ -6057,19 +6079,39 @@ test_misc(void) test_misc23(); /* Test intermediate group creation */ test_misc24(); /* Test inappropriate API opens of objects */ test_misc25a(); /* Exercise null object header message merge bug */ - test_misc25b(); /* Exercise null object header message merge bug on existing file */ + + if (!driver_uses_modified_filename) { + test_misc25b(); /* Exercise null object header message merge bug on existing file */ + } + test_misc25c(); /* Exercise another null object header message merge bug */ test_misc26(); /* Test closing property lists with long filter pipelines */ - test_misc27(); /* Test opening file with object that has bad # of object header messages */ - test_misc28(); /* Test that chunks are cached appropriately */ - test_misc29(); /* Test that speculative metadata reads are handled correctly */ - test_misc30(); /* Exercise local heap loading bug where free lists were getting dropped */ - test_misc31(); /* Test Reentering library through deprecated routines after H5close() */ - test_misc32(); /* Test filter memory allocation functions */ - test_misc33(); /* Test to verify that H5HL_offset_into() returns error if offset exceeds heap block */ - test_misc34(); /* Test behavior of 0 and NULL in H5MM API calls */ - test_misc35(); /* Test behavior of free-list & allocation statistics API calls */ - test_misc36(); /* Exercise H5atclose and H5is_library_terminating */ + + if (!driver_uses_modified_filename) { + test_misc27(); /* Test opening file with object that has bad # of object header messages */ + } + + test_misc28(); /* Test that chunks are cached appropriately */ + + if (!driver_uses_modified_filename) { + test_misc29(); /* Test that speculative metadata reads are handled correctly */ + } + + test_misc30(); /* Exercise local heap loading bug where free lists were getting dropped */ + + if (default_driver) { + test_misc31(); /* Test Reentering library through deprecated routines after H5close() */ + } + + test_misc32(); /* Test filter memory allocation functions */ + + if (!driver_uses_modified_filename) { + test_misc33(); /* Test to verify that H5HL_offset_into() returns error if offset exceeds heap block */ + } + + test_misc34(); /* Test behavior of 0 and NULL in H5MM API calls */ + test_misc35(); /* Test behavior of free-list & allocation statistics API calls */ + test_misc36(); /* Exercise H5atclose and H5is_library_terminating */ } /* test_misc() */ diff --git a/test/trefer.c b/test/trefer.c index 9b6c415..95e601e 100644 --- a/test/trefer.c +++ b/test/trefer.c @@ -2732,7 +2732,7 @@ test_reference_compat_conv(void) hid_t sid1, sid2, sid3; /* Dataspace IDs */ hid_t tid1, tid2; /* Datatype ID */ hsize_t dims1[] = {SPACE1_DIM1}, dims2[] = {SPACE2_DIM1, SPACE2_DIM2}, - dims3[] = {SPACE1_DIM1}; /* Purposedly set dimension larger to test NULL references */ + dims3[] = {SPACE1_DIM1}; /* Purposely set dimension larger to test NULL references */ hsize_t start[SPACE2_RANK]; /* Starting location of hyperslab */ hsize_t stride[SPACE2_RANK]; /* Stride of hyperslab */ hsize_t count[SPACE2_RANK]; /* Element count of hyperslab */ @@ -3452,11 +3452,17 @@ test_reference_perf(void) void test_reference(void) { - H5F_libver_t low, high; /* Low and high bounds */ + H5F_libver_t low, high; /* Low and high bounds */ + const char * env_h5_drvr; /* File Driver value from environment */ /* Output message about test being performed */ MESSAGE(5, ("Testing References\n")); + /* Get the VFD to use */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + test_reference_params(); /* Test for correct parameter checking */ test_reference_obj(); /* Test basic H5R object reference code */ test_reference_vlen_obj(); /* Test reference within vlen */ @@ -3476,7 +3482,11 @@ test_reference(void) } /* end high bound */ } /* end low bound */ - test_reference_obj_deleted(); /* Test H5R object reference code for deleted objects */ + /* The following test is currently broken with the Direct VFD */ + if (HDstrcmp(env_h5_drvr, "direct") != 0) { + test_reference_obj_deleted(); /* Test H5R object reference code for deleted objects */ + } + test_reference_group(); /* Test operations on dereferenced groups */ test_reference_attr(); /* Test attribute references */ test_reference_external(); /* Test external references */ diff --git a/test/trefstr.c b/test/trefstr.c index d0575ab..89e62db 100644 --- a/test/trefstr.c +++ b/test/trefstr.c @@ -309,7 +309,7 @@ test_refstr_asprintf_cat(void) /* Get pointer to raw string in ref-counted string */ s = H5RS_get_str(rs); CHECK_PTR(s, "H5RS_get_str"); - HDsprintf(buf, "%d-%s", (int)10, "foo"); + HDsnprintf(buf, sizeof(buf), "%d-%s", (int)10, "foo"); cmp = HDstrcmp(s, buf); VERIFY(cmp, 0, "HDstrcmp"); @@ -320,7 +320,7 @@ test_refstr_asprintf_cat(void) /* Get pointer to raw string in ref-counted string */ s = H5RS_get_str(rs); CHECK_PTR(s, "H5RS_get_str"); - HDsprintf(buf, "%d-%s-%f", (int)10, "foo", (double)20.0); + HDsnprintf(buf, sizeof(buf), "%d-%s-%f", (int)10, "foo", (double)20.0); cmp = HDstrcmp(s, buf); VERIFY(cmp, 0, "HDstrcmp"); @@ -360,7 +360,7 @@ test_refstr_acat(void) /* Get pointer to raw string in ref-counted string */ s = H5RS_get_str(rs); CHECK_PTR(s, "H5RS_get_str"); - HDsprintf(buf, "%s", "foo"); + HDsnprintf(buf, sizeof(buf), "%s", "foo"); cmp = HDstrcmp(s, buf); VERIFY(cmp, 0, "HDstrcmp"); @@ -371,7 +371,7 @@ test_refstr_acat(void) /* Get pointer to raw string in ref-counted string */ s = H5RS_get_str(rs); CHECK_PTR(s, "H5RS_get_str"); - HDsprintf(buf, "%s", "foobar"); + HDsnprintf(buf, sizeof(buf), "%s", "foobar"); cmp = HDstrcmp(s, buf); VERIFY(cmp, 0, "HDstrcmp"); @@ -386,7 +386,7 @@ test_refstr_acat(void) /* Get pointer to raw string in ref-counted string */ s = H5RS_get_str(rs); CHECK_PTR(s, "H5RS_get_str"); - HDsprintf(buf, "%s", "foobar"); + HDsnprintf(buf, sizeof(buf), "%s", "foobar"); large_str2 = HDmalloc(1024 + 6); CHECK_PTR(large_str2, "HDmalloc"); HDstrcpy(large_str2, "foobar"); diff --git a/test/tselect.c b/test/tselect.c index 1bfb663..0d4176b 100644 --- a/test/tselect.c +++ b/test/tselect.c @@ -1879,7 +1879,7 @@ verify_select_hyper_contig_dr__run_test(const uint16_t *cube_buf, size_t H5_ATTR uint16_t expected_value; /* Expected value in dataset */ unsigned i, j, k, l, m; /* Local index variables */ size_t s; /* Local index variable */ - hbool_t mis_match; /* Flag to indicate mis-match in expected value */ + hbool_t mis_match; /* Flag to indicate mismatch in expected value */ HDassert(cube_buf); HDassert(cube_size > 0); @@ -2357,7 +2357,7 @@ test_select_hyper_contig_dr__run_test(int test_num, const uint16_t *cube_buf, co /* Now write the contents of the in memory small cube to slices of * the on disk cube. After each write, read the on disk cube - * into memeory, and verify that it contains the expected + * into memory, and verify that it contains the expected * data. Verify that H5Sselect_shape_same() returns true on * the memory and file selections. */ @@ -2742,7 +2742,7 @@ test_select_hyper_checker_board_dr__select_checker_board(hid_t tgt_n_cube_sid, u i++; } while ((i <= 1) && (0 >= sel_offset)); - /* Wierdness alert: + /* Weirdness alert: * * Some how, it seems that selections can extend beyond the * boundaries of the target dataspace -- hence the following @@ -3401,7 +3401,7 @@ test_select_hyper_checker_board_dr__run_test(int test_num, const uint16_t *cube_ /* Now write checker board selections of the entries in memory * small cube to slices of the on disk cube. After each write, - * read the on disk large cube * into memeory, and verify that + * read the on disk large cube * into memory, and verify that * it contains the expected * data. Verify that * H5Sselect_shape_same() returns true on the memory and file * selections. @@ -5256,7 +5256,7 @@ test_select_hyper_union_stagger(void) error = H5Fclose(file_id); CHECK(error, FAIL, "H5Fclose"); - /* Initialize intput buffer */ + /* Initialize input buffer */ HDmemset(data_out, 0, 7 * 7 * sizeof(int)); /* Open file */ @@ -10862,13 +10862,15 @@ test_shape_same_dr__full_space_vs_slice(int test_num, int small_rank, int large_ HDassert(edge_size > 0); HDassert(edge_size <= 1000); - HDsprintf(test_desc_0, "\tn-cube slice through m-cube (n <= m) test %d.\n", test_num); + HDsnprintf(test_desc_0, sizeof(test_desc_0), "\tn-cube slice through m-cube (n <= m) test %d.\n", + test_num); MESSAGE(7, (test_desc_0)); /* This statement must be updated if SS_DR_MAX_RANK is changed */ - HDsprintf(test_desc_1, "\t\tranks: %d/%d offset: %d dim_selected: %d/%d/%d/%d/%d.\n", small_rank, - large_rank, offset, (int)dim_selected[0], (int)dim_selected[1], (int)dim_selected[2], - (int)dim_selected[3], (int)dim_selected[4]); + HDsnprintf(test_desc_1, sizeof(test_desc_1), + "\t\tranks: %d/%d offset: %d dim_selected: %d/%d/%d/%d/%d.\n", small_rank, large_rank, offset, + (int)dim_selected[0], (int)dim_selected[1], (int)dim_selected[2], (int)dim_selected[3], + (int)dim_selected[4]); MESSAGE(7, (test_desc_1)); /* copy the edge size into the dims array */ @@ -11120,15 +11122,16 @@ test_shape_same_dr__checkerboard(int test_num, int small_rank, int large_rank, i HDassert(dims_selected >= 0); HDassert(dims_selected <= large_rank); - HDsprintf(test_desc_0, "\tcheckerboard n-cube slice through m-cube (n <= m) test %d.\n", test_num); + HDsnprintf(test_desc_0, sizeof(test_desc_0), + "\tcheckerboard n-cube slice through m-cube (n <= m) test %d.\n", test_num); MESSAGE(7, (test_desc_0)); /* This statement must be updated if SS_DR_MAX_RANK is changed */ - HDsprintf(test_desc_1, - "\tranks: %d/%d edge/chkr size: %d/%d offset: %d dim_selected: %d/%d/%d/%d/%d:%d.\n", - small_rank, large_rank, (int)edge_size, (int)checker_size, offset, (int)dim_selected[0], - (int)dim_selected[1], (int)dim_selected[2], (int)dim_selected[3], (int)dim_selected[4], - dims_selected); + HDsnprintf(test_desc_1, sizeof(test_desc_1), + "\tranks: %d/%d edge/chkr size: %d/%d offset: %d dim_selected: %d/%d/%d/%d/%d:%d.\n", + small_rank, large_rank, (int)edge_size, (int)checker_size, offset, (int)dim_selected[0], + (int)dim_selected[1], (int)dim_selected[2], (int)dim_selected[3], (int)dim_selected[4], + dims_selected); MESSAGE(7, (test_desc_1)); /* copy the edge size into the dims array */ @@ -11251,7 +11254,7 @@ test_shape_same_dr__checkerboard(int test_num, int small_rank, int large_rank, i CHECK(ret, FAIL, "H5Sselect_hyperslab"); } /* end if */ - /* Wierdness alert: + /* Weirdness alert: * * Some how, it seems that selections can extend beyond the * boundaries of the target dataspace -- hence the following @@ -11346,7 +11349,7 @@ test_shape_same_dr__checkerboard(int test_num, int small_rank, int large_rank, i CHECK(ret, FAIL, "H5Sselect_hyperslab"); } /* end if */ - /* Wierdness alert: + /* Weirdness alert: * * Again, it seems that selections can extend beyond the * boundaries of the target dataspace -- hence the following @@ -11664,15 +11667,16 @@ test_shape_same_dr__irregular(int test_num, int small_rank, int large_rank, int HDassert(dims_selected >= 0); HDassert(dims_selected <= large_rank); - HDsprintf(test_desc_0, "\tirregular sub set of n-cube slice through m-cube (n <= m) test %d.\n", - test_num); + HDsnprintf(test_desc_0, sizeof(test_desc_0), + "\tirregular sub set of n-cube slice through m-cube (n <= m) test %d.\n", test_num); MESSAGE(7, (test_desc_0)); /* This statement must be updated if SS_DR_MAX_RANK is changed */ - HDsprintf(test_desc_1, "\tranks: %d/%d edge: %d s/p offset: %d/%d dim_selected: %d/%d/%d/%d/%d:%d.\n", - small_rank, large_rank, edge_size, slice_offset, pattern_offset, (int)dim_selected[0], - (int)dim_selected[1], (int)dim_selected[2], (int)dim_selected[3], (int)dim_selected[4], - dims_selected); + HDsnprintf(test_desc_1, sizeof(test_desc_1), + "\tranks: %d/%d edge: %d s/p offset: %d/%d dim_selected: %d/%d/%d/%d/%d:%d.\n", small_rank, + large_rank, edge_size, slice_offset, pattern_offset, (int)dim_selected[0], + (int)dim_selected[1], (int)dim_selected[2], (int)dim_selected[3], (int)dim_selected[4], + dims_selected); MESSAGE(7, (test_desc_1)); /* copy the edge size into the dims array */ @@ -12126,7 +12130,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ MESSAGE(7, ("Testing functionality to rebuild 2-D hyperslab selection\n")); /* Create 2-D dataspace */ @@ -12218,7 +12222,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ MESSAGE(7, ("Testing functionality to rebuild 3-D hyperslab selection\n")); @@ -12320,7 +12324,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ MESSAGE(7, ("Testing functionality to rebuild 4-D hyperslab selection\n")); @@ -12439,7 +12443,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ MESSAGE(7, ("Testing functionality to rebuild 5-D hyperslab selection\n")); @@ -12568,7 +12572,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ /* We use 5-D to test a special case with rebuilding routine TRUE, FALSE and TRUE */ @@ -12615,7 +12619,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ /* Adding some selections to make it real irregular */ start5[3] = 1; @@ -12643,7 +12647,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ /* Add more selections to make it regular again */ start5[3] = 5; @@ -12671,7 +12675,7 @@ test_space_rebuild(void) ret = FAIL; CHECK(ret, FAIL, "H5S_hyper_rebuild"); } - /* No need to do shape comparision */ + /* No need to do shape comparison */ H5Sclose(sid_reg1); CHECK(ret, FAIL, "H5Sclose"); @@ -12977,7 +12981,7 @@ test_space_update_diminfo(void) CHECK(ret, FAIL, "H5S_hyper_update_diminfo"); } /* end if */ - /* Add block parially overlapping first, with OR */ + /* Add block partially overlapping first, with OR */ start1[0] = 4; count1[0] = 1; block1[0] = 2; @@ -13022,7 +13026,7 @@ test_space_update_diminfo(void) CHECK(ret, FAIL, "H5S_hyper_update_diminfo"); } /* end if */ - /* Add equally sized block parially overlapping current, with XOR */ + /* Add equally sized block partially overlapping current, with XOR */ start1[0] = 3; count1[0] = 1; block1[0] = 5; @@ -13056,7 +13060,7 @@ test_space_update_diminfo(void) CHECK(ret, FAIL, "H5S_hyper_rebuild"); } /* end if */ - /* Add differently sized block parially overlapping current, with XOR */ + /* Add differently sized block partially overlapping current, with XOR */ start1[0] = 4; count1[0] = 1; block1[0] = 5; @@ -14232,7 +14236,7 @@ test_hyper_regular(void) ** ****************************************************************/ static void -test_hyper_unlim_check(hid_t sid, hsize_t *dims, hssize_t enpoints, hssize_t enblocks, hsize_t *eblock1, +test_hyper_unlim_check(hid_t sid, hsize_t *dims, hssize_t endpoints, hssize_t enblocks, hsize_t *eblock1, hsize_t *eblock2) { hid_t lim_sid; @@ -14257,7 +14261,7 @@ test_hyper_unlim_check(hid_t sid, hsize_t *dims, hssize_t enpoints, hssize_t enb /* Check number of elements */ npoints = H5Sget_select_npoints(lim_sid); CHECK(npoints, FAIL, "H5Sget_select_npoints"); - VERIFY(npoints, enpoints, "H5Sget_select_npoints"); + VERIFY(npoints, endpoints, "H5Sget_select_npoints"); /* Get selection type */ sel_type = H5Sget_select_type(lim_sid); @@ -15991,18 +15995,24 @@ test_h5s_set_extent_none(void) void test_select(void) { - hid_t plist_id; /* Property list for reading random hyperslabs */ - hid_t fapl; /* Property list accessing the file */ - int mdc_nelmts; /* Metadata number of elements */ - size_t rdcc_nelmts; /* Raw data number of elements */ - size_t rdcc_nbytes; /* Raw data number of bytes */ - double rdcc_w0; /* Raw data write percentage */ - hssize_t offset[SPACE7_RANK] = {1, 1}; /* Offset for testing selection offsets */ - herr_t ret; /* Generic return value */ + hid_t plist_id; /* Property list for reading random hyperslabs */ + hid_t fapl; /* Property list accessing the file */ + int mdc_nelmts; /* Metadata number of elements */ + size_t rdcc_nelmts; /* Raw data number of elements */ + size_t rdcc_nbytes; /* Raw data number of bytes */ + double rdcc_w0; /* Raw data write percentage */ + hssize_t offset[SPACE7_RANK] = {1, 1}; /* Offset for testing selection offsets */ + const char *env_h5_drvr; /* File Driver value from environment */ + herr_t ret; /* Generic return value */ /* Output message about test being performed */ MESSAGE(5, ("Testing Selections\n")); + /* Get the VFD to use */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + /* Create a dataset transfer property list */ plist_id = H5Pcreate(H5P_DATASET_XFER); CHECK(plist_id, FAIL, "H5Pcreate"); @@ -16065,10 +16075,13 @@ test_select(void) test_select_hyper_union_3d(); /* Test hyperslab union code for 3-D dataset */ test_select_hyper_valid_combination(); /* Test different input combinations */ - test_select_hyper_and_2d(); /* Test hyperslab intersection (AND) code for 2-D dataset */ - test_select_hyper_xor_2d(); /* Test hyperslab XOR code for 2-D dataset */ - test_select_hyper_notb_2d(); /* Test hyperslab NOTB code for 2-D dataset */ - test_select_hyper_nota_2d(); /* Test hyperslab NOTA code for 2-D dataset */ + /* The following tests are currently broken with the Direct VFD */ + if (HDstrcmp(env_h5_drvr, "direct") != 0) { + test_select_hyper_and_2d(); /* Test hyperslab intersection (AND) code for 2-D dataset */ + test_select_hyper_xor_2d(); /* Test hyperslab XOR code for 2-D dataset */ + test_select_hyper_notb_2d(); /* Test hyperslab NOTB code for 2-D dataset */ + test_select_hyper_nota_2d(); /* Test hyperslab NOTA code for 2-D dataset */ + } /* test the random hyperslab I/O with the default property list for reading */ test_select_hyper_union_random_5d(H5P_DEFAULT); /* Test hyperslab union code for random 5-D hyperslabs */ diff --git a/test/tsohm.c b/test/tsohm.c index b4ece0b..04b9522 100644 --- a/test/tsohm.c +++ b/test/tsohm.c @@ -636,7 +636,7 @@ size1_helper(hid_t file, const char *filename, hid_t fapl_id, hbool_t test_file_ if (GetTestExpress() > 1) test_file_closing = FALSE; - /* Intialize wdata */ + /* Initialize wdata */ HDmemset(&wdata, 0, sizeof(wdata)); wdata.i1 = 11; HDstrcpy(wdata.str, "string"); @@ -649,7 +649,7 @@ size1_helper(hid_t file, const char *filename, hid_t fapl_id, hbool_t test_file_ wdata.i8 = 88; wdata.f1 = 0.0F; - /* Intialize rdata */ + /* Initialize rdata */ HDmemset(&rdata, 0, sizeof(rdata)); dtype1_id = make_dtype_1(); @@ -762,7 +762,7 @@ error: * Function: getsize_testsize1 * * Purpose: Creates a test file, populates it, and returns its file size. - * Oject header information from the "first" dataset in the file + * Object header information from the "first" dataset in the file * is stored in pointer `oinfo`. * * Programmer: Jacob Smith @@ -1551,7 +1551,7 @@ size2_dump_struct(const char *name, size2_helper_struct *sizes) /*------------------------------------------------------------------------- * Function: size2_helper * - * Purpose: A helper functon for test_sohm_size2. + * Purpose: A helper function for test_sohm_size2. * * Creates a file using the given fcpl, then creates lots * of different kinds of messages within the file and @@ -1931,7 +1931,7 @@ size2_helper(hid_t fcpl_id, int test_file_closing, size2_helper_struct *ret_size /*------------------------------------------------------------------------- * Function: size2_verify * - * Purpose: A helper functon to verify the file created by size2_helper. + * Purpose: A helper function to verify the file created by size2_helper. * * Runs various tests (not exhaustive) to ensure that the * file FILENAME actually has the structure that size2_helper @@ -3815,23 +3815,45 @@ test_sohm_external_dtype(void) void test_sohm(void) { + const char *env_h5_drvr; + hbool_t default_driver; + MESSAGE(5, ("Testing Shared Object Header Messages\n")); - test_sohm_fcpl(); /* Test SOHMs and file creation plists */ - test_sohm_fcpl_errors(); /* Bogus H5P* calls for SOHMs */ - test_sohm_size1(); /* Tests the sizes of files with one SOHM */ -#if 0 /* TODO: REVEALS BUG TO BE FIXED - SEE JIRA HDFFV-10645 */ + /* Get the VFD to use */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + default_driver = h5_using_default_driver(env_h5_drvr); + + test_sohm_fcpl(); /* Test SOHMs and file creation plists */ + test_sohm_fcpl_errors(); /* Bogus H5P* calls for SOHMs */ + + /* Only run this test with sec2/default driver */ + if (default_driver) + test_sohm_size1(); /* Tests the sizes of files with one SOHM */ + +#if 0 /* TODO: REVEALS BUG TO BE FIXED - SEE JIRA HDFFV-10645 */ test_sohm_size_consistency_open_create(); -#endif /* Jira HDFFV-10645 */ - test_sohm_attrs(); /* Tests shared messages in attributes */ - test_sohm_size2(0); /* Tests the sizes of files with multiple SOHMs */ - test_sohm_size2(1); /* Tests the sizes of files with multiple - * SOHMs, closing and reopening file after - * each write. */ +#endif /* Jira HDFFV-10645 */ + test_sohm_attrs(); /* Tests shared messages in attributes */ + + /* Only run these tests with sec2/default driver */ + if (default_driver) { + test_sohm_size2(0); /* Tests the sizes of files with multiple SOHMs */ + test_sohm_size2(1); /* Tests the sizes of files with multiple + * SOHMs, closing and reopening file after + * each write. */ + } + test_sohm_delete(); /* Test deleting shared messages */ test_sohm_delete_revert(); /* Test that a file with SOHMs becomes an * empty file again when they are deleted. */ - test_sohm_extlink(); /* Test SOHMs when external links are used */ + + if (!h5_driver_uses_modified_filename()) { + test_sohm_extlink(); /* Test SOHMs when external links are used */ + } test_sohm_extend_dset(); /* Test extending shared datasets */ test_sohm_external_dtype(); /* Test using datatype in another file */ diff --git a/test/ttsafe.c b/test/ttsafe.c index 067a715..da78a7d 100644 --- a/test/ttsafe.c +++ b/test/ttsafe.c @@ -132,7 +132,7 @@ main(int argc, char *argv[]) TestSummary(); /* Clean up test files, if allowed */ - if (GetTestCleanup() && !HDgetenv("HDF5_NOCLEANUP")) + if (GetTestCleanup() && !HDgetenv(HDF5_NOCLEANUP)) TestCleanup(); /* Release test infrastructure */ diff --git a/test/ttsafe_attr_vlen.c b/test/ttsafe_attr_vlen.c index ecd0aa6..9fafaca 100644 --- a/test/ttsafe_attr_vlen.c +++ b/test/ttsafe_attr_vlen.c @@ -136,7 +136,7 @@ tts_attr_vlen_thread(void H5_ATTR_UNUSED *client_data) gid = H5Gopen2(fid, "/", H5P_DEFAULT); CHECK(gid, H5I_INVALID_HID, "H5Gopen"); - /* Open the attribte */ + /* Open the attribute */ aid = H5Aopen(gid, "root_attr", H5P_DEFAULT); CHECK(aid, H5I_INVALID_HID, "H5Aopen"); diff --git a/test/tvlstr.c b/test/tvlstr.c index 68f6124..5168d39 100644 --- a/test/tvlstr.c +++ b/test/tvlstr.c @@ -817,33 +817,33 @@ test_vl_rewrite(void) /* Create in file 1 */ for (i = 0; i < REWRITE_NDATASETS; i++) { - HDsprintf(name, "/set_%d", i); + HDsnprintf(name, sizeof(name), "/set_%d", i); write_scalar_dset(file1, type, space, name, name); } /* Effectively copy data from file 1 to 2 */ for (i = 0; i < REWRITE_NDATASETS; i++) { - HDsprintf(name, "/set_%d", i); + HDsnprintf(name, sizeof(name), "/set_%d", i); read_scalar_dset(file1, type, space, name, name); write_scalar_dset(file2, type, space, name, name); } /* Read back from file 2 */ for (i = 0; i < REWRITE_NDATASETS; i++) { - HDsprintf(name, "/set_%d", i); + HDsnprintf(name, sizeof(name), "/set_%d", i); read_scalar_dset(file2, type, space, name, name); } /* end for */ /* Remove from file 2. */ for (i = 0; i < REWRITE_NDATASETS; i++) { - HDsprintf(name, "/set_%d", i); + HDsnprintf(name, sizeof(name), "/set_%d", i); ret = H5Ldelete(file2, name, H5P_DEFAULT); CHECK(ret, FAIL, "H5Ldelete"); } /* end for */ /* Effectively copy from file 1 to file 2 */ for (i = 0; i < REWRITE_NDATASETS; i++) { - HDsprintf(name, "/set_%d", i); + HDsnprintf(name, sizeof(name), "/set_%d", i); read_scalar_dset(file1, type, space, name, name); write_scalar_dset(file2, type, space, name, name); } /* end for */ diff --git a/test/tvltypes.c b/test/tvltypes.c index 03a8ad3..0c4cf68 100644 --- a/test/tvltypes.c +++ b/test/tvltypes.c @@ -131,7 +131,7 @@ test_vltypes_dataset_create(void) dcpl = H5Pcreate(H5P_DATASET_CREATE); CHECK(dcpl, FAIL, "H5Pcreate"); - /* Set fill value writting time to be NEVER */ + /* Set fill value writing time to be NEVER */ ret = H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER); CHECK(ret, FAIL, "H5Pset_fill_time"); diff --git a/test/twriteorder.c b/test/twriteorder.c index e792487..c58b030 100644 --- a/test/twriteorder.c +++ b/test/twriteorder.c @@ -54,7 +54,7 @@ * * If the system, in which the writer and reader processes run, the readers * will always get all chain-linked blocks correctly. If the order of write - * is not maintained, some reader processes may found unexpect block data. + * is not maintained, some reader processes may found unexpected block data. * *************************************************************/ @@ -63,7 +63,7 @@ /* This test uses many POSIX things that are not available on * Windows. */ -#ifdef H5_HAVE_UNISTD_H +#if defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) #define DATAFILE "twriteorder.dat" /* #define READERS_MAX 10 */ /* max number of readers */ @@ -466,7 +466,7 @@ done: return ret_value; } -#else /* H5_HAVE_UNISTD_H */ +#else /* defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID) */ int main(void) @@ -475,4 +475,4 @@ main(void) return EXIT_SUCCESS; } /* end main() */ -#endif /* H5_HAVE_UNISTD_H */ +#endif /* defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID) */ diff --git a/test/unlink.c b/test/unlink.c index 94bd035..19b7b37 100644 --- a/test/unlink.c +++ b/test/unlink.c @@ -617,7 +617,7 @@ test_filespace(hid_t fapl) /* Create common objects for datasets */ - /* Create dataset creation property list for contigous storage */ + /* Create dataset creation property list for contiguous storage */ if ((contig_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR @@ -1099,7 +1099,7 @@ test_filespace(hid_t fapl) PASSED(); - /* Create simple group hiearchy, remove it & verify file size */ + /* Create simple group hierarchy, remove it & verify file size */ TESTING(" simple group hierarchy"); /* Create file */ @@ -1138,7 +1138,7 @@ test_filespace(hid_t fapl) PASSED(); - /* Create complex group hiearchy, remove it & verify file size */ + /* Create complex group hierarchy, remove it & verify file size */ TESTING(" complex group hierarchy"); /* Create file */ diff --git a/test/unregister.c b/test/unregister.c index 803f8373..ebb51b1 100644 --- a/test/unregister.c +++ b/test/unregister.c @@ -136,7 +136,7 @@ test_unregister_filters(hid_t fapl_id) /* Create multiple groups under the main group */ for (i = 0; i < GROUP_ITERATION; i++) { - HDsprintf(group_name, "group_%d", i); + HDsnprintf(group_name, sizeof(group_name), "group_%d", i); if ((gid_loop = H5Gcreate2(gid, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; if (H5Gclose(gid_loop) < 0) diff --git a/test/use_append_chunk.c b/test/use_append_chunk.c index 107615a..185cb26 100644 --- a/test/use_append_chunk.c +++ b/test/use_append_chunk.c @@ -63,7 +63,7 @@ /* This test uses many POSIX things that are not available on * Windows. */ -#ifdef H5_HAVE_UNISTD_H +#if defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) #include "use.h" @@ -270,7 +270,7 @@ done: return (ret_value); } -#else /* H5_HAVE_UNISTD_H */ +#else /* defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) */ int main(void) @@ -279,4 +279,4 @@ main(void) return EXIT_SUCCESS; } /* end main() */ -#endif /* H5_HAVE_UNISTD_H */ +#endif /* defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) */ diff --git a/test/use_append_mchunks.c b/test/use_append_mchunks.c index 47c9f92..60f63c0 100644 --- a/test/use_append_mchunks.c +++ b/test/use_append_mchunks.c @@ -55,7 +55,7 @@ /* This test uses many POSIX things that are not available on * Windows. */ -#ifdef H5_HAVE_UNISTD_H +#if defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) #include "use.h" @@ -265,7 +265,7 @@ done: return (ret_value); } /* end main() */ -#else /* H5_HAVE_UNISTD_H */ +#else /* defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) */ int main(void) @@ -274,4 +274,4 @@ main(void) return EXIT_SUCCESS; } /* end main() */ -#endif /* H5_HAVE_UNISTD_H */ +#endif /* defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) */ diff --git a/test/use_common.c b/test/use_common.c index 8f78f3b..7b5df53 100644 --- a/test/use_common.c +++ b/test/use_common.c @@ -298,7 +298,7 @@ write_uc_file(hbool_t tosend, hid_t file_id, options_t *opts) return -1; } - /* verify chunk_dims against set paramenters */ + /* verify chunk_dims against set parameters */ if (chunk_dims[0] != opts->chunkdims[0] || chunk_dims[1] != cz || chunk_dims[2] != cz) { HDfprintf(stderr, "chunk size is not as expected. Got dims=(%llu,%llu,%llu)\n", (unsigned long long)chunk_dims[0], (unsigned long long)chunk_dims[1], @@ -533,7 +533,7 @@ read_uc_file(hbool_t towait, options_t *opts) /* quit when all nplanes have been read */ loops_waiting_for_plane = 0; while (nplanes_seen < opts->nplanes) { - /* print progress message according to if new planes are availalbe */ + /* print progress message according to if new planes are available */ if (nplanes_seen < dims[0]) { if (loops_waiting_for_plane) { /* end the previous message */ diff --git a/test/use_disable_mdc_flushes.c b/test/use_disable_mdc_flushes.c index f28e574..d40e56d 100644 --- a/test/use_disable_mdc_flushes.c +++ b/test/use_disable_mdc_flushes.c @@ -360,7 +360,7 @@ write_file(void) return -1; } - /* verify chunk_dims against set paramenters */ + /* verify chunk_dims against set parameters */ if (chunk_dims[0] != chunkdims_g[0] || chunk_dims[1] != cz || chunk_dims[2] != cz) { HDfprintf(stderr, "chunk size is not as expected. Got dims=(%llu,%llu,%llu)\n", (unsigned long long)chunk_dims[0], (unsigned long long)chunk_dims[1], diff --git a/test/vds.c b/test/vds.c index ac9bb5d..4455ad4 100644 --- a/test/vds.c +++ b/test/vds.c @@ -196,7 +196,7 @@ vds_select_equal(hid_t space1, hid_t space2) if (nblocks1 != nblocks2) return FALSE; - /* Allocate block lists. Do not return directly afer + /* Allocate block lists. Do not return directly after * allocating, to make sure buffers are freed. */ if (NULL == (buf1 = (hsize_t *)HDmalloc((size_t)2 * (size_t)rank1 * (size_t)nblocks1 * sizeof(*buf1)))) @@ -479,7 +479,7 @@ test_api_get_ex_dcpl(test_api_config_t config, hid_t fapl, hid_t dcpl, hid_t *ex /* Verify examination DCPL is equal to original DCPL. Do not compare the * plist to itself, and do not do the comparison if we reopened the file, - * because in that case the extent of the source dset will not be corrent. + * because in that case the extent of the source dset will not be current. */ if ((*ex_dcpl != dcpl) && (config != TEST_API_REOPEN_FILE)) { if ((tri_ret = H5Pequal(dcpl, *ex_dcpl)) < 0) @@ -1044,7 +1044,7 @@ test_api(test_api_config_t config, hid_t fapl, H5F_libver_t low) if ((vspace[i] = H5Screate_simple(2, dims, NULL)) < 0) TEST_ERROR - /* Select row in virual dataspace */ + /* Select row in virtual dataspace */ start[0] = (hsize_t)i; if (H5Sselect_hyperslab(vspace[i], H5S_SELECT_SET, start, NULL, count, block) < 0) TEST_ERROR @@ -1182,7 +1182,7 @@ test_vds_prefix_first(unsigned config, hid_t vds_fapl, hid_t src_fapl) hid_t srcspace[4] = {-1, -1, -1, -1}; /* Source dataspaces */ hid_t vspace[4] = {-1, -1, -1, -1}; /* Virtual dset dataspaces */ hid_t memspace = -1; /* Memory dataspace */ - hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datsets */ + hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datasets */ hid_t vdset = -1; /* Virtual dataset */ hsize_t dims[4] = {10, 26, 0, 0}; /* Data space current size */ int buf[10][26]; /* Write and expected read buffer */ @@ -1461,7 +1461,7 @@ test_basic_io(unsigned config, hid_t vds_fapl, hid_t src_fapl) hid_t srcspace[4] = {-1, -1, -1, -1}; /* Source dataspaces */ hid_t vspace[4] = {-1, -1, -1, -1}; /* Virtual dset dataspaces */ hid_t memspace = -1; /* Memory dataspace */ - hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datsets */ + hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datasets */ hid_t vdset = -1; /* Virtual dataset */ hsize_t dims[4] = {10, 26, 0, 0}; /* Data space current size */ hsize_t start[4]; /* Hyperslab start */ @@ -4389,7 +4389,7 @@ test_unlim(unsigned config, hid_t vds_fapl, hid_t src_fapl) hid_t vspace[4] = {-1, -1, -1, -1}; /* Virtual dset dataspaces */ hid_t memspace = -1; /* Memory dataspace */ hid_t filespace = -1; /* File dataspace */ - hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datsets */ + hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datasets */ hid_t vdset = -1; /* Virtual dataset */ hsize_t dims[2] = {10, 10}; /* Data space current size */ hsize_t mdims[2] = {10, 20}; /* Data space maximum size */ @@ -7403,7 +7403,7 @@ test_printf(unsigned config, hid_t vds_fapl, hid_t src_fapl) hid_t vspace[2] = {-1, -1}; /* Virtual dset dataspaces */ hid_t memspace = -1; /* Memory dataspace */ hid_t filespace = -1; /* File dataspace */ - hid_t srcdset[6] = {-1, -1, -1, -1, -1, -1}; /* Source datsets */ + hid_t srcdset[6] = {-1, -1, -1, -1, -1, -1}; /* Source datasets */ hid_t vdset = -1; /* Virtual dataset */ hsize_t dims[2] = {10, 0}; /* Data space current size */ hsize_t mdims[2] = {10, 20}; /* Data space maximum size */ @@ -11111,7 +11111,7 @@ test_all(unsigned config, hid_t vds_fapl, hid_t src_fapl) hid_t vspace[3] = {-1, -1, -1}; /* Virtual dset dataspaces */ hid_t memspace = -1; /* Memory dataspace */ hid_t filespace = -1; /* File dataspace */ - hid_t srcdset[5] = {-1, -1, -1, -1, -1}; /* Source datsets */ + hid_t srcdset[5] = {-1, -1, -1, -1, -1}; /* Source datasets */ hid_t vdset = -1; /* Virtual dataset */ hsize_t dims[2] = {6, 6}; /* Data space current size */ hsize_t mdims[2] = {10, 10}; /* Data space maximum size */ @@ -12288,9 +12288,24 @@ main(void) hid_t src_fapl = -1; /* File access property list */ int test_api_config; unsigned bit_config; - H5F_libver_t low, high; /* Low and high bounds */ + H5F_libver_t low, high; /* Low and high bounds */ + const char * env_h5_drvr; /* File Driver value from environment */ int nerrors = 0; + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* + * Skip VDS tests for parallel-enabled and splitter VFDs. VDS currently + * doesn't support parallel reads and the splitter VFD has external + * link-related bugs. + */ + if (h5_using_parallel_driver(env_h5_drvr) || !HDstrcmp(env_h5_drvr, "splitter")) { + HDputs(" -- SKIPPED for incompatible VFD --"); + HDexit(EXIT_SUCCESS); + } + /* Testing setup */ h5_reset(); fapl = h5_fileaccess(); diff --git a/test/vds_env.c b/test/vds_env.c index 58d90fc..e01f2e0 100644 --- a/test/vds_env.c +++ b/test/vds_env.c @@ -58,7 +58,7 @@ test_vds_prefix_second(unsigned config, hid_t fapl) hid_t srcspace[4] = {-1, -1, -1, -1}; /* Source dataspaces */ hid_t vspace[4] = {-1, -1, -1, -1}; /* Virtual dset dataspaces */ hid_t memspace = -1; /* Memory dataspace */ - hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datsets */ + hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datasets */ hid_t vdset = -1; /* Virtual dataset */ hsize_t dims[4] = {10, 26, 0, 0}; /* Data space current size */ int buf[10][26]; /* Write and expected read buffer */ @@ -298,9 +298,24 @@ main(void) { hid_t fapl, my_fapl; unsigned bit_config; - H5F_libver_t low, high; /* Low and high bounds */ + H5F_libver_t low, high; /* Low and high bounds */ + const char * env_h5_drvr; /* File Driver value from environment */ int nerrors = 0; + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* + * Skip VDS tests for parallel-enabled and splitter VFDs. VDS currently + * doesn't support parallel reads and the splitter VFD has external + * link-related bugs. + */ + if (h5_using_parallel_driver(env_h5_drvr) || !HDstrcmp(env_h5_drvr, "splitter")) { + HDputs(" -- SKIPPED for incompatible VFD --"); + HDexit(EXIT_SUCCESS); + } + /* Testing setup */ h5_reset(); fapl = h5_fileaccess(); @@ -331,8 +346,9 @@ main(void) /* Display testing info */ low_string = h5_get_version_string(low); high_string = h5_get_version_string(high); - HDsprintf(msg, "Testing virtual dataset with file version bounds: (%s, %s):", low_string, - high_string); + HDsnprintf(msg, sizeof(msg), + "Testing virtual dataset with file version bounds: (%s, %s):", low_string, + high_string); HDputs(msg); for (bit_config = 0; bit_config < TEST_IO_NTESTS; bit_config++) { diff --git a/test/vfd.c b/test/vfd.c index f4cb988..4f28766 100644 --- a/test/vfd.c +++ b/test/vfd.c @@ -48,25 +48,27 @@ #define DSET2_DIM 4 #endif /* H5_HAVE_DIRECT */ -const char *FILENAME[] = {"sec2_file", /*0*/ - "core_file", /*1*/ - "family_file", /*2*/ - "new_family_v16_", /*3*/ - "multi_file", /*4*/ - "direct_file", /*5*/ - "log_file", /*6*/ - "stdio_file", /*7*/ - "windows_file", /*8*/ - "new_multi_file_v16", /*9*/ - "ro_s3_file", /*10*/ - "splitter_rw_file", /*11*/ - "splitter_wo_file", /*12*/ - "splitter.log", /*13*/ +const char *FILENAME[] = {"sec2_file", /*0*/ + "core_file", /*1*/ + "family_file", /*2*/ + "new_family_v16", /*3*/ + "multi_file", /*4*/ + "direct_file", /*5*/ + "log_file", /*6*/ + "stdio_file", /*7*/ + "windows_file", /*8*/ + "new_multi_file_v16", /*9*/ + "ro_s3_file", /*10*/ + "splitter_rw_file", /*11*/ + "splitter_wo_file", /*12*/ + "splitter.log", /*13*/ + "ctl_file", /*14*/ + "ctl_splitter_wo_file", /*15*/ NULL}; #define LOG_FILENAME "log_vfd_out.log" -#define COMPAT_BASENAME "family_v16_" +#define COMPAT_BASENAME "family_v16" #define MULTI_COMPAT_BASENAME "multi_file_v16" #define SPLITTER_DATASET_NAME "dataset" @@ -94,6 +96,13 @@ struct splitter_dataset_def { int n_dims; /* rank */ }; +/* Op code type enum for ctl callback test */ +typedef enum { + CTL_OPC_KNOWN_PASSTHROUGH, /* op code known to passthrough VFD */ + CTL_OPC_KNOWN_TERMINAL, /* op code known to terminal VFD */ + CTL_OPC_UNKNOWN /* unknown op code */ +} ctl_test_opc_type; + static int splitter_prepare_file_paths(H5FD_splitter_vfd_config_t *vfd_config, char *filename_rw_out); static int splitter_create_single_file_at(const char *filename, hid_t fapl_id, const struct splitter_dataset_def *data); @@ -104,6 +113,19 @@ static int splitter_RO_test(const struct splitter_dataset_def *data, hid_t child static int splitter_tentative_open_test(hid_t child_fapl_id); static int file_exists(const char *filename, hid_t fapl_id); +static herr_t run_ctl_test(uint64_t op_code, uint64_t flags, ctl_test_opc_type opc_type, hid_t fapl_id); +static H5FD_t *H5FD__ctl_test_vfd_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__ctl_test_vfd_close(H5FD_t *_file); +static haddr_t H5FD__ctl_test_vfd_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__ctl_test_vfd_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD__ctl_test_vfd_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__ctl_test_vfd_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, + size_t size, void *buf); +static herr_t H5FD__ctl_test_vfd_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, + size_t size, const void *buf); +static herr_t H5FD__ctl_test_vfd_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, + void **output); + /*------------------------------------------------------------------------- * Function: test_sec2 * @@ -186,7 +208,7 @@ test_sec2(void) if (os_file_handle == NULL) FAIL_PUTS_ERROR("NULL os-specific vfd/file handle was returned from H5Fget_vfd_handle"); - /* There is no garantee the size of metadata in file is constant. + /* There is no guarantee the size of metadata in file is constant. * Just try to check if it's reasonable. * * Currently it should be around 2 KB. @@ -382,7 +404,7 @@ test_core(void) if (os_file_handle == NULL) FAIL_PUTS_ERROR("NULL os-specific vfd/file handle was returned from H5Fget_vfd_handle"); - /* There is no garantee the size of metadata in file is constant. + /* There is no guarantee the size of metadata in file is constant. * Just try to check if it's reasonable. * * TODO: Needs justification of why is this is a reasonable size. @@ -529,7 +551,7 @@ test_core(void) } /* end if */ /* Check file size API. - * There is no garantee the size of metadata in file is constant. + * There is no guarantee the size of metadata in file is constant. * Just try to check if it's reasonable. * * TODO: Needs justification of why is this is a reasonable size. @@ -1133,7 +1155,7 @@ error: * See if we can open files created with v1.6 library. * The source file was created by the test/file_handle.c * of the v1.6 library. Then tools/misc/h5repart.c was - * used to concantenated. The command was "h5repart -m 5k + * used to concatenated. The command was "h5repart -m 5k * family_file%05d.h5 family_v16_%05d.h5". * * Return: Success: 0 @@ -1401,7 +1423,7 @@ H5_GCC_CLANG_DIAG_ON("format-nonliteral") /*------------------------------------------------------------------------- * Function: test_multi * - * Purpose: Tests the file handle interface for MUTLI driver + * Purpose: Tests the file handle interface for MULTI driver * * Return: SUCCEED/FAIL * @@ -2556,7 +2578,7 @@ done: * Function: driver_is_splitter_compatible * * Purpose: Determine whether the driver set in the FAPL ID is compatible - * with the Splitter VFD -- specificially, Write-Only channel. + * with the Splitter VFD -- specifically, Write-Only channel. * * Return: Success: 0 * Failure: -1 @@ -3129,7 +3151,7 @@ splitter_tentative_open_test(hid_t child_fapl_id) SPLITTER_TEST_FAULT("can't close file ID\n"); } if (!file_exists(filename_rw, child_fapl_id)) { - SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + SPLITTER_TEST_FAULT("R/W file mysteriously disappeared\n"); } if (!file_exists(vfd_config->wo_path, child_fapl_id)) { SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); @@ -3149,7 +3171,7 @@ splitter_tentative_open_test(hid_t child_fapl_id) SPLITTER_TEST_FAULT("can't close file ID\n"); } if (!file_exists(filename_rw, child_fapl_id)) { - SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + SPLITTER_TEST_FAULT("R/W file mysteriously disappeared\n"); } if (!file_exists(vfd_config->wo_path, child_fapl_id)) { SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); @@ -3180,7 +3202,7 @@ splitter_tentative_open_test(hid_t child_fapl_id) SPLITTER_TEST_FAULT("can't close file ID\n"); } if (!file_exists(filename_rw, child_fapl_id)) { - SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + SPLITTER_TEST_FAULT("R/W file mysteriously disappeared\n"); } if (!file_exists(vfd_config->wo_path, child_fapl_id)) { SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); @@ -3211,7 +3233,7 @@ splitter_tentative_open_test(hid_t child_fapl_id) SPLITTER_TEST_FAULT("can't close file ID\n"); } if (!file_exists(filename_rw, child_fapl_id)) { - SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + SPLITTER_TEST_FAULT("R/W file mysteriously disappeared\n"); } if (!file_exists(vfd_config->wo_path, child_fapl_id)) { SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); @@ -3398,6 +3420,500 @@ error: #undef SPLITTER_TEST_FAULT +/* + * Callback implementations for ctl feature testing VFD + */ +static H5FD_t * +H5FD__ctl_test_vfd_open(const char H5_ATTR_UNUSED *name, unsigned H5_ATTR_UNUSED flags, + hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED maxaddr) +{ + return HDcalloc(1, sizeof(H5FD_t)); +} +static herr_t +H5FD__ctl_test_vfd_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + HDfree(_file); + return SUCCEED; +} +static haddr_t +H5FD__ctl_test_vfd_get_eoa(const H5FD_t H5_ATTR_UNUSED *file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + return HADDR_UNDEF; +} +static herr_t +H5FD__ctl_test_vfd_set_eoa(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t H5_ATTR_UNUSED addr) +{ + return FAIL; +} +static haddr_t +H5FD__ctl_test_vfd_get_eof(const H5FD_t H5_ATTR_UNUSED *file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + return HADDR_UNDEF; +} +static herr_t +H5FD__ctl_test_vfd_read(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, + void H5_ATTR_UNUSED *buf) +{ + return FAIL; +} +static herr_t +H5FD__ctl_test_vfd_write(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf) +{ + return FAIL; +} +static herr_t +H5FD__ctl_test_vfd_ctl(H5FD_t H5_ATTR_UNUSED *_file, uint64_t op_code, uint64_t flags, + const void H5_ATTR_UNUSED *input, void H5_ATTR_UNUSED **output) +{ + herr_t ret_value = SUCCEED; + + switch (op_code) { + /* Op code for testing purposes */ + case H5FD_CTL__TEST_OPCODE: + break; + + /* Unknown op code */ + default: + if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) + ret_value = FAIL; + break; + } + + return ret_value; +} + +/* Minimal VFD for ctl feature tests */ +static const H5FD_class_t H5FD_ctl_test_vfd_g = { + (H5FD_class_value_t)201, /* value */ + "ctl_test_vfd", /* name */ + HADDR_MAX, /* maxaddr */ + H5F_CLOSE_SEMI, /* fc_degree */ + NULL, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + 0, /* fapl_size */ + NULL, /* fapl_get */ + NULL, /* fapl_copy */ + NULL, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__ctl_test_vfd_open, /* open */ + H5FD__ctl_test_vfd_close, /* close */ + NULL, /* cmp */ + NULL, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD__ctl_test_vfd_get_eoa, /* get_eoa */ + H5FD__ctl_test_vfd_set_eoa, /* set_eoa */ + H5FD__ctl_test_vfd_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD__ctl_test_vfd_read, /* read */ + H5FD__ctl_test_vfd_write, /* write */ + NULL, /* flush */ + NULL, /* truncate */ + NULL, /* lock */ + NULL, /* unlock */ + NULL, /* del */ + H5FD__ctl_test_vfd_ctl, /* ctl */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/*------------------------------------------------------------------------- + * Function: run_ctl_test + * + * Purpose: Helper method for VFD "ctl" callback test + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +run_ctl_test(uint64_t op_code, uint64_t flags, ctl_test_opc_type opc_type, hid_t fapl_id) +{ + hbool_t fail_if_unknown = FALSE; + hbool_t routing_flag_set = FALSE; + hbool_t is_passthrough_vfd = FALSE; + hbool_t expect_fail = FALSE; + H5FD_t *file_drv_ptr = NULL; + herr_t ctl_result = SUCCEED; + hid_t driver_id = H5I_INVALID_HID; + char filename[1024]; + + /* Check for a few ctl function flags */ + fail_if_unknown = (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG); + routing_flag_set = (flags & H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG); + + /* Determine if the top-level VFD is a passthrough VFD */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + PUTS_ERROR("couldn't get VFD ID from FAPL"); + + is_passthrough_vfd = ((driver_id == H5FD_SPLITTER) || (driver_id == H5FD_MULTI)); + + /* + * "Open" testing file. Note that our VFD for testing the ctl + * feature doesn't actually create or open files, so we don't + * need to create the testing file; we just need the VFD to + * give us a pointer to a H5FD_t structure. + */ + h5_fixname(FILENAME[14], fapl_id, filename, sizeof(filename)); + if (NULL == (file_drv_ptr = H5FDopen(filename, H5F_ACC_RDWR, fapl_id, HADDR_UNDEF))) + PUTS_ERROR("couldn't get pointer to H5FD_t structure"); + + /* Determine whether the H5FDctl call is expected to fail */ + expect_fail = fail_if_unknown && (CTL_OPC_UNKNOWN == opc_type); + if (is_passthrough_vfd) { + /* Should fail if op code is unknown to passthrough VFD + * (but known to terminal VFD), no routing flag is specified + * and the "fail if unknown" flag is specified. + */ + expect_fail = + expect_fail || ((CTL_OPC_KNOWN_TERMINAL == opc_type) && !routing_flag_set && fail_if_unknown); + } + + /* Issue opcode to VFD */ + if (expect_fail) { + H5E_BEGIN_TRY + { + ctl_result = H5FDctl(file_drv_ptr, op_code, flags, NULL, NULL); + } + H5E_END_TRY; + } + else + ctl_result = H5FDctl(file_drv_ptr, op_code, flags, NULL, NULL); + + /* Verify result of H5FDctl call */ + if (expect_fail) { + if (ctl_result == SUCCEED) + PUTS_ERROR("H5FDctl call succeeded when it should have failed"); + } + else { + if (ctl_result != SUCCEED) + PUTS_ERROR("H5FDctl call failed when it should have succeeded"); + } + + /* Close H5FD_t structure pointer */ + if (H5FDclose(file_drv_ptr) < 0) + PUTS_ERROR("couldn't close H5FD_t structure pointer"); + file_drv_ptr = NULL; + + return 0; + +error: + H5E_BEGIN_TRY + { + H5FDclose(file_drv_ptr); + } + H5E_END_TRY; + + return -1; +} + +/*------------------------------------------------------------------------- + * Function: test_ctl + * + * Purpose: Tests the VFD "ctl" callback + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +test_ctl(void) +{ + H5FD_splitter_vfd_config_t *splitter_config = NULL; + uint64_t op_code; + uint64_t flags; + hid_t driver_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t sub_fapl_id = H5I_INVALID_HID; + + TESTING("VFD ctl callback"); + HDputs(""); + + /* Register VFD for test */ + if ((driver_id = H5FDregister(&H5FD_ctl_test_vfd_g)) < 0) + PUTS_ERROR("couldn't register VFD for testing"); + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + PUTS_ERROR("couldn't create FAPL"); + if (H5Pset_driver(fapl_id, driver_id, NULL) < 0) + PUTS_ERROR("couldn't set testing VFD on FAPL"); + + TESTING_2("known op code to terminal VFD (without fail on unknown flag)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = 0; + + /* H5FDctl call should succeed normally */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("known op code to terminal VFD (with fail on unknown flag)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG; + + /* H5FDctl call should succeed normally */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("known op code to terminal VFD (without fail on unknown flag/route to terminal VFD)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* H5FDctl call should succeed normally */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("known op code to terminal VFD (with fail on unknown flag/route to terminal VFD)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* H5FDctl call should succeed normally */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to terminal VFD (without fail on unknown flag)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = 0; + + /* H5FDctl call should silently ignore unknown op code and succeed */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to terminal VFD (with fail on unknown flag)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG; + + /* H5FDctl call should fail due to 'fail if unknown' flag being specified */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to terminal VFD (without fail on unknown flag/route to terminal VFD)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* H5FDctl call should silently ignore unknown op code and succeed */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to terminal VFD (with fail on unknown flag/route to terminal VFD)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* H5FDctl call should fail due to 'fail if unknown' flag being specified */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + /* Set up splitter VFD config */ + if (NULL == (splitter_config = HDcalloc(1, sizeof(H5FD_splitter_vfd_config_t)))) + TEST_ERROR; + + splitter_config->magic = H5FD_SPLITTER_MAGIC; + splitter_config->version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + splitter_config->ignore_wo_errs = TRUE; + splitter_config->rw_fapl_id = H5P_DEFAULT; + splitter_config->wo_fapl_id = H5P_DEFAULT; + h5_fixname(FILENAME[15], splitter_config->wo_fapl_id, splitter_config->wo_path, H5FD_SPLITTER_PATH_MAX); + + if ((sub_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + PUTS_ERROR("couldn't create FAPL"); + if (H5Pset_driver(sub_fapl_id, driver_id, NULL) < 0) + PUTS_ERROR("couldn't set testing VFD on FAPL"); + splitter_config->rw_fapl_id = sub_fapl_id; + + if (H5Pset_fapl_splitter(fapl_id, splitter_config) < 0) + PUTS_ERROR("couldn't set splitter VFD on FAPL"); + + TESTING_2("known op code through passthrough VFD to terminal VFD (without fail on unknown flag/no " + "routing flag)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = 0; + + /* + * H5FDctl call should silently ignore unknown op code in + * passthrough VFD since no routing flag is specified and + * 'fail if unknown' flag is not specified. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2( + "known op code through passthrough VFD to terminal VFD (with fail on unknown flag/no routing flag)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG; + + /* + * H5FDctl call should fail since op code is unknown to + * passthrough VFD (though known to terminal VFD), no + * routing flag is specified and the 'fail if unknown' + * flag is specified. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("known op code through passthrough VFD to terminal VFD (without fail on unknown flag/route to " + "terminal VFD)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* + * H5Dctl call should succeed since the passthrough VFD + * doesn't recognize the op code, but has been instructed + * to route it down to the terminal VFD. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("known op code through passthrough VFD to terminal VFD (with fail on unknown flag/route to " + "terminal VFD)") + + op_code = H5FD_CTL__TEST_OPCODE; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* + * H5Dctl call should succeed since the passthrough VFD + * doesn't recognize the op code, but has been instructed + * to route it down to the terminal VFD. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_KNOWN_TERMINAL, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to passthrough VFD (without fail on unknown flag)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = 0; + + /* + * H5FDctl call should silently ignore unknown op code in + * passthrough VFD since no routing flag is specified and + * 'fail if unknown' flag is not specified. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to passthrough VFD (with fail on unknown flag)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG; + + /* + * H5FDctl call should fail since op code is unknown to + * passthrough VFD, no routing flag is specified and the + * 'fail if unknown' flag is specified. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to passthrough VFD (without fail on unknown flag/route to terminal VFD)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* + * H5Dctl call should succeed since the passthrough VFD + * doesn't recognize the op code, but has been instructed + * to route it down to the terminal VFD and the 'fail if + * unknown' flag has not been specified. Therefore, the + * terminal VFD should silently ignore the unknown op + * code. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("unknown op code to passthrough VFD (with fail on unknown flag/route to terminal VFD)") + + op_code = H5FD_CTL_OPC_RESERVED; + flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + + /* + * H5Dctl call should fail since the passthrough VFD + * doesn't recognize the op code, but has been instructed + * to route it down to the terminal VFD and the 'fail if + * unknown' flag has been specified. Therefore, the + * terminal VFD will throw an error for the unknown op + * code. + */ + if (run_ctl_test(op_code, flags, CTL_OPC_UNKNOWN, fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + TESTING_2("test cleanup") + + HDfree(splitter_config); + + if (H5FDunregister(driver_id) < 0) + TEST_ERROR; + if (H5Pclose(sub_fapl_id) < 0) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY + { + if (splitter_config) + HDfree(splitter_config); + H5FDunregister(driver_id); + H5Pclose(sub_fapl_id); + H5Pclose(fapl_id); + } + H5E_END_TRY; + + return -1; +} + /*------------------------------------------------------------------------- * Function: main * @@ -3410,7 +3926,18 @@ error: int main(void) { - int nerrors = 0; + char *env_h5_drvr = NULL; + int nerrors = 0; + + /* Don't run VFD tests when HDF5_DRIVER is set. These tests expect a + * specific VFD to be set and HDF5_DRIVER being set can interfere + * with that. + */ + env_h5_drvr = HDgetenv(HDF5_DRIVER); + if (env_h5_drvr) { + HDprintf(" -- SKIPPED VFD tests because %s is set -- \n", HDF5_DRIVER); + HDexit(EXIT_SUCCESS); + } h5_reset(); @@ -3429,6 +3956,7 @@ main(void) nerrors += test_windows() < 0 ? 1 : 0; nerrors += test_ros3() < 0 ? 1 : 0; nerrors += test_splitter() < 0 ? 1 : 0; + nerrors += test_ctl() < 0 ? 1 : 0; if (nerrors) { HDprintf("***** %d Virtual File Driver TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); diff --git a/test/vfd_plugin.c b/test/vfd_plugin.c new file mode 100644 index 0000000..182c048 --- /dev/null +++ b/test/vfd_plugin.c @@ -0,0 +1,439 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Tests basic VFD plugin operations. Uses a basic testing VFD + * which is loaded as a dynamic plugin. + */ + +#include "h5test.h" + +#include "null_vfd_plugin.h" + +#define DEFAULT_DRIVER_NAME "sec2" + +/*------------------------------------------------------------------------- + * Function: test_set_by_name() + * + * Purpose: Tests if we can load and register a VFD plugin by name. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_set_by_name(void) +{ + htri_t is_registered = FAIL; + hid_t driver_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + + TESTING("VFD plugin registration by name"); + + /* The null VFD should not be registered at the start of the test */ + if ((is_registered = H5FDis_driver_registered_by_name(NULL_VFD_NAME)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + /* Register the null VFD by name */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_driver_by_name(fapl_id, NULL_VFD_NAME, NULL) < 0) + TEST_ERROR; + + /* The null VFD should be registered now */ + if ((is_registered = H5FDis_driver_registered_by_name(NULL_VFD_NAME)) < 0) + TEST_ERROR; + if (FALSE == is_registered) + FAIL_PUTS_ERROR("NULL VFD was not registered"); + + /* Unregister the null VFD */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR; + if (H5FDunregister(driver_id) < 0) + TEST_ERROR; + + /* Close FAPL, which holds last reference to null VFD */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + /* The null VFD should not be registered now */ + if ((is_registered = H5FDis_driver_registered_by_name(NULL_VFD_NAME)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + } + H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_set_by_value() + * + * Purpose: Tests if we can load and register a VFD plugin by value + * (ID). + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_set_by_value(void) +{ + htri_t is_registered = FAIL; + hid_t driver_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + + TESTING("VFD plugin registration by value (ID)"); + + /* The null VFD should not be registered at the start of the test */ + if ((is_registered = H5FDis_driver_registered_by_value(NULL_VFD_VALUE)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + /* Register the null VFD by value */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_driver_by_value(fapl_id, NULL_VFD_VALUE, NULL) < 0) + TEST_ERROR; + + /* The null VFD should be registered now */ + if ((is_registered = H5FDis_driver_registered_by_value(NULL_VFD_VALUE)) < 0) + TEST_ERROR; + if (FALSE == is_registered) + FAIL_PUTS_ERROR("NULL VFD was not registered"); + + /* Unregister the null VFD */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR; + if (H5FDunregister(driver_id) < 0) + TEST_ERROR; + + /* Close FAPL, which holds last reference to null VFD */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + /* The null VFD should not be registered now */ + if ((is_registered = H5FDis_driver_registered_by_value(NULL_VFD_VALUE)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + } + H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_set_multi() + * + * Purpose: Tests if we can register a VFD plugin multiple times. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +#define N_REGISTRATIONS 10 +static herr_t +test_set_multi(void) +{ + htri_t is_registered = FAIL; + hid_t driver_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + int i; + + TESTING("registering a VFD plugin multiple times"); + + /* The null VFD should not be registered at the start of the test */ + if ((is_registered = H5FDis_driver_registered_by_name(NULL_VFD_NAME)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + /* Register the VFD multiple times */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + for (i = 0; i < N_REGISTRATIONS; i++) { + if (H5Pset_driver_by_name(fapl_id, NULL_VFD_NAME, NULL) < 0) + TEST_ERROR; + } + + /* The null VFD should be registered now */ + if ((is_registered = H5FDis_driver_registered_by_name(NULL_VFD_NAME)) < 0) + TEST_ERROR; + if (FALSE == is_registered) + FAIL_PUTS_ERROR("NULL VFD was not registered"); + + /* Unregister the null VFD */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR; + for (i = 0; i < N_REGISTRATIONS; i++) { + if (H5FDunregister(driver_id) < 0) + TEST_ERROR; + } + + /* Close FAPL, which holds last reference to null VFD */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + /* The null VFD should not be registered now */ + if ((is_registered = H5FDis_driver_registered_by_name(NULL_VFD_NAME)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + /* Repeat testing with the _by_value routines */ + + /* The null VFD should not be registered at the start of the test */ + if ((is_registered = H5FDis_driver_registered_by_value(NULL_VFD_VALUE)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + /* Register the VFD multiple times */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + for (i = 0; i < N_REGISTRATIONS; i++) { + if (H5Pset_driver_by_value(fapl_id, NULL_VFD_VALUE, NULL) < 0) + TEST_ERROR; + } + + /* The null VFD should be registered now */ + if ((is_registered = H5FDis_driver_registered_by_value(NULL_VFD_VALUE)) < 0) + TEST_ERROR; + if (FALSE == is_registered) + FAIL_PUTS_ERROR("NULL VFD was not registered"); + + /* Unregister the null VFD */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR; + for (i = 0; i < N_REGISTRATIONS; i++) { + if (H5FDunregister(driver_id) < 0) + TEST_ERROR; + } + + /* Close FAPL, which holds last reference to null VFD */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + /* The null VFD should not be registered now */ + if ((is_registered = H5FDis_driver_registered_by_value(NULL_VFD_VALUE)) < 0) + TEST_ERROR; + if (TRUE == is_registered) + FAIL_PUTS_ERROR("NULL VFD is inappropriately registered"); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + } + H5E_END_TRY; + + return FAIL; +} +#undef N_REGISTRATIONS + +/*------------------------------------------------------------------------- + * Function: test_get_config_str() + * + * Purpose: Tests if we can retrieve a configuration string set for a + * VFL driver. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_get_config_str(void) +{ + const char *const config_str = "{name: sec2}"; + ssize_t config_str_len = 0; + hid_t fapl_id = H5I_INVALID_HID; + char config_str_buf[128]; + + TESTING("Retrieval of VFD configuration string"); + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + /* Try to retrieve length of default configuration string - should be 0 */ + HDmemset(config_str_buf, 0, 128); + + if ((config_str_len = H5Pget_driver_config_str(fapl_id, config_str_buf, 128)) < 0) + TEST_ERROR; + if (0 != config_str_len) + TEST_ERROR; + if (HDstrlen(config_str_buf) > 0) + TEST_ERROR; + + /* Set a new configuration string on the FAPL and retrieve it */ + if (H5Pset_driver_by_name(fapl_id, DEFAULT_DRIVER_NAME, config_str) < 0) + TEST_ERROR; + if ((config_str_len = H5Pget_driver_config_str(fapl_id, config_str_buf, 128)) < 0) + TEST_ERROR; + if (HDstrlen(config_str) != config_str_len) + TEST_ERROR; + if (HDstrncmp(config_str_buf, config_str, 128)) + TEST_ERROR; + + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + } + H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_env_var + * + * Purpose: Tests loading of NULL VFD plugin with HDF5_DRIVER + * environment variable and setting of VFD configuration + * string with HDF5_DRIVER_CONFIG environment variable + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + *------------------------------------------------------------------------- + */ +static int +test_env_var(void) +{ + const char *const config_str = "{name: null}"; + ssize_t config_str_len = 0; + htri_t driver_is_registered; + char config_str_buf[128]; + + TESTING("Loading of VFD plugin with HDF5_DRIVER environment variable"); + + /* Try to retrieve length of default configuration string - should be 0 */ + HDmemset(config_str_buf, 0, 128); + + if ((config_str_len = H5Pget_driver_config_str(H5P_FILE_ACCESS_DEFAULT, config_str_buf, 128)) < 0) + TEST_ERROR; + if (0 != config_str_len) + TEST_ERROR; + if (HDstrlen(config_str_buf) > 0) + TEST_ERROR; + + /* Set default driver and driver configuration using environment variables */ + if (HDsetenv(HDF5_DRIVER, "null_vfd_plugin", 1) < 0) + TEST_ERROR; + if (HDsetenv(HDF5_DRIVER_CONFIG, config_str, 1) < 0) + TEST_ERROR; + + /* Close and re-open HDF5 to have it parse the environment variables */ + if (H5close() < 0) + TEST_ERROR; + if (H5open() < 0) + TEST_ERROR; + + /* Check driver */ + if ((driver_is_registered = H5FDis_driver_registered_by_name("null_vfd_plugin")) < 0) + TEST_ERROR; + if (!driver_is_registered) + TEST_ERROR; + if (H5Pget_driver(H5P_FILE_ACCESS_DEFAULT) == H5_DEFAULT_VFD) + TEST_ERROR; + + /* Check driver configuration string */ + HDmemset(config_str_buf, 0, 128); + if ((config_str_len = H5Pget_driver_config_str(H5P_FILE_ACCESS_DEFAULT, config_str_buf, 128)) < 0) + TEST_ERROR; + if (HDstrlen(config_str) != config_str_len) + TEST_ERROR; + if (HDstrncmp(config_str_buf, config_str, 128)) + TEST_ERROR; + + /* Unset environment variables */ + if (HDsetenv(HDF5_DRIVER, "", 1) < 0) + TEST_ERROR; + if (HDsetenv(HDF5_DRIVER_CONFIG, "", 1) < 0) + TEST_ERROR; + + PASSED(); + + return SUCCEED; + +error: + HDsetenv(HDF5_DRIVER, "", 1); + HDsetenv(HDF5_DRIVER_CONFIG, "", 1); + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Tests VFD plugin operations + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + + h5_reset(); + + HDputs("Testing VFD plugin functionality."); + + nerrors += (test_set_by_name() < 0) ? 1 : 0; + nerrors += (test_set_by_value() < 0) ? 1 : 0; + nerrors += (test_set_multi() < 0) ? 1 : 0; + nerrors += (test_get_config_str() < 0) ? 1 : 0; + nerrors += (test_env_var() < 0) ? 1 : 0; + + if (nerrors) { + HDprintf("***** %d VFD plugin TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); + HDexit(EXIT_FAILURE); + } + + HDputs("All VFD plugin tests passed."); + + HDexit(EXIT_SUCCESS); +} diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c index 8255bdd..0973dd8 100644 --- a/test/vfd_swmr.c +++ b/test/vfd_swmr.c @@ -113,7 +113,7 @@ static unsigned test_updater_generate_md_checksums(hbool_t file_create); * then updater_file_path field shouldn't be empty * B) Verify that info set in the fapl is retrieved correctly. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; July 2018 @@ -307,7 +307,7 @@ error: * used to create/open the file is the same as the * configuration retrieved from the file's fapl. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; July 2018 @@ -639,7 +639,7 @@ error: * Purpose: Verify the public routine H5Fvfd_swmr_end_tick() works * as described in the RFC for VFD SWMR. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; June 2020 @@ -882,7 +882,7 @@ error: * --Verify info in the header and index read from * the metadata file is as expected (empty index) * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2018 @@ -994,7 +994,7 @@ error: * file is as expected: * --num_entries and index * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2018 @@ -1297,7 +1297,7 @@ test_file_end_tick_concur(void) * Index entries are constructed at the front end by the * writer and verified at the back end by the reader. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2018 @@ -1634,7 +1634,7 @@ test_reader_md_concur(void) /* Get the file pointer */ file_writer = H5VL_object(fid_writer); - /* Send notification 1 to reader to start verfication */ + /* Send notification 1 to reader to start verification */ notify = 1; if (HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) FAIL_STACK_ERROR; @@ -1945,7 +1945,7 @@ error: * Purpose: Verify the entries on the EOT queue when opening files * with and without VFD SWMR configured. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; 11/18/2019 @@ -2258,7 +2258,7 @@ error: * enables/disables EOT when the files are opened * concurrently. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; June 2020 @@ -2485,7 +2485,7 @@ test_disable_enable_eot_concur(void) if ((fid_writer = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_writer)) < 0) FAIL_STACK_ERROR; - /* Send notification 1 to reader to start verfication */ + /* Send notification 1 to reader to start verification */ notify = 1; if (HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) FAIL_STACK_ERROR; @@ -2550,7 +2550,7 @@ error: * triggers end of tick processing when the files * are opened concurrently. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; June 2020 @@ -2759,7 +2759,7 @@ test_file_end_tick_concur(void) if ((fid_writer = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_writer)) < 0) FAIL_STACK_ERROR; - /* Send notification 1 to reader to start verfication */ + /* Send notification 1 to reader to start verification */ notify = 1; if (HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) FAIL_STACK_ERROR; @@ -2825,7 +2825,7 @@ error: * Purpose: Verify the entries on the EOT queue when opening files * with and without VFD SWMR configured. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; 11/18/2019 @@ -3018,7 +3018,7 @@ error: * f: the open fails * s: the open succeeds * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2019 @@ -3420,7 +3420,7 @@ out: * H5Fvfd_swmr_disable_end_of_tick() * enables/disables EOT for the specified file * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; June 2020 @@ -3694,7 +3694,7 @@ error: * Purpose: This is the helper routine used to verify whether * "flags" in the updater file is as expected. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2021 @@ -3738,7 +3738,7 @@ error: * Purpose: Verify "flags" in the updater file is as expected for * file creation. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2021 @@ -3881,7 +3881,7 @@ error: * Purpose: Verify "flags" in the updater file is as expected for * multiple opens of the same file. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2021 @@ -4074,7 +4074,7 @@ clean_chk_ud_files(char *md_file_path, char *updater_file_path) * is consistent with num_change_list_entries in each updater's * change list header * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2021 @@ -4189,7 +4189,7 @@ error: * --Generate checksum for the metadata file * --Write the tick number and the checksum to the checksum file * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2021 @@ -4288,7 +4288,7 @@ error: * The test is invoked when the file is created via H5Fcreate() * and via H5Fopen(). * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * * Programmer: Vailin Choi; October 2021 @@ -4391,7 +4391,7 @@ error: * * Purpose: Main function for VFD SWMR tests. * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * *------------------------------------------------------------------------- diff --git a/test/vfd_swmr_attrdset_writer.c b/test/vfd_swmr_attrdset_writer.c index d015ecf..6524b8b 100644 --- a/test/vfd_swmr_attrdset_writer.c +++ b/test/vfd_swmr_attrdset_writer.c @@ -1123,7 +1123,7 @@ add_attr(const state_t *s, hid_t did, unsigned int which) TEST_ERROR; } - /* Write to the attribure */ + /* Write to the attribute */ if (H5Awrite(aid, tid, s->vl_attr ? &val : (const void *)&which) < 0) { HDprintf("H5Awrite failed\n"); TEST_ERROR; @@ -1207,7 +1207,7 @@ modify_attr(const state_t *s, hid_t did, unsigned int which) TEST_ERROR; } - /* Write to the attribure */ + /* Write to the attribute */ if (H5Awrite(aid, tid, s->vl_attr ? &val : (const void *)&tmp_val) < 0) { HDprintf("H5Awrite failed\n"); TEST_ERROR; @@ -1277,7 +1277,7 @@ error: * * Also verify continuation block and compact<->dense storage if: * --[-c ] is 1 - * --not appliable for -m option + * --not applicable for -m option */ static bool verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *config, np_state_t *np) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 8a45a87..357c3b1 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -378,7 +378,7 @@ state_init(state_t *s, int argc, char **argv) s->vds = vds_single; break; case 'N': - /* Disable named pipes, mainly for running the writer and reader seperately */ + /* Disable named pipes, mainly for running the writer and reader separately */ s->use_named_pipe = false; break; case 'd': @@ -1168,7 +1168,7 @@ error: * --Generate checksum for the metadata file * --Write the tick number and the checksum to the checksum file * - * Return: 0 if test is sucessful + * Return: 0 if test is successful * 1 if test fails * *------------------------------------------------------------------------- @@ -2012,7 +2012,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini dset_id = s->dataset[which]; - /* Attempt to check the availablity of the chunks for a number times + /* Attempt to check the availability of the chunks for a number times * (NUM_ATTEMPTS) before reporting it as a failure */ for (i = 0; i < NUM_ATTEMPTS; i++) { if (H5Drefresh(dset_id) < 0) { @@ -2543,7 +2543,7 @@ error: int main(int argc, char **argv) { - mat_t * mat; + mat_t * mat = NULL; hid_t fcpl = H5I_INVALID_HID; unsigned which; state_t s; diff --git a/test/vfd_swmr_check_compat.c b/test/vfd_swmr_check_compat.c deleted file mode 100644 index b6e21a5..0000000 --- a/test/vfd_swmr_check_compat.c +++ /dev/null @@ -1,61 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by Akadio, Inc. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* Purpose: This is a small program that checks if the HDF5_DRIVER - * environment variable is set to a value that supports VFD SWMR. - * - * Also checks to see if this is a parallel build, as VFD SWMR - * and parallel HDF5 are fundamentally incompatible due to the - * page buffer changes. - * - * It is intended for use in shell scripts. - */ - -#include "h5test.h" - -/* This file needs to access the file driver testing code */ -#define H5FD_FRIEND /*suppress error about including H5FDpkg */ -#define H5FD_TESTING -#include "H5FDpkg.h" /* File drivers */ - -/*------------------------------------------------------------------------- - * Function: main - * - * Purpose: Inspects the HDF5_DRIVER environment variable, which - * determines the VFD that the test harness will use with - * the majority of the tests. - * - * Return: VFD supports SWMR: EXIT_SUCCESS - * - * VFD does not support SWMR - * or failure: EXIT_FAILURE - * - *------------------------------------------------------------------------- - */ -int -main(void) -{ -#ifdef H5_HAVE_PARALLEL - return EXIT_FAILURE; -#else - char *driver = NULL; - - driver = HDgetenv("HDF5_DRIVER"); - - /* Currently using SWMR support as a proxy for VFD SWMR support */ - if (H5FD__supports_swmr_test(driver)) - return EXIT_SUCCESS; - else - return EXIT_FAILURE; -#endif - -} /* end main() */ diff --git a/test/vfd_swmr_dsetchks_writer.c b/test/vfd_swmr_dsetchks_writer.c index c4eac7f..d649c3c 100644 --- a/test/vfd_swmr_dsetchks_writer.c +++ b/test/vfd_swmr_dsetchks_writer.c @@ -52,7 +52,7 @@ #define PWRITES 2 /* Writes that cover a partial chunk per write */ #define TWRITES 3 /* Writes that cover multiple chunks per write */ #define LWRITES 4 /* Writes that cover multiple partial chunks per write */ -#define INCR_EXT 5 /* Increase dataset dimenion sizes */ +#define INCR_EXT 5 /* Increase dataset dimension sizes */ #define DECR_EXT 6 /* Decrease dataset dimension sizes */ /* Fill values */ @@ -846,7 +846,7 @@ error: * * Dataset with fixed array/extensible array/version 2 btree index: * --INCR_EXT: increase dataset dimension sizes - * --DECR_EXT: decrease dataset dimenions sizes + * --DECR_EXT: decrease dataset dimension sizes */ static bool perform_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *config, np_state_t *np) @@ -1253,7 +1253,7 @@ error: } /* write_chunks() */ /* - * Increase or decrease the dimenion sizes for the specified datasets. + * Increase or decrease the dimension sizes for the specified datasets. */ static bool dsets_extent(unsigned action, const state_t *s, const dsets_state_t *ds) @@ -1827,7 +1827,7 @@ error: } /* verify_chunks() */ /* - * Verify the increase or decrease of dimenion sizes for the specified datasets. + * Verify the increase or decrease of dimension sizes for the specified datasets. */ static bool verify_dsets_extent(unsigned action, const state_t *s, const dsets_state_t *ds, unsigned which) @@ -2277,8 +2277,9 @@ error: int main(int argc, char **argv) { - hid_t fapl, fcpl; - bool writer; + hid_t fapl = H5I_INVALID_HID; + hid_t fcpl = H5I_INVALID_HID; + bool writer = false; state_t s; const char * personality; H5F_vfd_swmr_config_t config; diff --git a/test/vfd_swmr_dsetops_writer.c b/test/vfd_swmr_dsetops_writer.c index 5c7af89..7760dab 100644 --- a/test/vfd_swmr_dsetops_writer.c +++ b/test/vfd_swmr_dsetops_writer.c @@ -79,7 +79,7 @@ typedef struct { bool reg_ref; /* For -R option */ bool compact; /* -p option: create compact dataset */ bool compact_write; /* -t option: write to the whole compact dataset */ - unsigned int compact_elmts; /* -e option: # of elments for the compact dataset */ + unsigned int compact_elmts; /* -e option: # of elements for the compact dataset */ bool contig; /* -g option: create contiguous dataset */ bool chunked; /* -k option: create chunked datasets with 5 indexing types */ unsigned int rows; /* -m option for contiguous and/or chunked datasets */ diff --git a/test/vfd_swmr_generator.c b/test/vfd_swmr_generator.c index 2e87c18..a78abbc 100644 --- a/test/vfd_swmr_generator.c +++ b/test/vfd_swmr_generator.c @@ -238,7 +238,7 @@ gen_skeleton(const char *filename, hbool_t verbose, hbool_t vfd_swmr_write, int if (verbose) HDfprintf(stderr, "Closing objects\n"); - /* Close everythign */ + /* Close everything */ if (H5Pclose(dcpl) < 0) return -1; if (H5Sclose(sid) < 0) diff --git a/test/vfd_swmr_gfail_writer.c b/test/vfd_swmr_gfail_writer.c index ecda979..53b95d3 100644 --- a/test/vfd_swmr_gfail_writer.c +++ b/test/vfd_swmr_gfail_writer.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Purpose: To demostrate the four issues uncovered during the test plan + * Purpose: To demonstrate the four issues uncovered during the test plan * 1.4 part 1. * * Setup: @@ -19,7 +19,7 @@ * 1) Create an HDF5 file * 2) Create many groups * 3) Use the named pipe to send the reader a message to start verifying - * 4) Call H5Fvfd_swmr_end_tick immeidately after sending out the message to the reader + * 4) Call H5Fvfd_swmr_end_tick immediately after sending out the message to the reader * 5) Sleep for two ticks before receiving a message from the reader that informs the reader * started verifying * 6) If the option to delete the group is off(by default), just sleep for a relatively long time, @@ -163,7 +163,7 @@ usage(const char *progname) "-G: old-style type of group\n" "-n ngroups: the number of groups\n" "-N: do not use named pipes, \n" - " mainly for running the writer and reader seperately\n" + " mainly for running the writer and reader separately\n" "-t tick_len: length of a tick in tenths of a second.\n" "-m max_lag: maximum expected lag(in ticks) between writer and readers\n" "-B pbs: page buffer size in bytes:\n" diff --git a/test/vfd_swmr_gperf_writer.c b/test/vfd_swmr_gperf_writer.c index 1d20acb..dbceace 100644 --- a/test/vfd_swmr_gperf_writer.c +++ b/test/vfd_swmr_gperf_writer.c @@ -1117,7 +1117,7 @@ error: * Purpose: Add some attributes to the group. * First, the number of attributes should be the maximal number * of attributes that the compact storage can hold. - * Then, add another atribute, the storage becomes dense. + * Then, add another attribute, the storage becomes dense. * * Parameters: state_t *s * The struct that stores information of HDF5 file @@ -1253,7 +1253,7 @@ error: * Purpose: Add some attributes to the group and then delete one attribute. * First, the number of attributes to be added should be the * maximal number of attributes that the compact storage can hold. - * Then, delete one atribute, the storage is still compact. + * Then, delete one attribute, the storage is still compact. * * Parameters: state_t *s * The struct that stores information of HDF5 file @@ -1299,7 +1299,7 @@ add_del_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) * First, the number of attributes to be added exceeds * the maximal number of attributes that the compact storage can hold. * The storage changes from compact to dense. - * Then, delete one atribute, the storage is still dense. + * Then, delete one attribute, the storage is still dense. * * Parameters: state_t *s * The struct that stores information of HDF5 file @@ -1473,7 +1473,7 @@ del_ohr_block_attr(state_t *s, hid_t g, unsigned int which) * Function: add_group_attribute * * Purpose: Check the attribute test pattern and then call the - * correponding test function.. + * corresponding test function. * * Parameters: state_t *s * The struct that stores information of HDF5 file @@ -2642,7 +2642,7 @@ gen_tree_struct(state_t *s, unsigned int level, unsigned ne_per_level, hid_t pgr char name[sizeof("group-9999999999")]; unsigned int i; - hid_t grp_id; + hid_t grp_id = H5I_INVALID_HID; bool result = true; H5G_info_t group_info; struct timespec start_time, end_time; @@ -2763,7 +2763,7 @@ main(int argc, char **argv) H5F_vfd_swmr_config_t config; bool wg_ret = false; struct timespec start_time, end_time; - unsigned int num_elems_per_level; + unsigned int num_elems_per_level = 0; if (!state_init(&s, argc, argv)) { printf("state_init failed\n"); @@ -2814,7 +2814,7 @@ main(int argc, char **argv) if (s.nglevels > 0) { if (s.grp_op_pattern != ' ' || s.at_pattern != ' ') { printf("For nested group creation test, only the default option is supported.\n"); - printf("Please re-run the tests with the appopriate option.\n"); + printf("Please re-run the tests with the appropriate option.\n"); TEST_ERROR; } } @@ -2946,7 +2946,7 @@ main(int argc, char **argv) fprintf(stdout, " Not using VFD SWMR \n"); if (s.old_style_grp) - fprintf(stdout, " Groups: Created via the earlist file format(old-style) \n"); + fprintf(stdout, " Groups: Created via the earliest file format(old-style) \n"); else fprintf(stdout, " Groups: Created via the latest file format(new-style) \n"); diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 5952f83..00387ce 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -77,7 +77,7 @@ usage(const char *progname) "-c steps: `steps` between communication between the writer and reader\n" "-n ngroups: the number of groups\n" "-N: do not use named pipes, \n" - " mainly for running the writer and reader seperately\n" + " mainly for running the writer and reader separately\n" "-u numb_ticks: `numb_ticks` for the reader to wait before verification\n" "-t tick_len: length of a tick in tenths of a second.\n" "-m max_lag: maximum expected lag(in ticks) between writer and readers\n" @@ -1250,7 +1250,7 @@ error: * Purpose: Add some attributes to the group. * First, the number of attributes should be the maximal number * of attributes that the compact storage can hold. - * Then, add another atribute, the storage becomes dense. + * Then, add another attribute, the storage becomes dense. * * Parameters: state_t *s * The struct that stores information of HDF5 file, named pipe @@ -1430,7 +1430,7 @@ error2: * Purpose: Add some attributes to the group and then delete one attribute. * First, the number of attributes to be added should be the * maximal number of attributes that the compact storage can hold. - * Then, delete one atribute, the storage is still compact. + * Then, delete one attribute, the storage is still compact. * * Parameters: state_t *s * The struct that stores information of HDF5 file, named pipe @@ -1476,7 +1476,7 @@ add_del_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) * First, the number of attributes to be added exceeds * the maximal number of attributes that the compact storage can hold. * The storage changes from compact to dense. - * Then, delete one atribute, the storage is still dense. + * Then, delete one attribute, the storage is still dense. * * Parameters: state_t *s * The struct that stores information of HDF5 file, named pipe @@ -1652,7 +1652,7 @@ del_ohr_block_attr(state_t *s, hid_t g, unsigned int which) * Function: add_group_attribute * * Purpose: Check the attribute test pattern and then call the - * correponding test function.. + * corresponding test function.. * * Parameters: state_t *s * The struct that stores information of HDF5 file, named pipe @@ -2020,8 +2020,8 @@ vrfy_attr(state_t *s, hid_t g, unsigned int which, const char *aname, unsigned i } if (read_which != which) { - HDprintf("reader: the add_attribute verfication failed,expected value is %d\n", which); - HDprintf("reader: the add_attribute verfication failed, the value is %d\n", read_which); + HDprintf("reader: the add_attribute verification failed,expected value is %d\n", which); + HDprintf("reader: the add_attribute verification failed, the value is %d\n", read_which); HDprintf("The add_attribute verification failed\n"); TEST_ERROR; } @@ -2320,8 +2320,8 @@ verify_group_vlstr_attr(state_t *s, hid_t g, unsigned int which, bool vrfy_mod) dbgf(1, "read attr is= %s\n", astr_val); if (HDstrcmp(astr_val, astr_val_exp) != 0) { - HDprintf("reader: the vl add_attribute verfication failed,expected value is %s\n", astr_val_exp); - HDprintf("reader: the vl add_attribute verfication failed, the value is %s\n", astr_val); + HDprintf("reader: the vl add_attribute verification failed,expected value is %s\n", astr_val_exp); + HDprintf("reader: the vl add_attribute verification failed, the value is %s\n", astr_val); HDprintf("The vl add_attribute verification failed\n"); TEST_ERROR; } @@ -2883,7 +2883,7 @@ verify_del_ohr_block_attr(state_t *s, hid_t g, unsigned int which) * Function: verify_group_attribute * * Purpose: Check the attribute test pattern and then call the - * correponding verification function. + * corresponding verification function. * * Parameters: state_t *s * The struct that stores information of HDF5 file, named pipe @@ -5147,7 +5147,7 @@ main(int argc, char **argv) } else { - /* Send back the same nofity value for acknowledgement to tell the writer + /* Send back the same notify value for acknowledgement to tell the writer * move to the next step. */ if (s.use_named_pipes && s.attr_test != true && s.grp_op_test != true && step % s.csteps == 0) { diff --git a/test/vfd_swmr_indep_rw_writer.c b/test/vfd_swmr_indep_rw_writer.c index 5d1be50..c3572b6 100644 --- a/test/vfd_swmr_indep_rw_writer.c +++ b/test/vfd_swmr_indep_rw_writer.c @@ -492,7 +492,7 @@ open_dset(state_t *s) s->r_dsetid = dset_id; if (i == NUM_ATTEMPTS) { - HDfprintf(stderr, "dataset opening reachs the maximal number of attempts\n"); + HDfprintf(stderr, "dataset opening reaches the maximal number of attempts\n"); TEST_ERROR; } diff --git a/test/vfd_swmr_zoo_writer.c b/test/vfd_swmr_zoo_writer.c index 78ca06a..3efab61 100644 --- a/test/vfd_swmr_zoo_writer.c +++ b/test/vfd_swmr_zoo_writer.c @@ -145,7 +145,7 @@ parse_command_line_options(int argc, char **argv) } break; case 'N': - /* Disable named pipes, mainly for running the writer and reader seperately */ + /* Disable named pipes, mainly for running the writer and reader separately */ use_named_pipe = false; break; case 'q': diff --git a/test/vol.c b/test/vol.c index c7586d5..aba9311 100644 --- a/test/vol.c +++ b/test/vol.c @@ -814,9 +814,11 @@ test_basic_file_operation(const char *env_h5_drvr) if ((obj_count = H5Fget_obj_ids((hid_t)H5F_OBJ_ALL, H5F_OBJ_DATASET, 2, obj_id_list)) < 0) TEST_ERROR; - /* Can't compare VFD properties for split / multi / family VFDs */ + /* Can't compare VFD properties for several VFDs */ if ((hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && - HDstrcmp(env_h5_drvr, "family") != 0)) { + HDstrcmp(env_h5_drvr, "family") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0 && + HDstrcmp(env_h5_drvr, "core") != 0 && HDstrcmp(env_h5_drvr, "core_paged") != 0 && + HDstrcmp(env_h5_drvr, "mpio") != 0 && HDstrcmp(env_h5_drvr, "splitter") != 0)) { /* H5Fget_access_plist */ if ((fapl_id2 = H5Fget_access_plist(fid)) < 0) TEST_ERROR; @@ -876,9 +878,11 @@ test_basic_file_operation(const char *env_h5_drvr) if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) TEST_ERROR; - /* Can't compare VFD properties for split / multi / family VFDs */ + /* Can't compare VFD properties for several VFDs */ if ((hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && - HDstrcmp(env_h5_drvr, "family") != 0)) { + HDstrcmp(env_h5_drvr, "family") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0 && + HDstrcmp(env_h5_drvr, "core") != 0 && HDstrcmp(env_h5_drvr, "core_paged") != 0 && + HDstrcmp(env_h5_drvr, "mpio") != 0 && HDstrcmp(env_h5_drvr, "splitter") != 0)) { /* H5Fget_access_plist */ if ((fapl_id2 = H5Fget_access_plist(fid)) < 0) TEST_ERROR; @@ -891,9 +895,11 @@ test_basic_file_operation(const char *env_h5_drvr) if ((fid_reopen = H5Freopen(fid)) < 0) TEST_ERROR; - /* Can't compare VFD properties for split / multi / family VFDs */ + /* Can't compare VFD properties for several VFDs */ if ((hbool_t)(HDstrcmp(env_h5_drvr, "split") != 0 && HDstrcmp(env_h5_drvr, "multi") != 0 && - HDstrcmp(env_h5_drvr, "family") != 0)) { + HDstrcmp(env_h5_drvr, "family") != 0 && HDstrcmp(env_h5_drvr, "direct") != 0 && + HDstrcmp(env_h5_drvr, "core") != 0 && HDstrcmp(env_h5_drvr, "core_paged") != 0 && + HDstrcmp(env_h5_drvr, "mpio") != 0 && HDstrcmp(env_h5_drvr, "splitter") != 0)) { /* H5Fget_access_plist */ if ((fapl_id2 = H5Fget_access_plist(fid_reopen)) < 0) TEST_ERROR; @@ -942,7 +948,7 @@ error: *------------------------------------------------------------------------- */ static herr_t -test_basic_group_operation(void) +test_basic_group_operation(const char *env_h5_drvr) { hid_t fid = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; @@ -985,9 +991,10 @@ test_basic_group_operation(void) if (H5Gget_info_by_idx(fid, "/", H5_INDEX_NAME, H5_ITER_NATIVE, 0, &info, H5P_DEFAULT) < 0) TEST_ERROR; - /* H5Gflush */ - if (H5Gflush(gid) < 0) - TEST_ERROR; + /* H5Gflush - skip for MPIO file driver as flush calls cause assertions in the library */ + if (HDstrcmp(env_h5_drvr, "mpio") != 0) + if (H5Gflush(gid) < 0) + TEST_ERROR; /* H5Gclose */ if (H5Gclose(gid) < 0) @@ -1045,7 +1052,7 @@ error: *------------------------------------------------------------------------- */ static herr_t -test_basic_dataset_operation(void) +test_basic_dataset_operation(const char *env_h5_drvr) { hid_t fid = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; @@ -1110,9 +1117,10 @@ test_basic_dataset_operation(void) if (H5Dset_extent(did, &curr_dims) < 0) TEST_ERROR; - /* H5Dflush */ - if (H5Dflush(did) < 0) - TEST_ERROR; + /* H5Dflush - skip for MPIO file driver as flush calls cause assertions in the library */ + if (HDstrcmp(env_h5_drvr, "mpio") != 0) + if (H5Dflush(did) < 0) + TEST_ERROR; /* H5Dwrite */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, in_buf) < 0) @@ -1384,10 +1392,10 @@ test_basic_object_operation(void) if (H5Oclose(oid) < 0) TEST_ERROR; - if (H5Fclose(fid) < 0) - TEST_ERROR; if (H5Gclose(gid) < 0) TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; h5_delete_test_file(FILENAME[0], fapl_id); @@ -1463,10 +1471,10 @@ test_basic_link_operation(void) H5P_DEFAULT) < 0) TEST_ERROR; - if (H5Fclose(fid) < 0) - TEST_ERROR; if (H5Gclose(gid) < 0) TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; h5_delete_test_file(FILENAME[0], fapl_id); @@ -1500,7 +1508,7 @@ error: *------------------------------------------------------------------------- */ static herr_t -test_basic_datatype_operation(void) +test_basic_datatype_operation(const char *env_h5_drvr) { hid_t fid = H5I_INVALID_HID; hid_t fapl_id = H5I_INVALID_HID; @@ -1524,9 +1532,10 @@ test_basic_datatype_operation(void) if (H5Tcommit2(fid, NATIVE_VOL_TEST_DATATYPE_NAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR; - /* H5Tflush */ - if (H5Tflush(tid) < 0) - TEST_ERROR; + /* H5Tflush - skip for MPIO file driver as flush calls cause assertions in the library */ + if (HDstrcmp(env_h5_drvr, "mpio") != 0) + if (H5Tflush(tid) < 0) + TEST_ERROR; /* H5Trefresh */ if (H5Trefresh(tid) < 0) @@ -1975,11 +1984,11 @@ test_async_vol_props(void) FAIL_STACK_ERROR; /* Override possible environment variable & re-initialize default VOL connector */ - conn_env_str = HDgetenv("HDF5_VOL_CONNECTOR"); + conn_env_str = HDgetenv(HDF5_VOL_CONNECTOR); if (conn_env_str) { if (NULL == (conn_env_str = HDstrdup(conn_env_str))) TEST_ERROR - if (HDunsetenv("HDF5_VOL_CONNECTOR") < 0) + if (HDunsetenv(HDF5_VOL_CONNECTOR) < 0) TEST_ERROR if (H5VL__reparse_def_vol_conn_variable_test() < 0) TEST_ERROR @@ -2004,7 +2013,7 @@ test_async_vol_props(void) FAIL_STACK_ERROR; /* Set environment variable to use 'fake async' connector & re-init default connector */ - if (HDsetenv("HDF5_VOL_CONNECTOR", "fake_async", TRUE) < 0) + if (HDsetenv(HDF5_VOL_CONNECTOR, "fake_async", TRUE) < 0) TEST_ERROR if (H5VL__reparse_def_vol_conn_variable_test() < 0) TEST_ERROR @@ -2022,7 +2031,7 @@ test_async_vol_props(void) TEST_ERROR /* Reset environment variable & re-init default connector */ - if (HDunsetenv("HDF5_VOL_CONNECTOR") < 0) + if (HDunsetenv(HDF5_VOL_CONNECTOR) < 0) TEST_ERROR if (H5VL__reparse_def_vol_conn_variable_test() < 0) TEST_ERROR @@ -2072,7 +2081,7 @@ test_async_vol_props(void) /* Restore environment variable, if there was one */ if (conn_env_str) { - if (HDsetenv("HDF5_VOL_CONNECTOR", conn_env_str, TRUE) < 0) + if (HDsetenv(HDF5_VOL_CONNECTOR, conn_env_str, TRUE) < 0) TEST_ERROR HDfree(conn_env_str); @@ -2112,7 +2121,7 @@ main(void) int nerrors = 0; /* Get the VFD to use */ - env_h5_drvr = HDgetenv("HDF5_DRIVER"); + env_h5_drvr = HDgetenv(HDF5_DRIVER); if (env_h5_drvr == NULL) env_h5_drvr = "nomatch"; @@ -2124,12 +2133,12 @@ main(void) nerrors += test_register_opt_operation() < 0 ? 1 : 0; nerrors += test_native_vol_init() < 0 ? 1 : 0; nerrors += test_basic_file_operation(env_h5_drvr) < 0 ? 1 : 0; - nerrors += test_basic_group_operation() < 0 ? 1 : 0; - nerrors += test_basic_dataset_operation() < 0 ? 1 : 0; + nerrors += test_basic_group_operation(env_h5_drvr) < 0 ? 1 : 0; + nerrors += test_basic_dataset_operation(env_h5_drvr) < 0 ? 1 : 0; nerrors += test_basic_attribute_operation() < 0 ? 1 : 0; nerrors += test_basic_object_operation() < 0 ? 1 : 0; nerrors += test_basic_link_operation() < 0 ? 1 : 0; - nerrors += test_basic_datatype_operation() < 0 ? 1 : 0; + nerrors += test_basic_datatype_operation(env_h5_drvr) < 0 ? 1 : 0; nerrors += test_async_vol_props() < 0 ? 1 : 0; if (nerrors) { diff --git a/test/vol_plugin.c b/test/vol_plugin.c index 7656870..c7baf5a 100644 --- a/test/vol_plugin.c +++ b/test/vol_plugin.c @@ -150,8 +150,8 @@ error: static herr_t test_multiple_registration(void) { - htri_t is_registered = FAIL; - hid_t vol_ids[N_REGISTRATIONS]; + htri_t is_registered = FAIL; + hid_t vol_ids[N_REGISTRATIONS] = {0}; int i; TESTING("registering a VOL connector multiple times"); diff --git a/testpar/CMakeVFDTests.cmake b/testpar/CMakeVFDTests.cmake index f50ca16..d6a4025 100644 --- a/testpar/CMakeVFDTests.cmake +++ b/testpar/CMakeVFDTests.cmake @@ -15,29 +15,13 @@ ### T E S T I N G ### ############################################################################## ############################################################################## -set (VFD_LIST - sec2 - stdio - core - core_paged - split - multi - family -) +H5_CREATE_VFD_DIR() set (H5P_VFD_TESTS t_pflush1 t_pflush2 ) -if (H5_HAVE_DIRECT) - set (VFD_LIST ${VFD_LIST} direct) -endif () - -foreach (vfdtest ${VFD_LIST}) - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdtest}") -endforeach () - macro (ADD_VFD_TEST vfdname resultcode) if (NOT HDF5_ENABLE_USING_MEMCHECKER) foreach (h5_test ${H5P_VFD_TESTS}) diff --git a/testpar/t_2Gio.c b/testpar/t_2Gio.c index d5b9038..911be2c 100644 --- a/testpar/t_2Gio.c +++ b/testpar/t_2Gio.c @@ -33,7 +33,7 @@ #include "mpi.h" /* For this test, we don't want to inherit the RANK definition - * from testphdf5.h. We'll define MAX_RANK to accomodate 3D arrays + * from testphdf5.h. We'll define MAX_RANK to accommodate 3D arrays * and use that definition rather than RANK. */ #ifndef MAX_RANK @@ -3047,7 +3047,7 @@ compress_readAll(void) nerrors++; } -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES ret = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, xfer_plist, data_read); VRFY((ret >= 0), "H5Dwrite succeeded"); #endif @@ -3323,7 +3323,7 @@ none_selection_chunk(void) * Simple independent I/O. This tests that the defaults are properly set. * * TEST_ACTUAL_IO_RESET: - * Perfroms collective and then independent I/O wit hthe same dxpl to + * Performs collective and then independent I/O with hthe same dxpl to * make sure the peroperty is correctly reset to the default on each use. * Specifically, this test runs TEST_ACTUAL_IO_MULTI_CHUNK_NO_OPT_MIX_DISAGREE * (The most complex case that works on all builds) and then performs @@ -3332,7 +3332,7 @@ none_selection_chunk(void) * Note: DIRECT_MULTI_CHUNK_MIX and DIRECT_MULTI_CHUNK_MIX_DISAGREE * is not needed as they are covered by DIRECT_CHUNK_MIX and * MULTI_CHUNK_MIX_DISAGREE cases. _DIRECT_ cases are only for testing - * path way to multi-chunk-io by H5FD_MPIO_CHUNK_MULTI_IO insted of num-threshold. + * path way to multi-chunk-io by H5FD_MPIO_CHUNK_MULTI_IO instead of num-threshold. * * Modification: * - Refctore to remove multi-chunk-without-opimization test and update for @@ -3347,12 +3347,12 @@ none_selection_chunk(void) static void test_actual_io_mode(int selection_mode) { - H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_write = -1; - H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_read = -1; - H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_expected = -1; - H5D_mpio_actual_io_mode_t actual_io_mode_write = -1; - H5D_mpio_actual_io_mode_t actual_io_mode_read = -1; - H5D_mpio_actual_io_mode_t actual_io_mode_expected = -1; + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_write = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_read = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_expected = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + H5D_mpio_actual_io_mode_t actual_io_mode_write = H5D_MPIO_NO_COLLECTIVE; + H5D_mpio_actual_io_mode_t actual_io_mode_read = H5D_MPIO_NO_COLLECTIVE; + H5D_mpio_actual_io_mode_t actual_io_mode_expected = H5D_MPIO_NO_COLLECTIVE; const char * filename; const char * test_name; hbool_t direct_multi_chunk_io; @@ -3520,7 +3520,7 @@ test_actual_io_mode(int selection_mode) break; /* RESET tests that the properties are properly reset to defaults each time I/O is - * performed. To acheive this, we have RESET perform collective I/O (which would change + * performed. To achieve this, we have RESET perform collective I/O (which would change * the values from the defaults) followed by independent I/O (which should report the * default values). RESET doesn't need to have a unique selection, so we reuse * MULTI_CHUMK_MIX_DISAGREE, which was chosen because it is a complex case that works @@ -3536,7 +3536,7 @@ test_actual_io_mode(int selection_mode) * assgigned collective I/O, while each other process gets independent I/O. * Since the root process with only access the first chunk, it will report * collective I/O. The subsequent processes will access the first chunk - * collectively, and their other chunk indpendently, reporting mixed I/O. + * collectively, and their other chunk independently, reporting mixed I/O. */ if (mpi_rank == 0) { @@ -3605,8 +3605,8 @@ test_actual_io_mode(int selection_mode) default: test_name = "Undefined Selection Mode"; - actual_chunk_opt_mode_expected = -1; - actual_io_mode_expected = -1; + actual_chunk_opt_mode_expected = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + actual_io_mode_expected = H5D_MPIO_NO_COLLECTIVE; break; } @@ -3651,7 +3651,7 @@ test_actual_io_mode(int selection_mode) ret = H5Pset_dxpl_mpio_chunk_opt_num(dxpl_write, (unsigned)mpi_size * 2); VRFY((ret >= 0), "H5Pset_dxpl_mpio_chunk_opt_num succeeded"); - /* set this to manipulate testing senario about allocating processes + /* set this to manipulate testing scenario about allocating processes * to chunks */ ret = H5Pset_dxpl_mpio_chunk_opt_ratio(dxpl_write, (unsigned)99); VRFY((ret >= 0), "H5Pset_dxpl_mpio_chunk_opt_ratio succeeded"); @@ -3675,12 +3675,12 @@ test_actual_io_mode(int selection_mode) H5Eprint2(H5E_DEFAULT, stdout); VRFY((ret >= 0), "H5Dwrite() dataset multichunk write succeeded"); - /* Retreive Actual io valuess */ + /* Retrieve Actual io values */ ret = H5Pget_mpio_actual_io_mode(dxpl_write, &actual_io_mode_write); - VRFY((ret >= 0), "retriving actual io mode suceeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_write, &actual_chunk_opt_mode_write); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); /* Read */ ret = H5Dread(dataset, data_type, mem_space, file_space, dxpl_read, buffer); @@ -3688,12 +3688,12 @@ test_actual_io_mode(int selection_mode) H5Eprint2(H5E_DEFAULT, stdout); VRFY((ret >= 0), "H5Dread() dataset multichunk read succeeded"); - /* Retreive Actual io values */ + /* Retrieve Actual io values */ ret = H5Pget_mpio_actual_io_mode(dxpl_read, &actual_io_mode_read); - VRFY((ret >= 0), "retriving actual io mode succeeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_read, &actual_chunk_opt_mode_read); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); /* Check write vs read */ VRFY((actual_io_mode_read == actual_io_mode_write), @@ -3714,7 +3714,7 @@ test_actual_io_mode(int selection_mode) actual_io_mode_write); } - /* To test that the property is succesfully reset to the default, we perform some + /* To test that the property is successfully reset to the default, we perform some * independent I/O after the collective I/O */ if (selection_mode == TEST_ACTUAL_IO_RESET) { @@ -3731,9 +3731,9 @@ test_actual_io_mode(int selection_mode) /* Check Properties */ ret = H5Pget_mpio_actual_io_mode(dxpl_write, &actual_io_mode_write); - VRFY((ret >= 0), "retriving actual io mode succeeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_write, &actual_chunk_opt_mode_write); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); VRFY(actual_chunk_opt_mode_write == H5D_MPIO_NO_CHUNK_OPTIMIZATION, "actual_chunk_opt_mode has correct value for reset write (independent)"); @@ -3746,9 +3746,9 @@ test_actual_io_mode(int selection_mode) /* Check Properties */ ret = H5Pget_mpio_actual_io_mode(dxpl_read, &actual_io_mode_read); - VRFY((ret >= 0), "retriving actual io mode succeeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_read, &actual_chunk_opt_mode_read); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); VRFY(actual_chunk_opt_mode_read == H5D_MPIO_NO_CHUNK_OPTIMIZATION, "actual_chunk_opt_mode has correct value for reset read (independent)"); @@ -3794,11 +3794,11 @@ actual_io_mode_tests(void) test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_IND); test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_COL); - /* The Multi Chunk Mixed test requires atleast three processes. */ + /* The Multi Chunk Mixed test requires at least three processes. */ if (mpi_size > 2) test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_MIX); else - HDfprintf(stdout, "Multi Chunk Mixed test requires 3 proceses minimum\n"); + HDfprintf(stdout, "Multi Chunk Mixed test requires 3 processes minimum\n"); test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_MIX_DISAGREE); @@ -3842,7 +3842,7 @@ actual_io_mode_tests(void) * Test for Data Type Conversion as the cause of breaking collective I/O. * * TEST_DATA_TRANSFORMS: - * Test for Data Transfrom feature as the cause of breaking collective I/O. + * Test for Data Transform feature as the cause of breaking collective I/O. * * TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES: * Test for NULL dataspace as the cause of breaking collective I/O. @@ -3853,12 +3853,6 @@ actual_io_mode_tests(void) * TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL: * Test for Externl-File storage as the cause of breaking collective I/O. * - * TEST_FILTERS: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * Note: TEST_FILTERS mode will not work until H5Dcreate and H5write is supported for mpio and filter - * feature. Use test_no_collective_cause_mode_filter() function instead. - * - * * Programmer: Jonathan Kim * Date: Aug, 2012 */ @@ -3898,9 +3892,6 @@ test_no_collective_cause_mode(int selection_mode) hid_t file_space = -1; hsize_t chunk_dims[MAX_RANK]; herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ /* set to global value as default */ int l_facc_type = facc_type; char message[256]; @@ -3932,21 +3923,6 @@ test_no_collective_cause_mode(int selection_mode) is_chunked = 0; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY((ret >= 0), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info(H5Z_FILTER_FLETCHER32, &filter_info); - VRFY(((filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || - (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED)), - "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0), "set filter (flecher32) succeeded"); - } -#endif /* LATER */ - if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { sid = H5Screate(H5S_NULL); VRFY((sid >= 0), "H5Screate_simple succeeded"); @@ -4004,7 +3980,7 @@ test_no_collective_cause_mode(int selection_mode) } if (selection_mode & TEST_DATA_TRANSFORMS) { - test_name = "Broken Collective I/O - DATA Transfroms"; + test_name = "Broken Collective I/O - DATA Transforms"; no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS; no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS; } @@ -4022,14 +3998,6 @@ test_no_collective_cause_mode(int selection_mode) no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected |= H5D_MPIO_FILTERS; - no_collective_cause_global_expected |= H5D_MPIO_FILTERS; - } -#endif /* LATER */ - if (selection_mode & TEST_COLLECTIVE) { test_name = "Broken Collective I/O - Not Broken"; no_collective_cause_local_expected = H5D_MPIO_COLLECTIVE; @@ -4102,7 +4070,7 @@ test_no_collective_cause_mode(int selection_mode) /* Get the cause of broken collective I/O */ ret = H5Pget_mpio_no_collective_cause(dxpl_write, &no_collective_cause_local_write, &no_collective_cause_global_write); - VRFY((ret >= 0), "retriving no collective cause succeeded"); + VRFY((ret >= 0), "retrieving no collective cause succeeded"); /*--------------------- * Test Read access @@ -4122,7 +4090,7 @@ test_no_collective_cause_mode(int selection_mode) /* Get the cause of broken collective I/O */ ret = H5Pget_mpio_no_collective_cause(dxpl_read, &no_collective_cause_local_read, &no_collective_cause_global_read); - VRFY((ret >= 0), "retriving no collective cause succeeded"); + VRFY((ret >= 0), "retrieving no collective cause succeeded"); /* Check write vs read */ VRFY((no_collective_cause_local_read == no_collective_cause_local_write), @@ -4166,240 +4134,6 @@ test_no_collective_cause_mode(int selection_mode) return; } -#if 0 -/* - * Function: test_no_collective_cause_mode_filter - * - * Purpose: - * Test specific for using filter as a caus of broken collective I/O and - * checks that the H5Pget_mpio_no_collective_cause properties in the DXPL - * have the correct values. - * - * NOTE: - * This is a temporary function. - * test_no_collective_cause_mode(TEST_FILTERS) will replace this when - * H5Dcreate and H5write support for mpio and filter feature. - * - * Input: - * TEST_FILTERS_READ: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * - * Programmer: Jonathan Kim - * Date: Aug, 2012 - */ -static void -test_no_collective_cause_mode_filter(int selection_mode) -{ - uint32_t no_collective_cause_local_read = 0; - uint32_t no_collective_cause_local_expected = 0; - uint32_t no_collective_cause_global_read = 0; - uint32_t no_collective_cause_global_expected = 0; - - const char * filename; - const char * test_name; - hbool_t is_chunked=1; - int mpi_size = -1; - int mpi_rank = -1; - int length; - int * buffer; - int i; - MPI_Comm mpi_comm = MPI_COMM_NULL; - MPI_Info mpi_info = MPI_INFO_NULL; - hid_t fid = -1; - hid_t sid = -1; - hid_t dataset = -1; - hid_t data_type = H5T_NATIVE_INT; - hid_t fapl_write = -1; - hid_t fapl_read = -1; - hid_t dcpl = -1; - hid_t dxpl = -1; - hsize_t dims[MAX_RANK]; - hid_t mem_space = -1; - hid_t file_space = -1; - hsize_t chunk_dims[MAX_RANK]; - herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ - char message[256]; - - /* Set up MPI parameters */ - MPI_Comm_size(test_comm, &mpi_size); - MPI_Comm_rank(test_comm, &mpi_rank); - - MPI_Barrier(test_comm); - - HDassert(mpi_size >= 1); - - mpi_comm = test_comm; - mpi_info = MPI_INFO_NULL; - - /* Create the dataset creation plist */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - VRFY((dcpl >= 0), "dataset creation plist created successfully"); - - if (selection_mode == TEST_FILTERS_READ ) { -#ifdef LATER /* fletcher32 */ - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY ((ret >=0 ), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info (H5Z_FILTER_FLETCHER32, (unsigned int *) &filter_info); - VRFY ( ( (filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED) ) , "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0),"set filter (flecher32) succeeded"); -#endif /* LATER */ - } - else { - VRFY(0, "Unexpected mode, only test for TEST_FILTERS_READ."); - } - - /* Create the basic Space */ - dims[0] = dim0; - dims[1] = dim1; - sid = H5Screate_simple (MAX_RANK, dims, NULL); - VRFY((sid >= 0), "H5Screate_simple succeeded"); - - - filename = (const char *)GetTestParameters(); - HDassert(filename != NULL); - - /* Setup the file access template */ - fapl_write = create_faccess_plist(mpi_comm, mpi_info, FACC_DEFAULT); - VRFY((fapl_write >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_write); - VRFY((fid >= 0), "H5Fcreate succeeded"); - - /* If we are not testing contiguous datasets */ - if(is_chunked) { - /* Set up chunk information. */ - chunk_dims[0] = dims[0]/mpi_size; - chunk_dims[1] = dims[1]; - ret = H5Pset_chunk(dcpl, 2, chunk_dims); - VRFY((ret >= 0),"chunk creation property list succeeded"); - } - - - /* Create the dataset */ - dataset = H5Dcreate2(fid, DSET_NOCOLCAUSE, data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); - -#ifdef LATER /* fletcher32 */ - /* Set expected cause */ - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected = H5D_MPIO_FILTERS; - no_collective_cause_global_expected = H5D_MPIO_FILTERS; -#endif /* LATER */ - - /* Get the file dataspace */ - file_space = H5Dget_space(dataset); - VRFY((file_space >= 0), "H5Dget_space succeeded"); - - /* Create the memory dataspace */ - mem_space = H5Screate_simple (MAX_RANK, dims, NULL); - VRFY((mem_space >= 0), "mem_space created"); - - /* Get the number of elements in the selection */ - length = dim0 * dim1; - - /* Allocate and initialize the buffer */ - buffer = (int *)HDmalloc(sizeof(int) * length); - VRFY((buffer != NULL), "HDmalloc of buffer succeeded"); - for(i = 0; i < length; i++) - buffer[i] = i; - - /* Set up the dxpl for the write */ - dxpl = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - - if (selection_mode == TEST_FILTERS_READ) { - /* To test read in collective I/O mode , write in independent mode - * because write fails with mpio + filter */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - else { - /* To test write in collective I/O mode. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - - - /* Write */ - ret = H5Dwrite(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if(ret < 0) H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dwrite() dataset multichunk write succeeded"); - - - /* Make a copy of the dxpl to test the read operation */ - dxpl = H5Pcopy(dxpl); - VRFY((dxpl >= 0), "H5Pcopy succeeded"); - - if (dataset) - H5Dclose(dataset); - if (fapl_write) - H5Pclose(fapl_write); - if (fid) - H5Fclose(fid); - - - /*--------------------- - * Test Read access - *---------------------*/ - - /* Setup the file access template */ - fapl_read = create_faccess_plist(mpi_comm, mpi_info, facc_type); - VRFY((fapl_read >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fopen (filename, H5F_ACC_RDONLY, fapl_read); - dataset = H5Dopen2 (fid, DSET_NOCOLCAUSE, H5P_DEFAULT); - - /* Set collective I/O properties in the dxpl. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - - /* Read */ - ret = H5Dread(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if(ret < 0) H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dread() dataset multichunk read succeeded"); - - /* Get the cause of broken collective I/O */ - ret = H5Pget_mpio_no_collective_cause (dxpl, &no_collective_cause_local_read, &no_collective_cause_global_read); - VRFY((ret >= 0), "retriving no collective cause succeeded" ); - - /* Test values */ - HDmemset (message, 0, sizeof (message)); - HDsprintf(message, "Local cause of Broken Collective I/O has the correct value for %s.\n",test_name); - VRFY((no_collective_cause_local_read == (uint32_t)no_collective_cause_local_expected), message); - HDmemset (message, 0, sizeof (message)); - HDsprintf(message, "Global cause of Broken Collective I/O has the correct value for %s.\n",test_name); - VRFY((no_collective_cause_global_read == (uint32_t)no_collective_cause_global_expected), message); - - /* Release some resources */ - if (sid) - H5Sclose(sid); - if (fapl_read) - H5Pclose(fapl_read); - if (dcpl) - H5Pclose(dcpl); - if (dxpl) - H5Pclose(dxpl); - if (dataset) - H5Dclose(dataset); - if (mem_space) - H5Sclose(mem_space); - if (file_space) - H5Sclose(file_space); - if (fid) - H5Fclose(fid); - HDfree(buffer); - return; -} -#endif - /* Function: no_collective_cause_tests * * Purpose: Tests cases for broken collective IO. @@ -4420,13 +4154,6 @@ no_collective_cause_tests(void) test_no_collective_cause_mode(TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL); -#ifdef LATER /* fletcher32 */ - /* TODO: use this instead of below TEST_FILTERS_READ when H5Dcreate and - * H5Dwrite is ready for mpio + filter feature. - */ - /* test_no_collective_cause_mode (TEST_FILTERS); */ - test_no_collective_cause_mode_filter(TEST_FILTERS_READ); -#endif /* LATER */ /* * Test combined causes @@ -4547,7 +4274,7 @@ dataset_atomicity(void) /* file locking allows only one file open (serial) for writing */ if (MAINPROCESS) { fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); - VRFY((fid >= 0), "H5Fopen succeeed"); + VRFY((fid >= 0), "H5Fopen succeeded"); } /* should fail */ diff --git a/testpar/t_bigio.c b/testpar/t_bigio.c index e5654f4..0a971c5 100644 --- a/testpar/t_bigio.c +++ b/testpar/t_bigio.c @@ -1107,13 +1107,15 @@ single_rank_independent_io(void) HDprintf("\nSingle Rank Independent I/O\n"); if (MAIN_PROCESS) { - hsize_t dims[] = {LARGE_DIM}; - hid_t file_id = -1; - hid_t fapl_id = -1; - hid_t dset_id = -1; - hid_t fspace_id = -1; - hid_t mspace_id = -1; - void * data = NULL; + hsize_t dims[] = {LARGE_DIM}; + hid_t file_id = -1; + hid_t fapl_id = -1; + hid_t dset_id = -1; + hid_t fspace_id = -1; + hid_t mspace_id = -1; + herr_t ret; + int * data = NULL; + uint64_t i; fapl_id = H5Pcreate(H5P_FILE_ACCESS); VRFY_G((fapl_id >= 0), "H5P_FILE_ACCESS"); @@ -1135,6 +1137,10 @@ single_rank_independent_io(void) data = malloc(LARGE_DIM * sizeof(int)); + /* Initialize data */ + for (i = 0; i < LARGE_DIM; i++) + data[i] = (int)(i % (uint64_t)DXFER_BIGCOUNT); + if (mpi_rank_g == 0) H5Sselect_all(fspace_id); else @@ -1143,7 +1149,24 @@ single_rank_independent_io(void) dims[0] = LARGE_DIM; mspace_id = H5Screate_simple(1, dims, NULL); VRFY_G((mspace_id >= 0), "H5Screate_simple mspace_id succeeded"); + + /* Write data */ H5Dwrite(dset_id, H5T_NATIVE_INT, mspace_id, fspace_id, H5P_DEFAULT, data); + VRFY_G((ret >= 0), "H5Dwrite succeeded"); + + /* Wipe buffer */ + HDmemset(data, 0, LARGE_DIM * sizeof(int)); + + /* Read data back */ + H5Dread(dset_id, H5T_NATIVE_INT, mspace_id, fspace_id, H5P_DEFAULT, data); + VRFY_G((ret >= 0), "H5Dread succeeded"); + + /* Verify data */ + for (i = 0; i < LARGE_DIM; i++) + if (data[i] != (int)(i % (uint64_t)DXFER_BIGCOUNT)) { + HDfprintf(stderr, "verify failed\n"); + exit(1); + } free(data); H5Sclose(mspace_id); @@ -1331,7 +1354,7 @@ coll_chunk2(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = space_dim1*mpi_size @@ -1840,7 +1863,7 @@ main(int argc, char **argv) /* Set the bigio processing limit to be 'newsize' bytes */ hsize_t oldsize = H5_mpi_set_bigio_count(newsize); - /* Having set the bigio handling to a size that is managable, + /* Having set the bigio handling to a size that is manageable, * we'll set our 'bigcount' variable to be 2X that limit so * that we try to ensure that our bigio handling is actually * envoked and tested. diff --git a/testpar/t_cache.c b/testpar/t_cache.c index 8696092..70ada01 100644 --- a/testpar/t_cache.c +++ b/testpar/t_cache.c @@ -126,7 +126,7 @@ int total_writes = 0; * * local_pinned: Boolean flag that is set to true iff the entry * has been pinned in the local cache, but probably not all - * caches. Such pins will typically not be consistant across + * caches. Such pins will typically not be consistent across * processes, and thus cannot be marked as dirty unless they * happen to overlap some collective operation. * @@ -205,7 +205,7 @@ struct datum data[NUM_DATA_ENTRIES]; * even divisor of NUM_DATA_ENTRIES. So far, all tests have been with * powers of 10 that meet these criteria. * - * Further, this value must be consistant across all processes. + * Further, this value must be consistent across all processes. */ #define STD_VIRT_NUM_DATA_ENTRIES NUM_DATA_ENTRIES @@ -252,7 +252,7 @@ int data_index[NUM_DATA_ENTRIES]; * * Strangely, at least on Phoenix, the first solution runs faster by a * rather large margin. However, I can imagine this changing with - * different OS's and MPI implementatins. + * different OS's and MPI implementations. * * Thus I have left code supporting the second solution in place. * @@ -1282,7 +1282,7 @@ reset_server_counters(void) * Function: server_main() * * Purpose: Main function for the server process. This process exists - * to provide an independant view of the data array. + * to provide an independent view of the data array. * * The function handles request from the other processes in * the test until the count of done messages received equals @@ -3962,7 +3962,7 @@ setup_cache_for_test(hid_t *fid_ptr, H5F_t **file_ptr_ptr, H5C_t **cache_ptr_ptr * Purpose: Verify that the indicated entries have been written exactly * once each, and that the indicated total number of writes * has been processed by the server process. Flag an error if - * discrepency is noted. Finally reset the counters maintained + * discrepancy is noted. Finally reset the counters maintained * by the server process. * * This function should only be called by the metadata cache @@ -4050,7 +4050,7 @@ verify_writes(unsigned num_writes, haddr_t *written_entries_tbl) /* final barrier to ensure that all processes think that the server * counters have been reset before we leave the sync point. This - * barrier is probaby not necessary at this point in time (5/9/10), + * barrier is probably not necessary at this point in time (5/9/10), * but I can think of at least one likely change to the metadata write * strategies that will require it -- hence its insertion now. */ @@ -4794,7 +4794,7 @@ server_smoke_check(void) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) { HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -5141,7 +5141,7 @@ smoke_check_1(int metadata_write_strategy) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) { HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -5305,7 +5305,7 @@ smoke_check_2(int metadata_write_strategy) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) { HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -5453,7 +5453,7 @@ smoke_check_2(int metadata_write_strategy) * * Purpose: Third smoke check for the parallel cache. * - * Use random reads to vary the loads on the diffferent + * Use random reads to vary the loads on the different * processors. Also force different cache size adjustments. * * In this test, load process 0 heavily, and the other @@ -5510,7 +5510,7 @@ smoke_check_3(int metadata_write_strategy) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) { HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -5738,7 +5738,7 @@ smoke_check_3(int metadata_write_strategy) * * Purpose: Fourth smoke check for the parallel cache. * - * Use random reads to vary the loads on the diffferent + * Use random reads to vary the loads on the different * processors. Also force different cache size adjustments. * * In this test, load process 0 lightly, and the other @@ -5795,7 +5795,7 @@ smoke_check_4(int metadata_write_strategy) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) { HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -6070,7 +6070,7 @@ smoke_check_5(int metadata_write_strategy) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) { HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -6355,7 +6355,7 @@ trace_file_check(int metadata_write_strategy) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -6488,7 +6488,7 @@ trace_file_check(int metadata_write_strategy) } /* end if */ if (nerrors == 0) { - HDsprintf(trace_file_name, "t_cache_trace.txt.%d", (int)file_mpi_rank); + HDsnprintf(trace_file_name, sizeof(trace_file_name), "t_cache_trace.txt.%d", (int)file_mpi_rank); if ((trace_file_ptr = HDfopen(trace_file_name, "r")) == NULL) { @@ -6623,13 +6623,15 @@ trace_file_check(int metadata_write_strategy) static hbool_t smoke_check_6(int metadata_write_strategy) { - hbool_t success = TRUE; - int i; - int max_nerrors; - hid_t fid = -1; - H5F_t * file_ptr = NULL; - H5C_t * cache_ptr = NULL; - struct mssg_t mssg; + H5P_coll_md_read_flag_t md_reads_file_flag; + hbool_t md_reads_context_flag; + hbool_t success = TRUE; + int i; + int max_nerrors; + hid_t fid = -1; + H5F_t * file_ptr = NULL; + H5C_t * cache_ptr = NULL; + struct mssg_t mssg; switch (metadata_write_strategy) { @@ -6660,7 +6662,7 @@ smoke_check_6(int metadata_write_strategy) if (!server_main()) { - /* some error occured in the server -- report failure */ + /* some error occurred in the server -- report failure */ nerrors++; if (verbose) { HDfprintf(stdout, "%d:%s: server_main() failed.\n", world_mpi_rank, __func__); @@ -6685,7 +6687,9 @@ smoke_check_6(int metadata_write_strategy) virt_num_data_entries = NUM_DATA_ENTRIES; /* insert the first half collectively */ - H5CX_set_coll_metadata_read(TRUE); + md_reads_file_flag = H5P_USER_TRUE; + md_reads_context_flag = TRUE; + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); for (i = 0; i < virt_num_data_entries / 2; i++) { struct datum *entry_ptr; entry_ptr = &(data[i]); @@ -6704,9 +6708,13 @@ smoke_check_6(int metadata_write_strategy) H5_CHECK_OVERFLOW(cache_ptr->max_cache_size, size_t, double); HDassert((double)cache_ptr->max_cache_size * 0.8 > cache_ptr->coll_list_size); } + /* Restore collective metadata reads state */ + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); /* insert the other half independently */ - H5CX_set_coll_metadata_read(FALSE); + md_reads_file_flag = H5P_USER_FALSE; + md_reads_context_flag = FALSE; + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); for (i = virt_num_data_entries / 2; i < virt_num_data_entries; i++) { struct datum *entry_ptr; entry_ptr = &(data[i]); @@ -6724,6 +6732,8 @@ smoke_check_6(int metadata_write_strategy) /* Make sure coll entries do not cross the 80% threshold */ HDassert((double)cache_ptr->max_cache_size * 0.8 > cache_ptr->coll_list_size); } + /* Restore collective metadata reads state */ + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); /* flush the file */ if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) { @@ -6734,7 +6744,9 @@ smoke_check_6(int metadata_write_strategy) } /* Protect the first half of the entries collectively */ - H5CX_set_coll_metadata_read(TRUE); + md_reads_file_flag = H5P_USER_TRUE; + md_reads_context_flag = TRUE; + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); for (i = 0; i < (virt_num_data_entries / 2); i++) { struct datum *entry_ptr; entry_ptr = &(data[i]); @@ -6752,9 +6764,13 @@ smoke_check_6(int metadata_write_strategy) /* Make sure coll entries do not cross the 80% threshold */ HDassert((double)cache_ptr->max_cache_size * 0.8 > cache_ptr->coll_list_size); } + /* Restore collective metadata reads state */ + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); /* protect the other half independently */ - H5CX_set_coll_metadata_read(FALSE); + md_reads_file_flag = H5P_USER_FALSE; + md_reads_context_flag = FALSE; + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); for (i = virt_num_data_entries / 2; i < virt_num_data_entries; i++) { struct datum *entry_ptr; entry_ptr = &(data[i]); @@ -6772,6 +6788,8 @@ smoke_check_6(int metadata_write_strategy) /* Make sure coll entries do not cross the 80% threshold */ HDassert((double)cache_ptr->max_cache_size * 0.8 > cache_ptr->coll_list_size); } + /* Restore collective metadata reads state */ + H5F_set_coll_metadata_reads(file_ptr, &md_reads_file_flag, &md_reads_context_flag); for (i = 0; i < (virt_num_data_entries); i++) { unlock_entry(file_ptr, i, H5AC__NO_FLAGS_SET); diff --git a/testpar/t_cache_image.c b/testpar/t_cache_image.c index 17fb992..4229a77 100644 --- a/testpar/t_cache_image.c +++ b/testpar/t_cache_image.c @@ -449,7 +449,7 @@ create_data_sets(hid_t file_id, int min_dset, int max_dset) } /* set the dataset creation plist to specify that the raw data is - * to be partioned into 10X10 element chunks. + * to be partitioned into 10X10 element chunks. */ if (pass) { @@ -477,7 +477,7 @@ create_data_sets(hid_t file_id, int min_dset, int max_dset) /* create the dataset */ if (pass) { - HDsprintf(dset_name, "/dset%03d", i); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", i); dataset_ids[i] = H5Dcreate2(file_id, dset_name, H5T_STD_I32BE, dataspace_id, H5P_DEFAULT, properties, H5P_DEFAULT); @@ -766,7 +766,7 @@ delete_data_sets(hid_t file_id, int min_dset, int max_dset) while ( ( pass ) && ( i <= max_dset ) ) { - HDsprintf(dset_name, "/dset%03d", i); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", i); if ( H5Ldelete(file_id, dset_name, H5P_DEFAULT) < 0) { @@ -795,7 +795,7 @@ delete_data_sets(hid_t file_id, int min_dset, int max_dset) * Set pass to FALSE and issue a suitable failure * message if either the file contains a metadata cache image * superblock extension and mdci_sbem_expected is TRUE, or - * vise versa. + * vice versa. * * If mdci_sbem_expected is TRUE, also verify that the metadata * cache has been advised of this. @@ -807,7 +807,7 @@ delete_data_sets(hid_t file_id, int min_dset, int max_dset) * FAPL entry when opening the file, and verify that the * metadata cache is notified. * - * If config_fsm is TRUE, setup the persistant free space + * If config_fsm is TRUE, setup the persistent free space * manager. Note that this flag may only be set if * create_file is also TRUE. * @@ -884,7 +884,7 @@ open_hdf5_file(const hbool_t create_file, const hbool_t mdci_sbem_expected, cons if (show_progress) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* create a file access propertly list. */ + /* create a file access property list. */ if (pass) { fapl_id = H5Pcreate(H5P_FILE_ACCESS); @@ -956,7 +956,7 @@ open_hdf5_file(const hbool_t create_file, const hbool_t mdci_sbem_expected, cons if (show_progress) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* setup the persistant free space manager if indicated */ + /* setup the persistent free space manager if indicated */ if ((pass) && (config_fsm)) { fcpl_id = H5Pcreate(H5P_FILE_CREATE); @@ -1138,13 +1138,13 @@ open_hdf5_file(const hbool_t create_file, const hbool_t mdci_sbem_expected, cons if ((file_ptr->shared->page_buf) && ((!enable_page_buffer) || (l_facc_type == FACC_MPIO))) { pass = FALSE; - failure_mssg = "page buffer unexepectedly enabled."; + failure_mssg = "page buffer unexpectedly enabled."; } else if ((file_ptr->shared->page_buf != NULL) && ((enable_page_buffer) || (l_facc_type != FACC_MPIO))) { pass = FALSE; - failure_mssg = "page buffer unexepectedly disabled."; + failure_mssg = "page buffer unexpectedly disabled."; } } @@ -1334,7 +1334,7 @@ par_create_dataset(int dset_num, hid_t file_id, int mpi_rank, int mpi_size) show_progress = (show_progress && (mpi_rank == 0)); verbose = (verbose && (mpi_rank == 0)); - HDsprintf(dset_name, "/dset%03d", dset_num); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", dset_num); if (show_progress) { HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name); @@ -1360,7 +1360,7 @@ par_create_dataset(int dset_num, hid_t file_id, int mpi_rank, int mpi_size) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); /* set the dataset creation plist to specify that the raw data is - * to be partioned into 1X10X10 element chunks. + * to be partitioned into 1X10X10 element chunks. */ if (pass) { @@ -1707,7 +1707,7 @@ par_delete_dataset(int dset_num, hid_t file_id, int mpi_rank) show_progress = (show_progress && (mpi_rank == 0)); - HDsprintf(dset_name, "/dset%03d", dset_num); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", dset_num); if (show_progress) { HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name); @@ -1840,7 +1840,7 @@ par_verify_dataset(int dset_num, hid_t file_id, int mpi_rank) show_progress = (show_progress && (mpi_rank == 0)); verbose = (verbose && (mpi_rank == 0)); - HDsprintf(dset_name, "/dset%03d", dset_num); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", dset_num); if (show_progress) { HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name); @@ -2208,7 +2208,7 @@ serial_verify_dataset(int dset_num, hid_t file_id, int mpi_size) hid_t dset_id = -1; hid_t filespace_id = -1; - HDsprintf(dset_name, "/dset%03d", dset_num); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", dset_num); if (show_progress) { HDfprintf(stdout, "%s: dset name = \"%s\".\n", fcn_name, dset_name); @@ -2460,7 +2460,7 @@ verify_data_sets(hid_t file_id, int min_dset, int max_dset) /* open the dataset */ if (pass) { - HDsprintf(dset_name, "/dset%03d", i); + HDsnprintf(dset_name, sizeof(dset_name), "/dset%03d", i); dataset_ids[i] = H5Dopen2(file_id, dset_name, H5P_DEFAULT); if (dataset_ids[i] < 0) { @@ -2796,7 +2796,7 @@ verify_cache_image_RO(int file_name_id, int md_write_strat, int mpi_rank) /* Verify that all other processes receive the cache image block * from process 0. * - * Since we have alread verified that only process 0 has read the + * Since we have already verified that only process 0 has read the * image, it is sufficient to verify that the image was loaded on * all processes. */ @@ -3067,7 +3067,7 @@ verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank) /* Verify that all other processes receive the cache image block * from process 0. * - * Since we have alread verified that only process 0 has read the + * Since we have already verified that only process 0 has read the * image, it is sufficient to verify that the image was loaded on * all processes. */ @@ -3393,7 +3393,7 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) /* Verify that all other processes receive the cache image block * from process 0. * - * Since we have alread verified that only process 0 has read the + * Since we have already verified that only process 0 has read the * image, it is sufficient to verify that the image was loaded on * all processes. */ @@ -3488,7 +3488,7 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) /* Verify that all other processes receive the cache image block * from process 0. * - * Since we have alread verified that only process 0 has read the + * Since we have already verified that only process 0 has read the * image, it is sufficient to verify that the image was loaded on * all processes. */ @@ -3533,9 +3533,9 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); /* 13) Get the size of the file. Verify that it is less - * than 20 KB. Without deletions and persistant free + * than 20 KB. Without deletions and persistent free * space managers, size size is about 30 MB, so this - * is sufficient to verify that the persistant free + * is sufficient to verify that the persistent free * space managers are more or less doing their job. * * Note that this test will have to change if we use diff --git a/testpar/t_coll_chunk.c b/testpar/t_coll_chunk.c index 651a392..104460a 100644 --- a/testpar/t_coll_chunk.c +++ b/testpar/t_coll_chunk.c @@ -155,7 +155,7 @@ coll_chunk2(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size @@ -209,7 +209,7 @@ coll_chunk3(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size @@ -261,7 +261,7 @@ coll_chunk4(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size @@ -315,7 +315,7 @@ coll_chunk5(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size @@ -367,7 +367,7 @@ coll_chunk6(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size @@ -419,7 +419,7 @@ coll_chunk7(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size @@ -471,7 +471,7 @@ coll_chunk8(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size @@ -523,7 +523,7 @@ coll_chunk9(void) */ /* ------------------------------------------------------------------------ - * Descriptions for the selection: one singular selection accross many chunks + * Descriptions for the selection: one singular selection across many chunks * Two dimensions, Num of chunks = 2* mpi_size * * dim1 = SPACE_DIM1*mpi_size diff --git a/testpar/t_coll_md_read.c b/testpar/t_coll_md_read.c index fd62eb6..cabdea0 100644 --- a/testpar/t_coll_md_read.c +++ b/testpar/t_coll_md_read.c @@ -34,10 +34,9 @@ #define MULTI_CHUNK_IO_ADDRMAP_ISSUE_DIMS 2 -#define LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DATASET_NAME "linked_chunk_io_sort_chunk_issue" -#define LINK_CHUNK_IO_SORT_CHUNK_ISSUE_Y_DIM_SCALE 20000 -#define LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE 1 -#define LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS 1 +#define LINK_CHUNK_IO_SORT_CHUNK_ISSUE_COLL_THRESH_NUM 10000 +#define LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DATASET_NAME "linked_chunk_io_sort_chunk_issue" +#define LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS 1 /* * A test for issue HDFFV-10501. A parallel hang was reported which occurred @@ -339,21 +338,34 @@ test_multi_chunk_io_addrmap_issue(void) * collective metadata reads being made only by process 0 in H5D__sort_chunk(). * * NOTE: Due to the way that the threshold value which pertains to this test - * is currently calculated within HDF5, there are several conditions that this - * test must maintain. Refer to the function H5D__sort_chunk in H5Dmpio.c for - * a better idea of why. + * is currently calculated within HDF5, the following two conditions must be + * true to trigger the issue: * - * Condition 1: We need to make sure that the test always selects every single - * chunk in the dataset. It is fine if the selection is split up among multiple - * ranks, but their combined selection must cover the whole dataset. + * Condition 1: A certain threshold ratio must be met in order to have HDF5 + * obtain all chunk addresses collectively inside H5D__sort_chunk(). This is + * given by the following: * - * Condition 2: The number of chunks in the dataset divided by the number of MPI - * ranks must exceed or equal 10000. In other words, each MPI rank must be - * responsible for 10000 or more unique chunks. + * (sum_chunk * 100) / (dataset_nchunks * mpi_size) >= 30% * - * Condition 3: This test will currently only be reliably reproducable for 2 or 3 - * MPI ranks. The threshold value calculated reduces to a constant 100 / mpi_size, - * and is compared against a default value of 30%. + * where: + * * `sum_chunk` is the combined sum of the number of chunks selected in + * the dataset by all ranks (chunks selected by more than one rank count + * individually toward the sum for each rank selecting that chunk) + * * `dataset_nchunks` is the number of chunks in the dataset (selected + * or not) + * * `mpi_size` is the size of the MPI Communicator + * + * Condition 2: `sum_chunk` divided by `mpi_size` must exceed or equal a certain + * threshold (as of this writing, 10000). + * + * To satisfy both these conditions, we #define a macro, + * LINK_CHUNK_IO_SORT_CHUNK_ISSUE_COLL_THRESH_NUM, which corresponds to the + * value of the H5D_ALL_CHUNK_ADDR_THRES_COL_NUM macro in H5Dmpio.c (the + * 10000 threshold from condition 2). We then create a dataset of that many + * chunks and have each MPI rank write to and read from a piece of every single + * chunk in the dataset. This ensures chunk utilization is the max possible + * and exceeds our 30% target ratio, while always exactly matching the numeric + * chunk threshold value of condition 2. * * Failure in this test may either cause a hang, or, due to how the MPI calls * pertaining to this issue might mistakenly match up, may cause an MPI error @@ -375,10 +387,9 @@ void test_link_chunk_io_sort_chunk_issue(void) { const char *filename; - hsize_t * dataset_dims = NULL; - hsize_t max_dataset_dims[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS]; - hsize_t sel_dims[1]; - hsize_t chunk_dims[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS] = {LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS}; + hsize_t dataset_dims[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS]; + hsize_t sel_dims[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS]; + hsize_t chunk_dims[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS]; hsize_t start[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS]; hsize_t stride[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS]; hsize_t count[LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS]; @@ -412,14 +423,13 @@ test_link_chunk_io_sort_chunk_issue(void) file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); VRFY((file_id >= 0), "H5Fcreate succeeded"); - dataset_dims = HDmalloc(LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS * sizeof(*dataset_dims)); - VRFY((dataset_dims != NULL), "malloc succeeded"); - - dataset_dims[0] = (hsize_t)LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE * (hsize_t)mpi_size * - (hsize_t)LINK_CHUNK_IO_SORT_CHUNK_ISSUE_Y_DIM_SCALE; - max_dataset_dims[0] = H5S_UNLIMITED; + /* + * Create a one-dimensional dataset of exactly LINK_CHUNK_IO_SORT_CHUNK_ISSUE_COLL_THRESH_NUM + * chunks, where every rank writes to a piece of every single chunk to keep utilization high. + */ + dataset_dims[0] = (hsize_t)mpi_size * (hsize_t)LINK_CHUNK_IO_SORT_CHUNK_ISSUE_COLL_THRESH_NUM; - fspace_id = H5Screate_simple(LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS, dataset_dims, max_dataset_dims); + fspace_id = H5Screate_simple(LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS, dataset_dims, NULL); VRFY((fspace_id >= 0), "H5Screate_simple succeeded"); /* @@ -428,6 +438,9 @@ test_link_chunk_io_sort_chunk_issue(void) dcpl_id = H5Pcreate(H5P_DATASET_CREATE); VRFY((dcpl_id >= 0), "H5Pcreate succeeded"); + /* Chunk size is equal to MPI size since each rank writes to a piece of every chunk */ + chunk_dims[0] = (hsize_t)mpi_size; + VRFY((H5Pset_chunk(dcpl_id, LINK_CHUNK_IO_SORT_CHUNK_ISSUE_DIMS, chunk_dims) >= 0), "H5Pset_chunk succeeded"); @@ -437,23 +450,21 @@ test_link_chunk_io_sort_chunk_issue(void) /* * Setup hyperslab selection to split the dataset among the ranks. - * - * The ranks will write rows across the dataset. */ - stride[0] = LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE; - count[0] = (dataset_dims[0] / LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE) / (hsize_t)mpi_size; - start[0] = count[0] * (hsize_t)mpi_rank; - block[0] = LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE; + start[0] = (hsize_t)mpi_rank; + stride[0] = (hsize_t)mpi_size; + count[0] = LINK_CHUNK_IO_SORT_CHUNK_ISSUE_COLL_THRESH_NUM; + block[0] = 1; VRFY((H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) >= 0), "H5Sselect_hyperslab succeeded"); - sel_dims[0] = count[0] * (LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE); + sel_dims[0] = count[0]; mspace_id = H5Screate_simple(1, sel_dims, NULL); VRFY((mspace_id >= 0), "H5Screate_simple succeeded"); - data = HDcalloc(1, count[0] * (LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE) * sizeof(int)); + data = HDcalloc(1, count[0] * sizeof(int)); VRFY((data != NULL), "calloc succeeded"); dxpl_id = H5Pcreate(H5P_DATASET_XFER); @@ -476,33 +487,25 @@ test_link_chunk_io_sort_chunk_issue(void) VRFY((H5Pset_dxpl_mpio_chunk_opt(dxpl_id, H5FD_MPIO_CHUNK_ONE_IO) >= 0), "H5Pset_dxpl_mpio_chunk_opt succeeded"); - read_buf = HDmalloc(count[0] * (LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE) * sizeof(int)); + read_buf = HDmalloc(count[0] * sizeof(int)); VRFY((read_buf != NULL), "malloc succeeded"); VRFY((H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) >= 0), "H5Sselect_hyperslab succeeded"); - sel_dims[0] = count[0] * (LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE); + sel_dims[0] = count[0]; VRFY((H5Sclose(mspace_id) >= 0), "H5Sclose succeeded"); mspace_id = H5Screate_simple(1, sel_dims, NULL); VRFY((mspace_id >= 0), "H5Screate_simple succeeded"); - read_buf = HDrealloc(read_buf, count[0] * (LINK_CHUNK_IO_SORT_CHUNK_ISSUE_CHUNK_SIZE) * sizeof(int)); - VRFY((read_buf != NULL), "realloc succeeded"); - /* * Finally have each rank read their section of data back from the dataset. */ VRFY((H5Dread(dset_id, H5T_NATIVE_INT, mspace_id, fspace_id, dxpl_id, read_buf) >= 0), "H5Dread succeeded"); - if (dataset_dims) { - HDfree(dataset_dims); - dataset_dims = NULL; - } - if (data) { HDfree(data); data = NULL; diff --git a/testpar/t_dset.c b/testpar/t_dset.c index bbd4b28..8616bef 100644 --- a/testpar/t_dset.c +++ b/testpar/t_dset.c @@ -2605,7 +2605,7 @@ compress_readAll(void) nerrors++; } -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES ret = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, xfer_plist, data_read); VRFY((ret >= 0), "H5Dwrite succeeded"); #endif @@ -2881,7 +2881,7 @@ none_selection_chunk(void) * Simple independent I/O. This tests that the defaults are properly set. * * TEST_ACTUAL_IO_RESET: - * Perfroms collective and then independent I/O wit hthe same dxpl to + * Performs collective and then independent I/O with hthe same dxpl to * make sure the peroperty is correctly reset to the default on each use. * Specifically, this test runs TEST_ACTUAL_IO_MULTI_CHUNK_NO_OPT_MIX_DISAGREE * (The most complex case that works on all builds) and then performs @@ -2890,7 +2890,7 @@ none_selection_chunk(void) * Note: DIRECT_MULTI_CHUNK_MIX and DIRECT_MULTI_CHUNK_MIX_DISAGREE * is not needed as they are covered by DIRECT_CHUNK_MIX and * MULTI_CHUNK_MIX_DISAGREE cases. _DIRECT_ cases are only for testing - * path way to multi-chunk-io by H5FD_MPIO_CHUNK_MULTI_IO insted of num-threshold. + * path way to multi-chunk-io by H5FD_MPIO_CHUNK_MULTI_IO instead of num-threshold. * * Modification: * - Refctore to remove multi-chunk-without-opimization test and update for @@ -2905,12 +2905,12 @@ none_selection_chunk(void) static void test_actual_io_mode(int selection_mode) { - H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_write = -1; - H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_read = -1; - H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_expected = -1; - H5D_mpio_actual_io_mode_t actual_io_mode_write = -1; - H5D_mpio_actual_io_mode_t actual_io_mode_read = -1; - H5D_mpio_actual_io_mode_t actual_io_mode_expected = -1; + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_write = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_read = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode_expected = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + H5D_mpio_actual_io_mode_t actual_io_mode_write = H5D_MPIO_NO_COLLECTIVE; + H5D_mpio_actual_io_mode_t actual_io_mode_read = H5D_MPIO_NO_COLLECTIVE; + H5D_mpio_actual_io_mode_t actual_io_mode_expected = H5D_MPIO_NO_COLLECTIVE; const char * filename; const char * test_name; hbool_t direct_multi_chunk_io; @@ -3078,7 +3078,7 @@ test_actual_io_mode(int selection_mode) break; /* RESET tests that the properties are properly reset to defaults each time I/O is - * performed. To acheive this, we have RESET perform collective I/O (which would change + * performed. To achieve this, we have RESET perform collective I/O (which would change * the values from the defaults) followed by independent I/O (which should report the * default values). RESET doesn't need to have a unique selection, so we reuse * MULTI_CHUMK_MIX_DISAGREE, which was chosen because it is a complex case that works @@ -3094,7 +3094,7 @@ test_actual_io_mode(int selection_mode) * assgigned collective I/O, while each other process gets independent I/O. * Since the root process with only access the first chunk, it will report * collective I/O. The subsequent processes will access the first chunk - * collectively, and their other chunk indpendently, reporting mixed I/O. + * collectively, and their other chunk independently, reporting mixed I/O. */ if (mpi_rank == 0) { @@ -3163,8 +3163,8 @@ test_actual_io_mode(int selection_mode) default: test_name = "Undefined Selection Mode"; - actual_chunk_opt_mode_expected = -1; - actual_io_mode_expected = -1; + actual_chunk_opt_mode_expected = H5D_MPIO_NO_CHUNK_OPTIMIZATION; + actual_io_mode_expected = H5D_MPIO_NO_COLLECTIVE; break; } @@ -3209,7 +3209,7 @@ test_actual_io_mode(int selection_mode) ret = H5Pset_dxpl_mpio_chunk_opt_num(dxpl_write, (unsigned)mpi_size * 2); VRFY((ret >= 0), "H5Pset_dxpl_mpio_chunk_opt_num succeeded"); - /* set this to manipulate testing senario about allocating processes + /* set this to manipulate testing scenario about allocating processes * to chunks */ ret = H5Pset_dxpl_mpio_chunk_opt_ratio(dxpl_write, (unsigned)99); VRFY((ret >= 0), "H5Pset_dxpl_mpio_chunk_opt_ratio succeeded"); @@ -3233,12 +3233,12 @@ test_actual_io_mode(int selection_mode) H5Eprint2(H5E_DEFAULT, stdout); VRFY((ret >= 0), "H5Dwrite() dataset multichunk write succeeded"); - /* Retreive Actual io valuess */ + /* Retrieve Actual io values */ ret = H5Pget_mpio_actual_io_mode(dxpl_write, &actual_io_mode_write); - VRFY((ret >= 0), "retriving actual io mode suceeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_write, &actual_chunk_opt_mode_write); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); /* Read */ ret = H5Dread(dataset, data_type, mem_space, file_space, dxpl_read, buffer); @@ -3246,12 +3246,12 @@ test_actual_io_mode(int selection_mode) H5Eprint2(H5E_DEFAULT, stdout); VRFY((ret >= 0), "H5Dread() dataset multichunk read succeeded"); - /* Retreive Actual io values */ + /* Retrieve Actual io values */ ret = H5Pget_mpio_actual_io_mode(dxpl_read, &actual_io_mode_read); - VRFY((ret >= 0), "retriving actual io mode succeeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_read, &actual_chunk_opt_mode_read); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); /* Check write vs read */ VRFY((actual_io_mode_read == actual_io_mode_write), @@ -3272,7 +3272,7 @@ test_actual_io_mode(int selection_mode) actual_io_mode_write); } - /* To test that the property is succesfully reset to the default, we perform some + /* To test that the property is successfully reset to the default, we perform some * independent I/O after the collective I/O */ if (selection_mode == TEST_ACTUAL_IO_RESET) { @@ -3289,9 +3289,9 @@ test_actual_io_mode(int selection_mode) /* Check Properties */ ret = H5Pget_mpio_actual_io_mode(dxpl_write, &actual_io_mode_write); - VRFY((ret >= 0), "retriving actual io mode succeeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_write, &actual_chunk_opt_mode_write); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); VRFY(actual_chunk_opt_mode_write == H5D_MPIO_NO_CHUNK_OPTIMIZATION, "actual_chunk_opt_mode has correct value for reset write (independent)"); @@ -3304,9 +3304,9 @@ test_actual_io_mode(int selection_mode) /* Check Properties */ ret = H5Pget_mpio_actual_io_mode(dxpl_read, &actual_io_mode_read); - VRFY((ret >= 0), "retriving actual io mode succeeded"); + VRFY((ret >= 0), "retrieving actual io mode succeeded"); ret = H5Pget_mpio_actual_chunk_opt_mode(dxpl_read, &actual_chunk_opt_mode_read); - VRFY((ret >= 0), "retriving actual chunk opt mode succeeded"); + VRFY((ret >= 0), "retrieving actual chunk opt mode succeeded"); VRFY(actual_chunk_opt_mode_read == H5D_MPIO_NO_CHUNK_OPTIMIZATION, "actual_chunk_opt_mode has correct value for reset read (independent)"); @@ -3359,11 +3359,11 @@ actual_io_mode_tests(void) test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_IND); test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_COL); - /* The Multi Chunk Mixed test requires atleast three processes. */ + /* The Multi Chunk Mixed test requires at least three processes. */ if (mpi_size > 2) test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_MIX); else - HDfprintf(stdout, "Multi Chunk Mixed test requires 3 proceses minimum\n"); + HDfprintf(stdout, "Multi Chunk Mixed test requires 3 processes minimum\n"); test_actual_io_mode(TEST_ACTUAL_IO_MULTI_CHUNK_MIX_DISAGREE); @@ -3407,7 +3407,7 @@ actual_io_mode_tests(void) * Test for Data Type Conversion as the cause of breaking collective I/O. * * TEST_DATA_TRANSFORMS: - * Test for Data Transfrom feature as the cause of breaking collective I/O. + * Test for Data Transform feature as the cause of breaking collective I/O. * * TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES: * Test for NULL dataspace as the cause of breaking collective I/O. @@ -3418,12 +3418,6 @@ actual_io_mode_tests(void) * TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL: * Test for Externl-File storage as the cause of breaking collective I/O. * - * TEST_FILTERS: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * Note: TEST_FILTERS mode will not work until H5Dcreate and H5write is supported for mpio and filter - * feature. Use test_no_collective_cause_mode_filter() function instead. - * - * * Programmer: Jonathan Kim * Date: Aug, 2012 */ @@ -3465,9 +3459,6 @@ test_no_collective_cause_mode(int selection_mode) hid_t file_space = -1; hsize_t chunk_dims[RANK]; herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ /* set to global value as default */ int l_facc_type = facc_type; char message[256]; @@ -3499,21 +3490,6 @@ test_no_collective_cause_mode(int selection_mode) is_chunked = 0; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY((ret >= 0), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info(H5Z_FILTER_FLETCHER32, &filter_info); - VRFY(((filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || - (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED)), - "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0), "set filter (flecher32) succeeded"); - } -#endif /* LATER */ - if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { sid = H5Screate(H5S_NULL); VRFY((sid >= 0), "H5Screate_simple succeeded"); @@ -3571,7 +3547,7 @@ test_no_collective_cause_mode(int selection_mode) } if (selection_mode & TEST_DATA_TRANSFORMS) { - test_name = "Broken Collective I/O - DATA Transfroms"; + test_name = "Broken Collective I/O - DATA Transforms"; no_collective_cause_local_expected |= H5D_MPIO_DATA_TRANSFORMS; no_collective_cause_global_expected |= H5D_MPIO_DATA_TRANSFORMS; } @@ -3589,14 +3565,6 @@ test_no_collective_cause_mode(int selection_mode) no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected |= H5D_MPIO_FILTERS; - no_collective_cause_global_expected |= H5D_MPIO_FILTERS; - } -#endif /* LATER */ - if (selection_mode & TEST_COLLECTIVE) { test_name = "Broken Collective I/O - Not Broken"; no_collective_cause_local_expected = H5D_MPIO_COLLECTIVE; @@ -3669,7 +3637,7 @@ test_no_collective_cause_mode(int selection_mode) /* Get the cause of broken collective I/O */ ret = H5Pget_mpio_no_collective_cause(dxpl_write, &no_collective_cause_local_write, &no_collective_cause_global_write); - VRFY((ret >= 0), "retriving no collective cause succeeded"); + VRFY((ret >= 0), "retrieving no collective cause succeeded"); /*--------------------- * Test Read access @@ -3689,7 +3657,7 @@ test_no_collective_cause_mode(int selection_mode) /* Get the cause of broken collective I/O */ ret = H5Pget_mpio_no_collective_cause(dxpl_read, &no_collective_cause_local_read, &no_collective_cause_global_read); - VRFY((ret >= 0), "retriving no collective cause succeeded"); + VRFY((ret >= 0), "retrieving no collective cause succeeded"); /* Check write vs read */ VRFY((no_collective_cause_local_read == no_collective_cause_local_write), @@ -3699,10 +3667,12 @@ test_no_collective_cause_mode(int selection_mode) /* Test values */ HDmemset(message, 0, sizeof(message)); - HDsprintf(message, "Local cause of Broken Collective I/O has the correct value for %s.\n", test_name); + HDsnprintf(message, sizeof(message), + "Local cause of Broken Collective I/O has the correct value for %s.\n", test_name); VRFY((no_collective_cause_local_write == no_collective_cause_local_expected), message); HDmemset(message, 0, sizeof(message)); - HDsprintf(message, "Global cause of Broken Collective I/O has the correct value for %s.\n", test_name); + HDsnprintf(message, sizeof(message), + "Global cause of Broken Collective I/O has the correct value for %s.\n", test_name); VRFY((no_collective_cause_global_write == no_collective_cause_global_expected), message); /* Release some resources */ @@ -3733,240 +3703,6 @@ test_no_collective_cause_mode(int selection_mode) return; } -/* - * Function: test_no_collective_cause_mode_filter - * - * Purpose: - * Test specific for using filter as a caus of broken collective I/O and - * checks that the H5Pget_mpio_no_collective_cause properties in the DXPL - * have the correct values. - * - * NOTE: - * This is a temporary function. - * test_no_collective_cause_mode(TEST_FILTERS) will replace this when - * H5Dcreate and H5write support for mpio and filter feature. - * - * Input: - * TEST_FILTERS_READ: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * - * Programmer: Jonathan Kim - * Date: Aug, 2012 - */ -#ifdef LATER -static void -test_no_collective_cause_mode_filter(int selection_mode) -{ - uint32_t no_collective_cause_local_read = 0; - uint32_t no_collective_cause_local_expected = 0; - uint32_t no_collective_cause_global_read = 0; - uint32_t no_collective_cause_global_expected = 0; - - const char *filename; - const char *test_name = "I/O"; - hbool_t is_chunked = 1; - int mpi_size = -1; - int mpi_rank = -1; - int length; - int * buffer; - int i; - MPI_Comm mpi_comm = MPI_COMM_NULL; - MPI_Info mpi_info = MPI_INFO_NULL; - hid_t fid = -1; - hid_t sid = -1; - hid_t dataset = -1; - hid_t data_type = H5T_NATIVE_INT; - hid_t fapl_write = -1; - hid_t fapl_read = -1; - hid_t dcpl = -1; - hid_t dxpl = -1; - hsize_t dims[RANK]; - hid_t mem_space = -1; - hid_t file_space = -1; - hsize_t chunk_dims[RANK]; - herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ - char message[256]; - - /* Set up MPI parameters */ - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - - MPI_Barrier(MPI_COMM_WORLD); - - HDassert(mpi_size >= 1); - - mpi_comm = MPI_COMM_WORLD; - mpi_info = MPI_INFO_NULL; - - /* Create the dataset creation plist */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - VRFY((dcpl >= 0), "dataset creation plist created successfully"); - - if (selection_mode == TEST_FILTERS_READ) { -#ifdef LATER /* fletcher32 */ - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY((ret >= 0), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info(H5Z_FILTER_FLETCHER32, (unsigned int *)&filter_info); - VRFY(((filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || - (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED)), - "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0), "set filter (flecher32) succeeded"); -#endif /* LATER */ - } - else { - VRFY(0, "Unexpected mode, only test for TEST_FILTERS_READ."); - } - - /* Create the basic Space */ - dims[0] = (hsize_t)dim0; - dims[1] = (hsize_t)dim1; - sid = H5Screate_simple(RANK, dims, NULL); - VRFY((sid >= 0), "H5Screate_simple succeeded"); - - filename = (const char *)GetTestParameters(); - HDassert(filename != NULL); - - /* Setup the file access template */ - fapl_write = create_faccess_plist(mpi_comm, mpi_info, FACC_DEFAULT); - VRFY((fapl_write >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_write); - VRFY((fid >= 0), "H5Fcreate succeeded"); - - /* If we are not testing contiguous datasets */ - if (is_chunked) { - /* Set up chunk information. */ - chunk_dims[0] = dims[0] / (hsize_t)mpi_size; - chunk_dims[1] = dims[1]; - ret = H5Pset_chunk(dcpl, 2, chunk_dims); - VRFY((ret >= 0), "chunk creation property list succeeded"); - } - - /* Create the dataset */ - dataset = H5Dcreate2(fid, DSET_NOCOLCAUSE, data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); - -#ifdef LATER /* fletcher32 */ - /* Set expected cause */ - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected = H5D_MPIO_FILTERS; - no_collective_cause_global_expected = H5D_MPIO_FILTERS; -#endif /* LATER */ - - /* Get the file dataspace */ - file_space = H5Dget_space(dataset); - VRFY((file_space >= 0), "H5Dget_space succeeded"); - - /* Create the memory dataspace */ - mem_space = H5Screate_simple(RANK, dims, NULL); - VRFY((mem_space >= 0), "mem_space created"); - - /* Get the number of elements in the selection */ - length = dim0 * dim1; - - /* Allocate and initialize the buffer */ - buffer = (int *)HDmalloc(sizeof(int) * length); - VRFY((buffer != NULL), "HDmalloc of buffer succeeded"); - for (i = 0; i < length; i++) - buffer[i] = i; - - /* Set up the dxpl for the write */ - dxpl = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - - if (selection_mode == TEST_FILTERS_READ) { - /* To test read in collective I/O mode , write in independent mode - * because write fails with mpio + filter */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - else { - /* To test write in collective I/O mode. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - - /* Write */ - ret = H5Dwrite(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if (ret < 0) - H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dwrite() dataset multichunk write succeeded"); - - /* Make a copy of the dxpl to test the read operation */ - dxpl = H5Pcopy(dxpl); - VRFY((dxpl >= 0), "H5Pcopy succeeded"); - - if (dataset) - H5Dclose(dataset); - if (fapl_write) - H5Pclose(fapl_write); - if (fid) - H5Fclose(fid); - - /*--------------------- - * Test Read access - *---------------------*/ - - /* Setup the file access template */ - fapl_read = create_faccess_plist(mpi_comm, mpi_info, facc_type); - VRFY((fapl_read >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl_read); - dataset = H5Dopen2(fid, DSET_NOCOLCAUSE, H5P_DEFAULT); - - /* Set collective I/O properties in the dxpl. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - - /* Read */ - ret = H5Dread(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if (ret < 0) - H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dread() dataset multichunk read succeeded"); - - /* Get the cause of broken collective I/O */ - ret = H5Pget_mpio_no_collective_cause(dxpl, &no_collective_cause_local_read, - &no_collective_cause_global_read); - VRFY((ret >= 0), "retriving no collective cause succeeded"); - - /* Test values */ - HDmemset(message, 0, sizeof(message)); - HDsprintf(message, "Local cause of Broken Collective I/O has the correct value for %s.\n", test_name); - VRFY((no_collective_cause_local_read == (uint32_t)no_collective_cause_local_expected), message); - HDmemset(message, 0, sizeof(message)); - HDsprintf(message, "Global cause of Broken Collective I/O has the correct value for %s.\n", test_name); - VRFY((no_collective_cause_global_read == (uint32_t)no_collective_cause_global_expected), message); - - /* Release some resources */ - if (sid) - H5Sclose(sid); - if (fapl_read) - H5Pclose(fapl_read); - if (dcpl) - H5Pclose(dcpl); - if (dxpl) - H5Pclose(dxpl); - if (dataset) - H5Dclose(dataset); - if (mem_space) - H5Sclose(mem_space); - if (file_space) - H5Sclose(file_space); - if (fid) - H5Fclose(fid); - HDfree(buffer); - return; -} -#endif - /* Function: no_collective_cause_tests * * Purpose: Tests cases for broken collective IO. @@ -3987,13 +3723,6 @@ no_collective_cause_tests(void) test_no_collective_cause_mode(TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL); -#ifdef LATER /* fletcher32 */ - /* TODO: use this instead of below TEST_FILTERS_READ when H5Dcreate and - * H5Dwrite is ready for mpio + filter feature. - */ - /* test_no_collective_cause_mode (TEST_FILTERS); */ - test_no_collective_cause_mode_filter(TEST_FILTERS_READ); -#endif /* LATER */ /* * Test combined causes @@ -4114,7 +3843,7 @@ dataset_atomicity(void) /* file locking allows only one file open (serial) for writing */ if (MAINPROCESS) { fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); - VRFY((fid >= 0), "H5Fopen succeeed"); + VRFY((fid >= 0), "H5Fopen succeeded"); /* should fail */ H5E_BEGIN_TRY diff --git a/testpar/t_file.c b/testpar/t_file.c index 9554052..9970538 100644 --- a/testpar/t_file.c +++ b/testpar/t_file.c @@ -106,7 +106,7 @@ test_split_comm_access(void) /* delete the test file */ if (sub_mpi_rank == 0) { - mrc = MPI_File_delete((char *)filename, info); + mrc = MPI_File_delete(filename, info); /*VRFY((mrc==MPI_SUCCESS), ""); */ } } @@ -180,7 +180,7 @@ test_page_buffer_access(void) data = (int *)HDmalloc(sizeof(int) * (size_t)num_elements); - /* intialize all the elements to have a value of -1 */ + /* initialize all the elements to have a value of -1 */ for (i = 0; i < num_elements; i++) data[i] = -1; if (MAINPROCESS) { @@ -516,25 +516,25 @@ create_file(const char *filename, hid_t fcpl, hid_t fapl, int metadata_write_str VRFY((mem_dataspace >= 0), ""); for (k = 0; k < NUM_DSETS; k++) { - HDsprintf(dset_name, "D1dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "D1dset%d", k); dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); VRFY((dset_id >= 0), ""); ret = H5Dclose(dset_id); VRFY((ret == 0), ""); - HDsprintf(dset_name, "D2dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "D2dset%d", k); dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); VRFY((dset_id >= 0), ""); ret = H5Dclose(dset_id); VRFY((ret == 0), ""); - HDsprintf(dset_name, "D3dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "D3dset%d", k); dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); VRFY((dset_id >= 0), ""); ret = H5Dclose(dset_id); VRFY((ret == 0), ""); - HDsprintf(dset_name, "dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "dset%d", k); dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); VRFY((dset_id >= 0), ""); @@ -557,13 +557,13 @@ create_file(const char *filename, hid_t fcpl, hid_t fapl, int metadata_write_str for (i = 0; i < num_elements; i++) VRFY((data_array[i] == mpi_rank + 1), "Dataset Verify failed"); - HDsprintf(dset_name, "D1dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "D1dset%d", k); ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); VRFY((ret == 0), ""); - HDsprintf(dset_name, "D2dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "D2dset%d", k); ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); VRFY((ret == 0), ""); - HDsprintf(dset_name, "D3dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "D3dset%d", k); ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); VRFY((ret == 0), ""); } @@ -669,7 +669,7 @@ open_file(const char *filename, hid_t fapl, int metadata_write_strategy, hsize_t VRFY((mem_dataspace >= 0), ""); for (k = 0; k < NUM_DSETS; k++) { - HDsprintf(dset_name, "dset%d", k); + HDsnprintf(dset_name, sizeof(dset_name), "dset%d", k); dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT); VRFY((dset_id >= 0), ""); diff --git a/testpar/t_filters_parallel.c b/testpar/t_filters_parallel.c index 78af0fb..8a55519 100644 --- a/testpar/t_filters_parallel.c +++ b/testpar/t_filters_parallel.c @@ -26,73 +26,139 @@ const char *FILENAME[] = {"t_filters_parallel", NULL}; char filenames[1][256]; +static MPI_Comm comm = MPI_COMM_WORLD; +static MPI_Info info = MPI_INFO_NULL; +static int mpi_rank; +static int mpi_size; + int nerrors = 0; -size_t cur_filter_idx = 0; -#define GZIP_INDEX 0 -#define FLETCHER32_INDEX 1 +/* Arrays of filter ID values and filter names (should match each other) */ +H5Z_filter_t filterIDs[] = { + H5Z_FILTER_DEFLATE, H5Z_FILTER_SHUFFLE, H5Z_FILTER_FLETCHER32, + H5Z_FILTER_SZIP, H5Z_FILTER_NBIT, H5Z_FILTER_SCALEOFFSET, +}; + +const char *filterNames[] = {"Deflate", "Shuffle", "Fletcher32", "SZIP", "Nbit", "ScaleOffset"}; + +/* Function pointer typedef for test functions */ +typedef void (*test_func)(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id); -#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0]) +/* Typedef for filter arguments for user-defined filters */ +typedef struct filter_options_t { + unsigned int flags; + size_t cd_nelmts; + const unsigned int cd_values[]; +} filter_options_t; /* - * Used to check if a filter is available before running a test. + * Enum for verify_space_alloc_status which specifies + * how many chunks have been written to in a dataset */ -#define CHECK_CUR_FILTER_AVAIL() \ - { \ - htri_t filter_is_avail; \ - \ - if (cur_filter_idx == GZIP_INDEX) { \ - if ((filter_is_avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE)) != TRUE) { \ - if (MAINPROCESS) { \ - HDputs(" - SKIPPED - Deflate filter not available"); \ - } \ - return; \ - } \ - } \ - } +typedef enum num_chunks_written_t { + DATASET_JUST_CREATED, + NO_CHUNKS_WRITTEN, + SOME_CHUNKS_WRITTEN, + ALL_CHUNKS_WRITTEN +} num_chunks_written_t; -static herr_t set_dcpl_filter(hid_t dcpl); +static herr_t set_dcpl_filter(hid_t dcpl_id, H5Z_filter_t filter_id, filter_options_t *filter_options); +static herr_t verify_space_alloc_status(hid_t dset_id, hid_t dcpl_id, num_chunks_written_t chunks_written); -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES /* Tests for writing data in parallel */ -static void test_write_one_chunk_filtered_dataset(void); -static void test_write_filtered_dataset_no_overlap(void); -static void test_write_filtered_dataset_overlap(void); -static void test_write_filtered_dataset_single_no_selection(void); -static void test_write_filtered_dataset_all_no_selection(void); -static void test_write_filtered_dataset_point_selection(void); -static void test_write_filtered_dataset_interleaved_write(void); -static void test_write_transformed_filtered_dataset_no_overlap(void); -static void test_write_3d_filtered_dataset_no_overlap_separate_pages(void); -static void test_write_3d_filtered_dataset_no_overlap_same_pages(void); -static void test_write_3d_filtered_dataset_overlap(void); -static void test_write_cmpd_filtered_dataset_no_conversion_unshared(void); -static void test_write_cmpd_filtered_dataset_no_conversion_shared(void); -static void test_write_cmpd_filtered_dataset_type_conversion_unshared(void); -static void test_write_cmpd_filtered_dataset_type_conversion_shared(void); +static void test_write_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_no_overlap_partial(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_single_unlim_dim_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_single_unlim_dim_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_multi_unlim_dim_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_multi_unlim_dim_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_interleaved_write(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_transformed_filtered_dataset_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_3d_filtered_dataset_no_overlap_separate_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_3d_filtered_dataset_no_overlap_same_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_no_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_no_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_type_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_type_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); #endif /* Tests for reading data in parallel */ -static void test_read_one_chunk_filtered_dataset(void); -static void test_read_filtered_dataset_no_overlap(void); -static void test_read_filtered_dataset_overlap(void); -static void test_read_filtered_dataset_single_no_selection(void); -static void test_read_filtered_dataset_all_no_selection(void); -static void test_read_filtered_dataset_point_selection(void); -static void test_read_filtered_dataset_interleaved_read(void); -static void test_read_transformed_filtered_dataset_no_overlap(void); -static void test_read_3d_filtered_dataset_no_overlap_separate_pages(void); -static void test_read_3d_filtered_dataset_no_overlap_same_pages(void); -static void test_read_3d_filtered_dataset_overlap(void); -static void test_read_cmpd_filtered_dataset_no_conversion_unshared(void); -static void test_read_cmpd_filtered_dataset_no_conversion_shared(void); -static void test_read_cmpd_filtered_dataset_type_conversion_unshared(void); -static void test_read_cmpd_filtered_dataset_type_conversion_shared(void); - -#if MPI_VERSION >= 3 -/* Other miscellaneous tests */ -static void test_shrinking_growing_chunks(void); -#endif +static void test_read_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_interleaved_read(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_transformed_filtered_dataset_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_3d_filtered_dataset_no_overlap_separate_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_3d_filtered_dataset_no_overlap_same_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_no_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_no_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_type_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_type_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); /* * Tests for attempting to round-trip the data going from @@ -103,21 +169,40 @@ static void test_shrinking_growing_chunks(void); * * written in parallel -> read serially */ -static void test_write_serial_read_parallel(void); -#if MPI_VERSION >= 3 -static void test_write_parallel_read_serial(void); -#endif +static void test_write_serial_read_parallel(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); -static MPI_Comm comm = MPI_COMM_WORLD; -static MPI_Info info = MPI_INFO_NULL; -static int mpi_rank; -static int mpi_size; +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES +static void test_write_parallel_read_serial(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); -static void (*tests[])(void) = { -#if MPI_VERSION >= 3 +/* Other miscellaneous tests */ +static void test_shrinking_growing_chunks(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_edge_chunks_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_edge_chunks_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_edge_chunks_partial_write(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_fill_values(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id); +static void test_fill_value_undefined(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_fill_time_never(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +#endif + +static test_func tests[] = { +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES test_write_one_chunk_filtered_dataset, test_write_filtered_dataset_no_overlap, + test_write_filtered_dataset_no_overlap_partial, test_write_filtered_dataset_overlap, + test_write_filtered_dataset_single_unlim_dim_no_overlap, + test_write_filtered_dataset_single_unlim_dim_overlap, + test_write_filtered_dataset_multi_unlim_dim_no_overlap, + test_write_filtered_dataset_multi_unlim_dim_overlap, test_write_filtered_dataset_single_no_selection, test_write_filtered_dataset_all_no_selection, test_write_filtered_dataset_point_selection, @@ -147,33 +232,168 @@ static void (*tests[])(void) = { test_read_cmpd_filtered_dataset_type_conversion_unshared, test_read_cmpd_filtered_dataset_type_conversion_shared, test_write_serial_read_parallel, -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES test_write_parallel_read_serial, test_shrinking_growing_chunks, + test_edge_chunks_no_overlap, + test_edge_chunks_overlap, + test_edge_chunks_partial_write, + test_fill_values, + test_fill_value_undefined, + test_fill_time_never, #endif }; /* * Function to call the appropriate HDF5 filter-setting function - * depending on the currently set index. Used to re-run the tests + * depending on the given filter ID. Used to re-run the tests * with different filters to check that the data still comes back * correctly under a variety of circumstances, such as the * Fletcher32 checksum filter increasing the size of the chunk. */ static herr_t -set_dcpl_filter(hid_t dcpl) +set_dcpl_filter(hid_t dcpl_id, H5Z_filter_t filter_id, filter_options_t *filter_options) +{ + switch (filter_id) { + case H5Z_FILTER_DEFLATE: + return H5Pset_deflate(dcpl_id, DEFAULT_DEFLATE_LEVEL); + case H5Z_FILTER_SHUFFLE: + return H5Pset_shuffle(dcpl_id); + case H5Z_FILTER_FLETCHER32: + return H5Pset_fletcher32(dcpl_id); + case H5Z_FILTER_SZIP: { + unsigned pixels_per_block = H5_SZIP_MAX_PIXELS_PER_BLOCK; + hsize_t chunk_dims[H5S_MAX_RANK] = {0}; + size_t i, chunk_nelemts; + + VRFY(H5Pget_chunk(dcpl_id, H5S_MAX_RANK, chunk_dims) >= 0, "H5Pget_chunk succeeded"); + + for (i = 0, chunk_nelemts = 1; i < H5S_MAX_RANK; i++) + if (chunk_dims[i] > 0) + chunk_nelemts *= chunk_dims[i]; + + if (chunk_nelemts < H5_SZIP_MAX_PIXELS_PER_BLOCK) { + /* + * Can't set SZIP for chunk of 1 data element. + * Pixels-per-block value must be both even + * and non-zero. + */ + if (chunk_nelemts == 1) + return SUCCEED; + + if ((chunk_nelemts % 2) == 0) + pixels_per_block = (unsigned)chunk_nelemts; + else + pixels_per_block = (unsigned)(chunk_nelemts - 1); + } + else + pixels_per_block = H5_SZIP_MAX_PIXELS_PER_BLOCK; + + return H5Pset_szip(dcpl_id, 0, pixels_per_block); + } + case H5Z_FILTER_NBIT: + return H5Pset_nbit(dcpl_id); + case H5Z_FILTER_SCALEOFFSET: + return H5Pset_scaleoffset(dcpl_id, H5Z_SO_INT, 0); + default: { + if (!filter_options) + return FAIL; + + return H5Pset_filter(dcpl_id, filter_id, filter_options->flags, filter_options->cd_nelmts, + filter_options->cd_values); + } + } +} + +/* + * Function to verify the status of dataset storage space allocation + * based on the dataset's allocation time setting and how many chunks + * in the dataset have been written to. + */ +static herr_t +verify_space_alloc_status(hid_t dset_id, hid_t dcpl_id, num_chunks_written_t chunks_written) { - switch (cur_filter_idx) { - case GZIP_INDEX: - return H5Pset_deflate(dcpl, DEFAULT_DEFLATE_LEVEL); - case FLETCHER32_INDEX: - return H5Pset_fletcher32(dcpl); - default: - return H5Pset_deflate(dcpl, DEFAULT_DEFLATE_LEVEL); + int nfilters; + herr_t ret_value = SUCCEED; + + VRFY(((nfilters = H5Pget_nfilters(dcpl_id)) >= 0), "H5Pget_nfilters succeeded"); + + /* + * Only verify space allocation status when there are filters + * in the dataset's filter pipeline. When filters aren't in the + * pipeline, the space allocation time and status can vary based + * on whether the file was created in parallel or serial mode. + */ + if (nfilters > 0) { + H5D_space_status_t space_status; + H5D_alloc_time_t alloc_time; + + VRFY((H5Pget_alloc_time(dcpl_id, &alloc_time) >= 0), "H5Pget_alloc_time succeeded"); + VRFY((H5Dget_space_status(dset_id, &space_status) >= 0), "H5Dget_space_status succeeded"); + + switch (alloc_time) { + case H5D_ALLOC_TIME_EARLY: + /* + * Early space allocation should always result in the + * full dataset storage space being allocated. + */ + VRFY(space_status == H5D_SPACE_STATUS_ALLOCATED, "verified space allocation status"); + break; + case H5D_ALLOC_TIME_LATE: + /* + * Late space allocation should always result in the + * full dataset storage space being allocated when + * the dataset gets written to. However, if the dataset + * is extended the dataset's space allocation status + * can become partly allocated until the dataset is + * written to again. + */ + if (chunks_written == SOME_CHUNKS_WRITTEN || chunks_written == ALL_CHUNKS_WRITTEN) + VRFY((space_status == H5D_SPACE_STATUS_ALLOCATED) || + (space_status == H5D_SPACE_STATUS_PART_ALLOCATED), + "verified space allocation status"); + else if (chunks_written == NO_CHUNKS_WRITTEN) + /* + * A special case where we wrote to a dataset that + * uses late space allocation, but the write was + * either a no-op (no selection in the dataset + * from any rank) or something caused the write to + * fail late in the process of performing the actual + * write. In either case, space should still have + * been allocated. + */ + VRFY(space_status == H5D_SPACE_STATUS_ALLOCATED, "verified space allocation status"); + else + VRFY(space_status == H5D_SPACE_STATUS_NOT_ALLOCATED, "verified space allocation status"); + break; + case H5D_ALLOC_TIME_DEFAULT: + case H5D_ALLOC_TIME_INCR: + /* + * Incremental space allocation should result in + * the dataset's storage space being incrementally + * allocated as chunks are written to. Once all chunks + * have been written to, the space allocation should be + * seen as fully allocated. + */ + if (chunks_written == SOME_CHUNKS_WRITTEN) + VRFY((space_status == H5D_SPACE_STATUS_PART_ALLOCATED), + "verified space allocation status"); + else if (chunks_written == ALL_CHUNKS_WRITTEN) + VRFY((space_status == H5D_SPACE_STATUS_ALLOCATED), "verified space allocation status"); + else + VRFY(space_status == H5D_SPACE_STATUS_NOT_ALLOCATED, "verified space allocation status"); + break; + default: + if (MAINPROCESS) + MESG("unknown space allocation time"); + MPI_Abort(MPI_COMM_WORLD, 1); + } } + + return ret_value; } -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES /* * Tests parallel write of filtered data in the special * case where a dataset is composed of a single chunk. @@ -182,7 +402,8 @@ set_dcpl_filter(hid_t dcpl) * 02/01/2017 */ static void -test_write_one_chunk_filtered_dataset(void) +test_write_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; @@ -195,26 +416,18 @@ test_write_one_chunk_filtered_dataset(void) hsize_t count[WRITE_ONE_CHUNK_FILTERED_DATASET_DIMS]; hsize_t block[WRITE_ONE_CHUNK_FILTERED_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) HDputs("Testing write to one-chunk filtered dataset"); - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ dataset_dims[0] = (hsize_t)WRITE_ONE_CHUNK_FILTERED_DATASET_NROWS; @@ -231,19 +444,21 @@ test_write_one_chunk_filtered_dataset(void) VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); VRFY((H5Pset_chunk(plist_id, WRITE_ONE_CHUNK_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes @@ -293,15 +508,12 @@ test_write_one_chunk_filtered_dataset(void) ((C_DATATYPE)i / (WRITE_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * WRITE_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -311,10 +523,10 @@ test_write_one_chunk_filtered_dataset(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -324,10 +536,11 @@ test_write_one_chunk_filtered_dataset(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -343,7 +556,8 @@ test_write_one_chunk_filtered_dataset(void) * 02/01/2017 */ static void -test_write_filtered_dataset_no_overlap(void) +test_write_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; @@ -356,27 +570,18 @@ test_write_filtered_dataset_no_overlap(void) hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) HDputs("Testing write to unshared filtered chunks"); - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_NROWS; @@ -393,20 +598,22 @@ test_write_filtered_dataset_no_overlap(void) VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes @@ -454,15 +661,12 @@ test_write_filtered_dataset_no_overlap(void) correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -472,10 +676,10 @@ test_write_filtered_dataset_no_overlap(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -485,102 +689,91 @@ test_write_filtered_dataset_no_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case where - * more than one process is writing to a particular chunk - * in the operation. In this case, the chunks have to be - * redistributed before the operation so that only one process - * writes to a particular chunk. - * - * Programmer: Jordan Henderson - * 02/01/2017 + * Tests parallel write of filtered data in the case where only + * one process is writing to a particular chunk in the operation + * and that process only writes to part of a chunk. */ static void -test_write_filtered_dataset_overlap(void) +test_write_filtered_dataset_no_overlap_partial(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to shared filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + HDputs("Testing partial write to unshared filtered chunks"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; - sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - - filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + sel_dims[1] = (hsize_t)(WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS / + WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS); + + filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; - count[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; - block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * block[0]; + count[0] = 1; + count[1] = (hsize_t)(WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS / + WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS); + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + block[1] = (hsize_t)1; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS * count[0]); start[1] = 0; if (VERBOSE_MED) { @@ -611,33 +804,38 @@ test_write_filtered_dataset_overlap(void) for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)( - (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) { + size_t rank_n_elems = (size_t)(mpi_size * (WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS * + WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS)); + size_t data_idx = i; - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + for (size_t j = 0; j < rank_n_elems; j++) { + if ((j % WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS) == 0) { + correct_buf[(i * rank_n_elems) + j] = (C_DATATYPE)data_idx; + data_idx++; + } + } + } - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - /* Verify correct data was written */ + /* Verify the correct data was written */ read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -647,10 +845,10 @@ test_write_filtered_dataset_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -658,98 +856,85 @@ test_write_filtered_dataset_overlap(void) /* * Tests parallel write of filtered data in the case where - * a single process in the write operation has no selection - * in the dataset's dataspace. In this case, the process with - * no selection still has to participate in the collective - * space re-allocation for the filtered chunks and also must - * participate in the re-insertion of the filtered chunks - * into the chunk index. + * more than one process is writing to a particular chunk + * in the operation. In this case, the chunks have to be + * redistributed before the operation so that only one process + * writes to a particular chunk. * * Programmer: Jordan Henderson * 02/01/2017 */ static void -test_write_filtered_dataset_single_no_selection(void) +test_write_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - size_t segment_length; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to filtered chunks with a single process having no selection"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to shared filtered chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - - if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - sel_dims[0] = sel_dims[1] = 0; + dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / - (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; + count[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; + count[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; start[1] = 0; if (VERBOSE_MED) { @@ -764,11 +949,8 @@ test_write_filtered_dataset_single_no_selection(void) filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); - else - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); /* Fill data buffer */ data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); @@ -784,37 +966,29 @@ test_write_filtered_dataset_single_no_selection(void) data[i] = (C_DATATYPE)GEN_DATA(i); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - - /* Compute the correct offset into the buffer for the process having no selection and clear it */ - segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; - HDmemset(correct_buf + - ((size_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), - 0, segment_length * sizeof(*data)); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf[i] = (C_DATATYPE)( + (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - /* Verify the correct data was written */ + /* Verify correct data was written */ read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -824,294 +998,318 @@ test_write_filtered_dataset_single_no_selection(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case - * where no process in the write operation has a - * selection in the dataset's dataspace. This test is - * to ensure that there are no assertion failures or - * similar issues due to size 0 allocations and the - * like. In this case, the file and dataset are created - * but the dataset is populated with the default fill - * value. - * - * Programmer: Jordan Henderson - * 02/02/2017 + * Tests parallel write of filtered data in the case where + * a dataset has a single unlimited dimension and each + * MPI rank writes to its own separate chunk. On each + * iteration, the dataset is extended in its extensible + * dimension by "MPI size" chunks per rank and the new + * chunks are written to, read back and verified. */ static void -test_write_filtered_dataset_all_no_selection(void) +test_write_filtered_dataset_single_unlim_dim_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to filtered chunks with all processes having no selection"); + HDputs("Testing write to unshared filtered chunks w/ single unlimited dimension"); - CHECK_CUR_FILTER_AVAIL(); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS; + max_dims[0] = dataset_dims[0]; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS; + + filespace = H5Screate_simple(WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = sel_dims[1] = 0; - - filespace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - memspace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - dset_id = H5Dcreate2(file_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < (size_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NLOOPS; i++) { + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = 1; + count[1] = + (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS / (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + stride[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * block[0] * count[0]); + start[1] = i * count[1] * block[1]; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - if (data) - HDfree(data); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - /* Verify the correct data was written */ - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + HDmemset(read_buf, 255, data_size); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - if (correct_buf) - HDfree(correct_buf); + /* Verify the correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); + + if (i < (size_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NLOOPS - 1) { + /* Extend the dataset by count[1] chunks in the extensible dimension */ + dataset_dims[1] += count[1] * block[1]; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } + + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data by using - * point selections instead of hyperslab selections. - * - * Programmer: Jordan Henderson - * 02/02/2017 + * Tests parallel write of filtered data in the case where + * a dataset has a single unlimited dimension and each + * MPI rank writes to a portion of each chunk in the dataset. + * On each iteration, the dataset is extended in its extensible + * dimension by two chunks and the new chunks are written to + * by all ranks, then read back and verified. */ static void -test_write_filtered_dataset_point_selection(void) +test_write_filtered_dataset_single_unlim_dim_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *read_buf = NULL; - hsize_t * coords = NULL; - hsize_t dataset_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - size_t i, j, data_size, correct_buf_size; - size_t num_points; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to filtered chunks with point selection"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + HDputs("Testing write to shared filtered chunks w/ single unlimited dimension"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NCOLS; + max_dims[0] = dataset_dims[0]; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_SHARED_ONE_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Set up point selection */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - - num_points = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS * - (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; - coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); - VRFY((NULL != coords), "Coords HDcalloc succeeded"); - - for (i = 0; i < num_points; i++) - for (j = 0; j < WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) - coords[(i * WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = - (j > 0) ? (i % (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) - : ((hsize_t)mpi_rank + - ((hsize_t)mpi_size * (i / (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), - "Point selection succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)( - (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + for (i = 0; i < (size_t)WRITE_SHARED_ONE_UNLIM_DIM_NLOOPS; i++) { + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NROWS / (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS; + count[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NCOLS / (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + stride[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + block[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = i * count[1] * block[1]; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - if (data) - HDfree(data); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - /* Verify the correct data was written */ - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + dset_id = H5Dopen2(group_id, WRITE_SHARED_ONE_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + HDmemset(read_buf, 255, data_size); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + /* Verify correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - if (coords) - HDfree(coords); - if (correct_buf) - HDfree(correct_buf); + if (i < (size_t)WRITE_SHARED_ONE_UNLIM_DIM_NLOOPS - 1) { + /* Extend the dataset by count[1] chunks in the extensible dimension */ + dataset_dims[1] += count[1] * block[1]; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } + + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -1119,272 +1317,424 @@ test_write_filtered_dataset_point_selection(void) /* * Tests parallel write of filtered data in the case where - * each process writes an equal amount of data to each chunk - * in the dataset. Each chunk is distributed among the - * processes in round-robin fashion by blocks of size 1 until - * the whole chunk is selected, leading to an interleaved - * write pattern. - * - * Programmer: Jordan Henderson - * 02/02/2017 + * a dataset has two unlimited dimensions and each + * MPI rank writes to its own separate chunks. On each + * iteration, the dataset is extended in its first + * extensible dimension by the size of one chunk per rank + * and in its second extensible dimension by the size of + * one chunk. Then, all chunks are written to, read back + * and verified. */ static void -test_write_filtered_dataset_interleaved_write(void) +test_write_filtered_dataset_multi_unlim_dim_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t chunk_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t sel_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t start[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t stride[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t count[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t block[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing interleaved write to filtered chunks"); + HDputs("Testing write to unshared filtered chunks w/ two unlimited dimensions"); - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NROWS; - dataset_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; - chunk_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; - chunk_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; - sel_dims[0] = (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / mpi_size); - sel_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; - - filespace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, dataset_dims, NULL); + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NCOLS; + max_dims[0] = H5S_UNLIMITED; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NCOLS; + + filespace = H5Screate_simple(WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, INTERLEAVED_WRITE_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file - */ - count[0] = - (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS); - count[1] = - (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS); - stride[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; - stride[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; - block[0] = 1; - block[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; + for (i = 0; i < (size_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NLOOPS; i++) { + C_DATATYPE *tmp_realloc = NULL; + size_t j; - if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); - HDfflush(stdout); - } + /* Set selected dimensions */ + sel_dims[0] = (i + 1) * WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + sel_dims[1] = (i + 1) * WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + tmp_realloc = (C_DATATYPE *)HDrealloc(data, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + data = tmp_realloc; - /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + tmp_realloc = (C_DATATYPE *)HDrealloc(read_buf, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + read_buf = tmp_realloc; - data = (C_DATATYPE *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + for (j = 0; j < data_size / sizeof(*data); j++) + data[j] = (C_DATATYPE)GEN_DATA(j); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - for (i = 0; i < data_size / sizeof(*data); i++) - data[i] = (C_DATATYPE)GEN_DATA(i); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (i + 1); + count[1] = (i + 1); + stride[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * block[0] * count[0]); + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add Column Index */ - correct_buf[i] = - (C_DATATYPE)((i % (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)) / - (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Add the amount that gets added when a rank moves down to its next section - vertically in the dataset */ - + ((hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS * - (i / (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)))); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + HDmemset(read_buf, 255, data_size); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + /* Verify the correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); + + if (i < (size_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NLOOPS - 1) { + /* + * Extend the dataset by the size of one chunk per rank + * in the first extensible dimension. Extend the dataset + * by the size of chunk in the second extensible dimension. + */ + dataset_dims[0] += (hsize_t)mpi_size * block[0]; + dataset_dims[1] += block[1]; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } if (data) HDfree(data); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - /* Verify the correct data was written */ - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + return; +} - dset_id = H5Dopen2(file_id, "/" INTERLEAVED_WRITE_FILTERED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); +/* + * Tests parallel write of filtered data in the case where + * a dataset has two unlimited dimensions and each MPI + * rank writes to a portion of each chunk in the dataset. + * On each iteration, the dataset is extended in its extensible + * dimensions by the size of a chunk and then all chunks are + * written to by all ranks, then read back and verified. + */ +static void +test_write_filtered_dataset_multi_unlim_dim_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + if (MAINPROCESS) + HDputs("Testing write to shared filtered chunks w/ two unlimited dimensions"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - if (correct_buf) - HDfree(correct_buf); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_NCOLS; + max_dims[0] = H5S_UNLIMITED; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; + + filespace = H5Screate_simple(WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_SHARED_TWO_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + for (i = 0; i < (size_t)WRITE_SHARED_TWO_UNLIM_DIM_NLOOPS; i++) { + C_DATATYPE *tmp_realloc = NULL; + size_t j; + + /* Set selected dimensions */ + sel_dims[0] = (i + 1); + sel_dims[1] = (i + 1) * (size_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + + tmp_realloc = (C_DATATYPE *)HDrealloc(data, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + data = tmp_realloc; + + tmp_realloc = (C_DATATYPE *)HDrealloc(read_buf, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + read_buf = tmp_realloc; + + for (j = 0; j < data_size / sizeof(*data); j++) + data[j] = (C_DATATYPE)GEN_DATA(j); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (i + 1); + count[1] = (i + 1); + stride[0] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_SHARED_TWO_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + HDmemset(read_buf, 255, data_size); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + /* Verify correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); + + if (i < (size_t)WRITE_SHARED_TWO_UNLIM_DIM_NLOOPS - 1) { + /* Extend the dataset by the size of a chunk in each extensible dimension */ + dataset_dims[0] += (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + dataset_dims[1] += (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } + + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of transformed and filtered data - * in the case where only one process is writing to a - * particular chunk in the operation. Normally, a data - * transform function will cause the parallel library to - * break to independent I/O and this isn't allowed when - * there are filters in the pipeline. However, in this - * case the parallel library recognizes that the used - * data transform function "x" is the same as not applying - * the transform function. Therefore it does not apply - * the transform function resulting in not breaking to - * independent I/O. + * Tests parallel write of filtered data in the case where + * a single process in the write operation has no selection + * in the dataset's dataspace. In this case, the process with + * no selection still has to participate in the collective + * space re-allocation for the filtered chunks and also must + * participate in the re-insertion of the filtered chunks + * into the chunk index. * - * Programmer: Jan-Willem Blokland - * 08/20/2021 + * Programmer: Jordan Henderson + * 02/01/2017 */ static void -test_write_transformed_filtered_dataset_no_overlap(void) +test_write_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t segment_length; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared transformed and filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to filtered chunks with a single process having no selection"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - filespace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + sel_dims[0] = sel_dims[1] = 0; + + filespace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / - (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); + count[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / + (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; start[1] = 0; if (VERBOSE_MED) { @@ -1399,38 +1749,43 @@ test_write_transformed_filtered_dataset_no_overlap(void) filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + else + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); /* Fill data buffer */ data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - data = (C_DATATYPE *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + if (mpi_rank != WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) { + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); + + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); + } correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - for (i = 0; i < data_size / sizeof(*data); i++) - data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Create property list for collective dataset write and data transform */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - /* Set data transform expression */ - VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + /* Compute the correct offset into the buffer for the process having no selection and clear it */ + segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; + HDmemset(correct_buf + + ((size_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), + 0, segment_length * sizeof(*data)); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status - data should only have been written if MPI size > 1 */ + verify_space_alloc_status(dset_id, plist_id, (mpi_size > 1 ? SOME_CHUNKS_WRITTEN : NO_CHUNKS_WRITTEN)); + if (data) HDfree(data); @@ -1440,10 +1795,10 @@ test_write_transformed_filtered_dataset_no_overlap(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -1453,131 +1808,93 @@ test_write_transformed_filtered_dataset_no_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case where - * the dataset has 3 dimensions and each process writes - * to its own "page" in the 3rd dimension. + * Tests parallel write of filtered data in the case + * where no process in the write operation has a + * selection in the dataset's dataspace. This test is + * to ensure that there are no assertion failures or + * similar issues due to size 0 allocations and the + * like. In this case, the file and dataset are created + * but the dataset is populated with the default fill + * value. * * Programmer: Jordan Henderson - * 02/06/2017 + * 02/02/2017 */ static void -test_write_3d_filtered_dataset_no_overlap_separate_pages(void) +test_write_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks on separate pages in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to filtered chunks with all processes having no selection"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - sel_dims[2] = 1; + dataset_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = sel_dims[1] = 0; - filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file - */ - count[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / - (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / - (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - count[2] = 1; - stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = 0; - start[1] = 0; - start[2] = (hsize_t)mpi_rank; + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], - start[2], block[0], block[1], block[2]); - HDfflush(stdout); - } + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -1588,18 +1905,12 @@ test_write_3d_filtered_dataset_no_overlap_separate_pages(void) for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status - no ranks should have written any data */ + verify_space_alloc_status(dset_id, plist_id, NO_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -1609,10 +1920,10 @@ test_write_3d_filtered_dataset_no_overlap_separate_pages(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -1622,132 +1933,104 @@ test_write_3d_filtered_dataset_no_overlap_separate_pages(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case where - * the dataset has 3 dimensions and each process writes - * to each "page" in the 3rd dimension. However, no chunk - * on a given "page" is written to by more than one process. + * Tests parallel write of filtered data by using + * point selections instead of hyperslab selections. * * Programmer: Jordan Henderson - * 02/06/2017 + * 02/02/2017 */ static void -test_write_3d_filtered_dataset_no_overlap_same_pages(void) +test_write_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id, dset_id, plist_id; - hid_t filespace, memspace; + C_DATATYPE *read_buf = NULL; + hsize_t * coords = NULL; + hsize_t dataset_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + size_t i, j, data_size, correct_buf_size; + size_t num_points; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks on the same pages in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to filtered chunks with point selection"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; - dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - sel_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + dataset_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; - filespace = - H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file - */ - count[0] = 1; - count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / - (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - count[2] = (hsize_t)mpi_size; - stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); - start[1] = 0; - start[2] = 0; + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], - start[2], block[0], block[1], block[2]); - HDfflush(stdout); - } + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Select hyperslab in the file */ + /* Set up point selection */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + num_points = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS * + (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; + coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); + VRFY((NULL != coords), "Coords HDcalloc succeeded"); + + for (i = 0; i < num_points; i++) + for (j = 0; j < WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) + coords[(i * WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = + (j > 0) ? (i % (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) + : ((hsize_t)mpi_rank + + ((hsize_t)mpi_size * (i / (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); + + VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), + "Point selection succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -1759,18 +2042,16 @@ test_write_3d_filtered_dataset_no_overlap_same_pages(void) data[i] = (C_DATATYPE)GEN_DATA(i); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + - (i / (dataset_dims[0] * dataset_dims[1]))); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf[i] = (C_DATATYPE)( + (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -1780,23 +2061,26 @@ test_write_3d_filtered_dataset_no_overlap_same_pages(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + if (coords) + HDfree(coords); if (correct_buf) HDfree(correct_buf); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -1804,106 +2088,95 @@ test_write_3d_filtered_dataset_no_overlap_same_pages(void) /* * Tests parallel write of filtered data in the case where - * the dataset has 3 dimensions and each process writes - * to each "page" in the 3rd dimension. Further, each chunk - * in each "page" is written to equally by all processes. + * each process writes an equal amount of data to each chunk + * in the dataset. Each chunk is distributed among the + * processes in round-robin fashion by blocks of size 1 until + * the whole chunk is selected, leading to an interleaved + * write pattern. * * Programmer: Jordan Henderson - * 02/06/2017 + * 02/02/2017 */ static void -test_write_3d_filtered_dataset_overlap(void) +test_write_filtered_dataset_interleaved_write(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t dataset_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t chunk_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t sel_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t start[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t stride[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t count[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t block[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to shared filtered chunks in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing interleaved write to filtered chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); - sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; - sel_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + dataset_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NROWS; + dataset_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; + chunk_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; + chunk_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; + sel_dims[0] = (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / mpi_size); + sel_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; - filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, INTERLEAVED_WRITE_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); - count[1] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); - count[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; - stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; - stride[2] = 1; + count[0] = + (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS); + count[1] = + (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS); + stride[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; + stride[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; block[0] = 1; - block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; - block[2] = 1; + block[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; start[0] = (hsize_t)mpi_rank; start[1] = 0; - start[2] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], - start[2], block[0], block[1], block[2]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } @@ -1915,8 +2188,8 @@ test_write_3d_filtered_dataset_overlap(void) "Hyperslab selection succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -1928,30 +2201,25 @@ test_write_3d_filtered_dataset_overlap(void) data[i] = (C_DATATYPE)GEN_DATA(i); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add the Column Index */ - correct_buf[i] = (C_DATATYPE)( - (i % (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * - WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / - (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the amount that gets added when a rank moves down to its next - section vertically in the dataset */ - + ((hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS) * - (i / (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * - WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + /* Add Column Index */ + correct_buf[i] = + (C_DATATYPE)((i % (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)) / + (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + /* Add the amount that gets added when a rank moves down to its next section + vertically in the dataset */ + + ((hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS * + (i / (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)))); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -1961,10 +2229,10 @@ test_write_3d_filtered_dataset_overlap(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, INTERLEAVED_WRITE_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -1974,116 +2242,106 @@ test_write_3d_filtered_dataset_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to unshared - * chunks using a compound datatype which doesn't - * require a datatype conversion. + * Tests parallel write of transformed and filtered data + * in the case where only one process is writing to a + * particular chunk in the operation. Normally, a data + * transform function will cause the parallel library to + * break to independent I/O and this isn't allowed when + * there are filters in the pipeline. However, in this + * case the parallel library recognizes that the used + * data transform function "x" is the same as not applying + * the transform function. Therefore it does not apply + * the transform function resulting in not breaking to + * independent I/O. * - * Programmer: Jordan Henderson - * 02/10/2017 + * Programmer: Jan-Willem Blokland + * 08/20/2021 */ static void -test_write_cmpd_filtered_dataset_no_conversion_unshared(void) +test_write_transformed_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to unshared transformed and filtered chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; - chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = - H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, memtype, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); + count[0] = 1; + count[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / + (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; if (VERBOSE_MED) { HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -2100,37 +2358,32 @@ test_write_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); - VRFY((NULL != data), "HDcalloc succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { - data[i].field1 = (short)GEN_DATA(i); - data[i].field2 = (int)GEN_DATA(i); - data[i].field3 = (long)GEN_DATA(i); - } - - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); - - correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); - } + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Create property list for data transform */ + plist_id = H5Pcopy(dxpl_id); + VRFY((plist_id >= 0), "DXPL copy succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + /* Set data transform expression */ + VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) >= 0), "Dataset write succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + "Dataset write succeeded"); if (data) HDfree(data); @@ -2138,17 +2391,24 @@ test_write_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); /* Verify the correct data was written */ - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, - H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + + /* Verify space allocation status */ + plist_id = H5Dget_create_plist(dset_id); + VRFY((plist_id >= 0), "H5Dget_create_plist succeeded"); + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (correct_buf) HDfree(correct_buf); if (read_buf) @@ -2157,120 +2417,110 @@ test_write_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to shared - * chunks using a compound datatype which doesn't - * require a datatype conversion. + * Tests parallel write of filtered data in the case where + * the dataset has 3 dimensions and each process writes + * to its own "page" in the 3rd dimension. * * Programmer: Jordan Henderson - * 02/10/2017 + * 02/06/2017 */ static void -test_write_cmpd_filtered_dataset_no_conversion_shared(void) +test_write_3d_filtered_dataset_no_overlap_separate_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id, dset_id, plist_id, memtype; - hid_t filespace, memspace; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to shared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to unshared filtered chunks on separate pages in 3D dataset"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + sel_dims[2] = 1; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = - H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, memtype, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; - + count[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / + (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / + (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + count[2] = 1; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + block[2] = 1; + start[0] = 0; + start[1] = 0; + start[2] = (hsize_t)mpi_rank; + if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], + start[2], block[0], block[1], block[2]); HDfflush(stdout); } @@ -2281,43 +2531,27 @@ test_write_cmpd_filtered_dataset_no_conversion_shared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); - VRFY((NULL != data), "HDcalloc succeeded"); - - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { - data[i].field1 = (short)GEN_DATA(i); - data[i].field2 = (int)GEN_DATA(i); - data[i].field3 = (long)GEN_DATA(i); - } + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = - (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf[i].field2 = - (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - correct_buf[i].field3 = - (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - } + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); if (data) HDfree(data); @@ -2325,14 +2559,14 @@ test_write_cmpd_filtered_dataset_no_conversion_shared(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); /* Verify the correct data was written */ - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = - H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -2341,136 +2575,114 @@ test_write_cmpd_filtered_dataset_no_conversion_shared(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to unshared - * chunks using a compound datatype which requires a - * datatype conversion. - * - * NOTE: This test currently should fail because the - * datatype conversion causes the parallel library to - * break to independent I/O and this isn't allowed when - * there are filters in the pipeline. + * Tests parallel write of filtered data in the case where + * the dataset has 3 dimensions and each process writes + * to each "page" in the 3rd dimension. However, no chunk + * on a given "page" is written to by more than one process. * * Programmer: Jordan Henderson - * 02/07/2017 + * 02/06/2017 */ static void -test_write_cmpd_filtered_dataset_type_conversion_unshared(void) +test_write_3d_filtered_dataset_no_overlap_same_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, filetype = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset with Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to unshared filtered chunks on the same pages in 3D dataset"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; - chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + sel_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); + filespace = + H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - sel_dims, NULL); + memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); - - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); + count[0] = 1; + count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / + (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + count[2] = (hsize_t)mpi_size; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + block[2] = 1; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); + start[1] = 0; + start[2] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], + start[2], block[0], block[1], block[2]); HDfflush(stdout); } @@ -2481,50 +2693,43 @@ test_write_cmpd_filtered_dataset_type_conversion_unshared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); - VRFY((NULL != data), "HDcalloc succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { - data[i].field1 = (short)GEN_DATA(i); - data[i].field2 = (int)GEN_DATA(i); - data[i].field3 = (long)GEN_DATA(i); - } + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + + (i / (dataset_dims[0] * dataset_dims[1]))); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Ensure that this test currently fails since type conversions break collective mode */ - H5E_BEGIN_TRY - { - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) < 0), - "Dataset write succeeded"); - } - H5E_END_TRY; + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); if (data) HDfree(data); - /* Verify that no data was written */ VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + /* Verify the correct data was written */ + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -2533,93 +2738,260 @@ test_write_cmpd_filtered_dataset_type_conversion_unshared(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to shared - * chunks using a compound datatype which requires - * a datatype conversion. - * - * NOTE: This test currently should fail because the - * datatype conversion causes the parallel library to - * break to independent I/O and this isn't allowed when - * there are filters in the pipeline. + * Tests parallel write of filtered data in the case where + * the dataset has 3 dimensions and each process writes + * to each "page" in the 3rd dimension. Further, each chunk + * in each "page" is written to equally by all processes. * * Programmer: Jordan Henderson - * 02/10/2017 + * 02/06/2017 */ static void -test_write_cmpd_filtered_dataset_type_conversion_shared(void) +test_write_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id, dset_id, plist_id, filetype, memtype; - hid_t filespace, memspace; - - if (MAINPROCESS) - HDputs( - "Testing write to shared filtered chunks in Compound Datatype dataset with Datatype conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write to shared filtered chunks in 3D dataset"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; + sel_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + + filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); + count[1] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); + count[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + stride[2] = 1; + block[0] = 1; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + block[2] = 1; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + start[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], + start[2], block[0], block[1], block[2]); + HDfflush(stdout); + } + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + /* Add the Column Index */ + correct_buf[i] = (C_DATATYPE)( + (i % (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * + WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / + (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the amount that gets added when a rank moves down to its next + section vertically in the dataset */ + + ((hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS) * + (i / (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * + WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + if (data) + HDfree(data); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + /* Verify the correct data was written */ + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (correct_buf) + HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel write of filtered data to unshared + * chunks using a compound datatype which doesn't + * require a datatype conversion. + * + * Programmer: Jordan Henderson + * 02/10/2017 + */ +static void +test_write_cmpd_filtered_dataset_no_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; + chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); memspace = - H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); /* Create the compound type for memory. */ memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); @@ -2632,32 +3004,26 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), "Datatype insertion succeeded"); - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, + memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); if (VERBOSE_MED) { HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -2675,7 +3041,7 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) "Hyperslab selection succeeded"); data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); VRFY((NULL != data), "HDcalloc succeeded"); correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); @@ -2684,40 +3050,39 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { data[i].field1 = (short)GEN_DATA(i); data[i].field2 = (int)GEN_DATA(i); data[i].field3 = (long)GEN_DATA(i); } - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); - /* Ensure that this test currently fails since type conversions break collective mode */ - H5E_BEGIN_TRY - { - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) < 0), - "Dataset write succeeded"); + correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); } - H5E_END_TRY; + + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); if (data) HDfree(data); - /* Verify that no data was written */ VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + /* Verify the correct data was written */ read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, - H5P_DEFAULT); + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -2726,689 +3091,642 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } -#endif /* - * Tests parallel read of filtered data in the special - * case where a dataset is composed of a single chunk. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the singular chunk and contributes its piece to a - * global buffer that is checked for consistency. + * Tests parallel write of filtered data to shared + * chunks using a compound datatype which doesn't + * require a datatype conversion. * * Programmer: Jordan Henderson - * 05/14/2018 + * 02/10/2017 */ static void -test_read_one_chunk_filtered_dataset(void) +test_write_cmpd_filtered_dataset_no_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t chunk_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t sel_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t start[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t stride[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t count[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t block[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from one-chunk filtered dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS; - dataset_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; - - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); - - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = ((C_DATATYPE)i % (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * - READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)) + - ((C_DATATYPE)i / (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * - READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)); - - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_ONE_CHUNK_FILTERED_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; - - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, READ_ONE_CHUNK_FILTERED_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + HDputs("Testing write to shared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); - dset_id = H5Dcreate2(file_id, READ_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + memspace = + H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dopen2(file_id, "/" READ_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); - sel_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, memtype, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ count[0] = 1; - count[1] = 1; - stride[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; - stride[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; - block[0] = sel_dims[0]; - block[1] = sel_dims[1]; - start[0] = ((hsize_t)mpi_rank * sel_dims[0]); + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + data = (COMPOUND_C_DATATYPE *)HDcalloc( + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + /* Fill data buffer */ + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { + data[i].field1 = (short)GEN_DATA(i); + data[i].field2 = (int)GEN_DATA(i); + data[i].field3 = (long)GEN_DATA(i); + } - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = + (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + correct_buf[i].field2 = + (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + correct_buf[i].field3 = + (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + } - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + if (data) + HDfree(data); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + /* Verify the correct data was written */ + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); if (correct_buf) HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data in the case where only - * one process is reading from a particular chunk in the operation. + * Tests parallel write of filtered data to unshared + * chunks using a compound datatype which requires a + * datatype conversion. * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global buffer - * that is checked for consistency. + * NOTE: This test currently should fail for mpi_size > 1 + * because the datatype conversion causes the parallel + * library to break to independent I/O and this isn't + * allowed when there are filters in the pipeline, + * unless there is only one MPI rank. * * Programmer: Jordan Henderson - * 05/15/2018 + * 02/07/2017 */ static void -test_read_filtered_dataset_no_overlap(void) +test_write_cmpd_filtered_dataset_type_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks"); + HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset with Datatype " + "conversion"); - CHECK_CUR_FILTER_AVAIL(); + /* Skip for MPI communicator size of 1 */ + if (mpi_size == 1) { + SKIPPED(); + return; + } - dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - /* Setup the buffer for writing and for comparison */ - correct_buf_size = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS * - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); - - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + /* Create the dataspace for the dataset */ + dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; + chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + memspace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and reads - * it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS * count[0]); - start[1] = 0; + count[0] = 1; + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + data = (COMPOUND_C_DATATYPE *)HDcalloc( + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + /* Fill data buffer */ + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { + data[i].field1 = (short)GEN_DATA(i); + data[i].field2 = (int)GEN_DATA(i); + data[i].field3 = (long)GEN_DATA(i); + } - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + /* Ensure that this test currently fails since type conversions break collective mode */ + H5E_BEGIN_TRY + { + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) < 0), "Dataset write succeeded"); + } + H5E_END_TRY; - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, NO_CHUNKS_WRITTEN); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + if (data) + HDfree(data); - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + /* Verify that no data was written */ + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); if (correct_buf) HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data in the case where - * more than one process is reading from a particular chunk - * in the operation. + * Tests parallel write of filtered data to shared + * chunks using a compound datatype which requires + * a datatype conversion. * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each chunk of the dataset and contributes its pieces - * to a global buffer that is checked for consistency. + * NOTE: This test currently should fail for mpi_size > 1 + * because the datatype conversion causes the parallel + * library to break to independent I/O and this isn't + * allowed when there are filters in the pipeline, + * unless there is only one MPI rank. * * Programmer: Jordan Henderson - * 05/15/2018 + * 02/10/2017 */ static void -test_read_filtered_dataset_overlap(void) +test_write_cmpd_filtered_dataset_type_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from shared filtered chunks"); + HDputs( + "Testing write to shared filtered chunks in Compound Datatype dataset with Datatype conversion"); - CHECK_CUR_FILTER_AVAIL(); + /* Skip for MPI communicator size of 1 */ + if (mpi_size == 1) { + SKIPPED(); + return; + } - dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS; + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)( - (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + memspace = + H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - dset_id = H5Dcreate2(file_id, READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); - - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); - sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; - sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ - count[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; - count[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; - block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * block[0]; + count[0] = 1; + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); + data = (COMPOUND_C_DATATYPE *)HDcalloc( + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); + VRFY((NULL != data), "HDcalloc succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + /* Fill data buffer */ + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { + data[i].field1 = (short)GEN_DATA(i); + data[i].field2 = (int)GEN_DATA(i); + data[i].field3 = (long)GEN_DATA(i); + } - /* - * Since these chunks are shared, run multiple rounds of MPI_Allgatherv - * to collect all of the pieces into their appropriate locations. The - * number of times MPI_Allgatherv is run should be equal to the number - * of chunks in the first dimension of the dataset. - */ + /* Ensure that this test currently fails since type conversions break collective mode */ + H5E_BEGIN_TRY { - size_t loop_count = count[0]; - size_t total_recvcounts = 0; + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) < 0), "Dataset write succeeded"); + } + H5E_END_TRY; - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, NO_CHUNKS_WRITTEN); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + if (data) + HDfree(data); - for (i = 0; i < (size_t)mpi_size; i++) { - recvcounts[i] = (int)dataset_dims[1]; - total_recvcounts += (size_t)recvcounts[i]; - } + /* Verify that no data was written */ + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * dataset_dims[1]); + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - for (; loop_count; loop_count--) { - VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], - recvcounts[mpi_rank], C_DATATYPE_MPI, - &global_buf[(count[0] - loop_count) * total_recvcounts], - recvcounts, displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); - } - } + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); if (correct_buf) HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } +#endif /* - * Tests parallel read of filtered data in the case where - * a single process in the read operation has no selection - * in the dataset's dataspace. + * Tests parallel read of filtered data in the special + * case where a dataset is composed of a single chunk. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank (except for one) - * reads a part of the dataset and contributes its piece - * to a global buffer that is checked for consistency. + * data to the dataset. Then, each rank reads a part of + * the singular chunk and contributes its piece to a + * global buffer that is checked for consistency. * * Programmer: Jordan Henderson - * 05/15/2018 + * 05/14/2018 */ static void -test_read_filtered_dataset_single_no_selection(void) +test_read_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t chunk_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t sel_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t start[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t stride[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t count[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t block[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; hsize_t flat_dims[1]; size_t i, read_buf_size, correct_buf_size; - size_t segment_length; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from filtered chunks with a single process having no selection"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from one-chunk filtered dataset"); - dataset_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS; + dataset_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); @@ -3417,13 +3735,10 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - - /* Compute the correct offset into the buffer for the process having no selection and clear it */ - segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; - HDmemset(correct_buf + ((size_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), - 0, segment_length * sizeof(*correct_buf)); + correct_buf[i] = ((C_DATATYPE)i % (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * + READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)) + + ((C_DATATYPE)i / (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * + READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -3437,60 +3752,58 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_ONE_CHUNK_FILTERED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_ONE_CHUNK_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - - if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - sel_dims[0] = sel_dims[1] = 0; + sel_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -3506,14 +3819,13 @@ test_read_filtered_dataset_single_no_selection(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = 1; - count[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / - (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; + stride[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; + block[0] = sel_dims[0]; + block[1] = sel_dims[1]; + start[0] = ((hsize_t)mpi_rank * sel_dims[0]); start[1] = 0; if (VERBOSE_MED) { @@ -3524,24 +3836,15 @@ test_read_filtered_dataset_single_no_selection(void) HDfflush(stdout); } - if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); - else - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); @@ -3552,25 +3855,17 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((NULL != recvcounts), "HDcalloc succeeded"); for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * - READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS); - recvcounts[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC] = 0; + recvcounts[i] = (int)flat_dims[0]; displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); VRFY((NULL != displs), "HDcalloc succeeded"); for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * (size_t)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * - READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS)); + displs[i] = (int)(i * flat_dims[0]); - if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, 0, C_DATATYPE_MPI, global_buf, recvcounts, displs, - C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); - else - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, - recvcounts, displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -3588,52 +3883,63 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data in the case where - * no process in the read operation has a selection in the - * dataset's dataspace. This test is to ensure that there - * are no assertion failures or similar issues due to size - * 0 allocations and the like. + * Tests parallel read of filtered data in the case where only + * one process is reading from a particular chunk in the operation. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank will simply issue - * a no-op read. + * data to the dataset. Then, each rank reads a part of + * the dataset and contributes its piece to a global buffer + * that is checked for consistency. * * Programmer: Jordan Henderson * 05/15/2018 */ static void -test_read_filtered_dataset_all_no_selection(void) +test_read_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - size_t read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from filtered chunks with all processes having no selection"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from unshared filtered chunks"); - dataset_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + correct_buf_size = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS * + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); + if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); VRFY((plist_id >= 0), "FAPL creation succeeded"); @@ -3646,79 +3952,129 @@ test_read_filtered_dataset_all_no_selection(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = sel_dims[1] = 0; + sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; - memspace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + /* + * Each process defines the dataset selection in the file and reads + * it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); + read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)flat_dims[0]; + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0]); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); if (read_buf) HDfree(read_buf); if (correct_buf) @@ -3727,50 +4083,52 @@ test_read_filtered_dataset_all_no_selection(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data by using point - * selections instead of hyperslab selections. + * Tests parallel read of filtered data in the case where + * more than one process is reading from a particular chunk + * in the operation. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank will read part - * of the dataset using a point selection and will - * contribute its piece to a global buffer that is - * checked for consistency. + * data to the dataset. Then, each rank reads a part of + * each chunk of the dataset and contributes its pieces + * to a global buffer that is checked for consistency. * * Programmer: Jordan Henderson * 05/15/2018 */ static void -test_read_filtered_dataset_point_selection(void) +test_read_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *correct_buf = NULL; C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t * coords = NULL; - hsize_t dataset_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t flat_dims[1]; - size_t i, j, read_buf_size, correct_buf_size; - size_t num_points; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from filtered chunks with point selection"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from shared filtered chunks"); - dataset_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); @@ -3795,56 +4153,58 @@ test_read_filtered_dataset_point_selection(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -3852,37 +4212,40 @@ test_read_filtered_dataset_point_selection(void) memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Set up point selection */ + /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - num_points = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS * - (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; - coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); - VRFY((NULL != coords), "Coords HDcalloc succeeded"); - - for (i = 0; i < num_points; i++) - for (j = 0; j < READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) - coords[(i * READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = - (j > 0) ? (i % (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) - : ((hsize_t)mpi_rank + - ((hsize_t)mpi_size * (i / (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); - - VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), - "Point selection succeeded"); + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; + count[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); @@ -3895,9 +4258,8 @@ test_read_filtered_dataset_point_selection(void) * of chunks in the first dimension of the dataset. */ { - size_t original_loop_count = dataset_dims[0] / (hsize_t)mpi_size; - size_t cur_loop_count = original_loop_count; - size_t total_recvcounts = 0; + size_t loop_count = count[0]; + size_t total_recvcounts = 0; recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); VRFY((NULL != recvcounts), "HDcalloc succeeded"); @@ -3913,12 +4275,11 @@ test_read_filtered_dataset_point_selection(void) for (i = 0; i < (size_t)mpi_size; i++) displs[i] = (int)(i * dataset_dims[1]); - for (; cur_loop_count; cur_loop_count--) { - VRFY((MPI_SUCCESS == - MPI_Allgatherv(&read_buf[(original_loop_count - cur_loop_count) * dataset_dims[1]], - recvcounts[mpi_rank], C_DATATYPE_MPI, - &global_buf[(original_loop_count - cur_loop_count) * total_recvcounts], - recvcounts, displs, C_DATATYPE_MPI, comm)), + for (; loop_count; loop_count--) { + VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], + recvcounts[mpi_rank], C_DATATYPE_MPI, + &global_buf[(count[0] - loop_count) * total_recvcounts], + recvcounts, displs, C_DATATYPE_MPI, comm)), "MPI_Allgatherv succeeded"); } } @@ -3936,12 +4297,10 @@ test_read_filtered_dataset_point_selection(void) if (correct_buf) HDfree(correct_buf); - HDfree(coords); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -3949,47 +4308,45 @@ test_read_filtered_dataset_point_selection(void) /* * Tests parallel read of filtered data in the case where - * each process reads an equal amount of data from each - * chunk in the dataset. Each chunk is distributed among the - * processes in round-robin fashion by blocks of size 1 until - * the whole chunk is selected, leading to an interleaved - * read pattern. + * a single process in the read operation has no selection + * in the dataset's dataspace. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank will read part - * of each chunk of the dataset and will contribute its - * pieces to a global buffer that is checked for consistency. + * data to the dataset. Then, each rank (except for one) + * reads a part of the dataset and contributes its piece + * to a global buffer that is checked for consistency. * * Programmer: Jordan Henderson * 05/15/2018 */ static void -test_read_filtered_dataset_interleaved_read(void) +test_read_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t chunk_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t sel_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t start[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t stride[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t count[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t block[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t dataset_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t flat_dims[1]; size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t segment_length; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing interleaved read from filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from filtered chunks with a single process having no selection"); - dataset_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NROWS; - dataset_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; + dataset_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); @@ -3998,18 +4355,13 @@ test_read_filtered_dataset_interleaved_read(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add Column Index */ - correct_buf[i] = - (C_DATATYPE)((i % (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) - - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)) / - (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Add the amount that gets added when a rank moves down to its next section - vertically in the dataset */ - + ((hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS * - (i / (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)))); + /* Compute the correct offset into the buffer for the process having no selection and clear it */ + segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; + HDmemset(correct_buf + ((size_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), + 0, segment_length * sizeof(*correct_buf)); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4023,56 +4375,62 @@ test_read_filtered_dataset_interleaved_read(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(INTERLEAVED_READ_FILTERED_DATASET_DIMS, dataset_dims, NULL); + filespace = + H5Screate_simple(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; - chunk_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, INTERLEAVED_READ_FILTERED_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, INTERLEAVED_READ_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" INTERLEAVED_READ_FILTERED_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / mpi_size); - sel_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; + sel_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + sel_dims[0] = sel_dims[1] = 0; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -4088,15 +4446,14 @@ test_read_filtered_dataset_interleaved_read(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = - (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS); - count[1] = - (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NCOLS / INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS); - stride[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; - stride[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; - block[0] = 1; - block[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; + count[0] = 1; + count[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / + (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; start[1] = 0; if (VERBOSE_MED) { @@ -4107,58 +4464,53 @@ test_read_filtered_dataset_interleaved_read(void) HDfflush(stdout); } - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + else + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, NULL) >= 0), + "Dataset read succeeded"); + } + else { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + } global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - /* - * Since these chunks are shared, run multiple rounds of MPI_Allgatherv - * to collect all of the pieces into their appropriate locations. The - * number of times MPI_Allgatherv is run should be equal to the number - * of chunks in the first dimension of the dataset. - */ - { - size_t loop_count = count[0]; - size_t total_recvcounts = 0; - - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * + READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS); + recvcounts[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC] = 0; - for (i = 0; i < (size_t)mpi_size; i++) { - recvcounts[i] = (int)dataset_dims[1]; - total_recvcounts += (size_t)recvcounts[i]; - } + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * dataset_dims[1]); + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * (size_t)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * + READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS)); - for (; loop_count; loop_count--) { - VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], - recvcounts[mpi_rank], C_DATATYPE_MPI, - &global_buf[(count[0] - loop_count) * total_recvcounts], - recvcounts, displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); - } - } + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, 0, C_DATATYPE_MPI, global_buf, recvcounts, displs, + C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + else + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, + recvcounts, displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -4176,7 +4528,7 @@ test_read_filtered_dataset_interleaved_read(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -4184,55 +4536,44 @@ test_read_filtered_dataset_interleaved_read(void) /* * Tests parallel read of filtered data in the case where - * the dataset has 3 dimensions and each process reads from - * its own "page" in the 3rd dimension. + * no process in the read operation has a selection in the + * dataset's dataspace. This test is to ensure that there + * are no assertion failures or similar issues due to size + * 0 allocations and the like. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads its own "page" - * of the dataset and contributes its piece to a global buffer - * that is checked for consistency. + * data to the dataset. Then, each rank will simply issue + * a no-op read. * * Programmer: Jordan Henderson - * 05/16/2018 + * 05/15/2018 */ static void -test_read_3d_filtered_dataset_no_overlap_separate_pages(void) +test_read_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - MPI_Datatype vector_type; - MPI_Datatype resized_vector_type; - C_DATATYPE * read_buf = NULL; - C_DATATYPE * correct_buf = NULL; - C_DATATYPE * global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + size_t read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks on separate pages in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from filtered chunks with all processes having no selection"); - dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; + dataset_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); - if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); VRFY((plist_id >= 0), "FAPL creation succeeded"); @@ -4245,145 +4586,77 @@ test_read_3d_filtered_dataset_no_overlap_separate_pages(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - chunk_dims[2] = 1; + chunk_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY( - (H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - sel_dims[2] = 1; - - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + sel_dims[0] = sel_dims[1] = 0; - memspace = H5Screate_simple(1, flat_dims, NULL); + memspace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory - */ - count[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - count[2] = 1; - stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = 0; - start[1] = 0; - start[2] = (hsize_t)mpi_rank; - - if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); - HDfflush(stdout); - } - - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); - - /* - * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each - * rank to write to the nth position of the global data buffer, where n is the rank number. - */ - VRFY((MPI_SUCCESS == MPI_Type_vector((int)flat_dims[0], 1, mpi_size, C_DATATYPE_MPI, &vector_type)), - "MPI_Type_vector succeeded"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); - - /* - * Resize the type to allow interleaving, - * so make it only one MPI_LONG wide - */ - VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, sizeof(long), &resized_vector_type)), - "MPI_Type_create_resized"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); - - VRFY((MPI_SUCCESS == MPI_Allgather(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, 1, - resized_vector_type, comm)), - "MPI_Allgather succeeded"); - - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - - VRFY((MPI_SUCCESS == MPI_Type_free(&vector_type)), "MPI_Type_free succeeded"); - VRFY((MPI_SUCCESS == MPI_Type_free(&resized_vector_type)), "MPI_Type_free succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (global_buf) - HDfree(global_buf); if (read_buf) HDfree(read_buf); if (correct_buf) @@ -4392,70 +4665,61 @@ test_read_3d_filtered_dataset_no_overlap_separate_pages(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of transformed and filtered data in the - * case where only one process is reading from a particular - * chunk in the operation. Normally, a data transform function - * will cause the parallel library to break to independent I/O - * and this isn't allowed when there are filters in the pipeline. - * However, in this case the parallel library recognizes that - * the used data transform function "x" is the same as not - * applying the transform function. Therefore it does not apply - * the transform function resulting in not breaking to - * independent I/O. + * Tests parallel read of filtered data by using point + * selections instead of hyperslab selections. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global buffer - * that is checked for consistency. + * data to the dataset. Then, each rank will read part + * of the dataset using a point selection and will + * contribute its piece to a global buffer that is + * checked for consistency. * - * Programmer: Jan-Willem Blokland - * 08/20/2021 + * Programmer: Jordan Henderson + * 05/15/2018 */ static void -test_read_transformed_filtered_dataset_no_overlap(void) +test_read_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; + C_DATATYPE *read_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t * coords = NULL; + hsize_t dataset_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t i, j, read_buf_size, correct_buf_size; + size_t num_points; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from unshared transformed and filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from filtered chunks with point selection"); - dataset_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS * - (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); + correct_buf[i] = (C_DATATYPE)( + (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4469,66 +4733,58 @@ test_read_transformed_filtered_dataset_no_overlap(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY( - (H5Pset_chunk(plist_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Set data transform expression */ - VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, correct_buf) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + sel_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -4536,71 +4792,70 @@ test_read_transformed_filtered_dataset_no_overlap(void) memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Select hyperslab in the file */ + /* Set up point selection */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - /* - * Each process defines the dataset selection in the file and reads - * it to the selection in memory - */ - count[0] = 1; - count[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / - (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); - start[1] = 0; - - if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); - HDfflush(stdout); - } - - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read and data transform */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + num_points = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS * + (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; + coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); + VRFY((NULL != coords), "Coords HDcalloc succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + for (i = 0; i < num_points; i++) + for (j = 0; j < READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) + coords[(i * READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = + (j > 0) ? (i % (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) + : ((hsize_t)mpi_rank + + ((hsize_t)mpi_size * (i / (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); - /* Set data transform expression */ - VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), + "Point selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* + * Since these chunks are shared, run multiple rounds of MPI_Allgatherv + * to collect all of the pieces into their appropriate locations. The + * number of times MPI_Allgatherv is run should be equal to the number + * of chunks in the first dimension of the dataset. + */ + { + size_t original_loop_count = dataset_dims[0] / (hsize_t)mpi_size; + size_t cur_loop_count = original_loop_count; + size_t total_recvcounts = 0; - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)dataset_dims[1]; + total_recvcounts += (size_t)recvcounts[i]; + } - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * dataset_dims[1]); + + for (; cur_loop_count; cur_loop_count--) { + VRFY((MPI_SUCCESS == + MPI_Allgatherv(&read_buf[(original_loop_count - cur_loop_count) * dataset_dims[1]], + recvcounts[mpi_rank], C_DATATYPE_MPI, + &global_buf[(original_loop_count - cur_loop_count) * total_recvcounts], + recvcounts, displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + } + } VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -4615,10 +4870,12 @@ test_read_transformed_filtered_dataset_no_overlap(void) if (correct_buf) HDfree(correct_buf); + HDfree(coords); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -4626,56 +4883,67 @@ test_read_transformed_filtered_dataset_no_overlap(void) /* * Tests parallel read of filtered data in the case where - * the dataset has 3 dimensions and each process reads from - * each "page" in the 3rd dimension. However, no chunk on a - * given "page" is read from by more than one process. + * each process reads an equal amount of data from each + * chunk in the dataset. Each chunk is distributed among the + * processes in round-robin fashion by blocks of size 1 until + * the whole chunk is selected, leading to an interleaved + * read pattern. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each "page" of the dataset and contributes its piece to a - * global buffer that is checked for consistency. + * data to the dataset. Then, each rank will read part + * of each chunk of the dataset and will contribute its + * pieces to a global buffer that is checked for consistency. * * Programmer: Jordan Henderson - * 05/16/2018 + * 05/15/2018 */ static void -test_read_3d_filtered_dataset_no_overlap_same_pages(void) +test_read_filtered_dataset_interleaved_read(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id, dset_id, plist_id; - hid_t filespace, memspace; - int * recvcounts = NULL; - int * displs = NULL; - - if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks on the same pages in 3D dataset"); + hsize_t dataset_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t chunk_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t sel_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t start[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t stride[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t count[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t block[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; - CHECK_CUR_FILTER_AVAIL(); + if (MAINPROCESS) + HDputs("Testing interleaved read from filtered chunks"); - dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + dataset_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NROWS; + dataset_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + - (i / (dataset_dims[0] * dataset_dims[1]))); + /* Add Column Index */ + correct_buf[i] = + (C_DATATYPE)((i % (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) + + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)) / + (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) + + /* Add the amount that gets added when a rank moves down to its next section + vertically in the dataset */ + + ((hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS * + (i / (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)))); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4689,63 +4957,61 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(INTERLEAVED_READ_FILTERED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - chunk_dims[2] = 1; + chunk_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; + chunk_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= - 0), + VRFY((H5Pset_chunk(plist_id, INTERLEAVED_READ_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, INTERLEAVED_READ_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, INTERLEAVED_READ_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - sel_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + sel_dims[0] = (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / mpi_size); + sel_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + flat_dims[0] = sel_dims[0] * sel_dims[1]; memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); @@ -4758,19 +5024,16 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = 1; - count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - count[2] = (hsize_t)mpi_size; - stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); + count[0] = + (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS); + count[1] = + (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NCOLS / INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS); + stride[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; + stride[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; start[1] = 0; - start[2] = 0; if (VERBOSE_MED) { HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -4783,39 +5046,49 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* + * Since these chunks are shared, run multiple rounds of MPI_Allgatherv + * to collect all of the pieces into their appropriate locations. The + * number of times MPI_Allgatherv is run should be equal to the number + * of chunks in the first dimension of the dataset. + */ + { + size_t loop_count = count[0]; + size_t total_recvcounts = 0; - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)dataset_dims[1]; + total_recvcounts += (size_t)recvcounts[i]; + } - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * dataset_dims[1]); + + for (; loop_count; loop_count--) { + VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], + recvcounts[mpi_rank], C_DATATYPE_MPI, + &global_buf[(count[0] - loop_count) * total_recvcounts], + recvcounts, displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + } + } VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -4833,7 +5106,7 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -4842,45 +5115,44 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) /* * Tests parallel read of filtered data in the case where * the dataset has 3 dimensions and each process reads from - * each "page" in the 3rd dimension. Further, each chunk in - * each "page" is read from equally by all processes. + * its own "page" in the 3rd dimension. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads part of each - * chunk of each "page" and contributes its pieces to a - * global buffer that is checked for consistency. + * data to the dataset. Then, each rank reads its own "page" + * of the dataset and contributes its piece to a global buffer + * that is checked for consistency. * * Programmer: Jordan Henderson * 05/16/2018 */ static void -test_read_3d_filtered_dataset_overlap(void) +test_read_3d_filtered_dataset_no_overlap_separate_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { MPI_Datatype vector_type; MPI_Datatype resized_vector_type; C_DATATYPE * read_buf = NULL; C_DATATYPE * correct_buf = NULL; C_DATATYPE * global_buf = NULL; - hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t start[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t stride[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t count[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t block[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; hsize_t flat_dims[1]; size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from shared filtered chunks in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from unshared filtered chunks on separate pages in 3D dataset"); - dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NROWS; - dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; - dataset_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); @@ -4889,20 +5161,7 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add the Column Index */ - correct_buf[i] = (C_DATATYPE)( - (i % (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * - READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / - (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the amount that gets added when a rank moves down to its next - section vertically in the dataset */ - + ((hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS) * - (i / (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * - READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4916,58 +5175,62 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + filespace = + H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; chunk_dims[2] = 1; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY( + (H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); - sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; - sel_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + sel_dims[2] = 1; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; @@ -4983,18 +5246,20 @@ test_read_3d_filtered_dataset_overlap(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); - count[1] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); - count[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; - stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + count[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + count[2] = 1; + stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; stride[2] = 1; - block[0] = 1; - block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; block[2] = 1; - start[0] = (hsize_t)mpi_rank; + start[0] = 0; start[1] = 0; - start[2] = 0; + start[2] = (hsize_t)mpi_rank; if (VERBOSE_MED) { HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -5007,51 +5272,36 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - { - size_t run_length = - (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH); - size_t num_blocks = (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + /* + * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each + * rank to write to the nth position of the global data buffer, where n is the rank number. + */ + VRFY((MPI_SUCCESS == MPI_Type_vector((int)flat_dims[0], 1, mpi_size, C_DATATYPE_MPI, &vector_type)), + "MPI_Type_vector succeeded"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); - /* - * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each - * rank to write to the nth position of the global data buffer, where n is the rank number. - */ - VRFY( - (MPI_SUCCESS == MPI_Type_vector((int)num_blocks, (int)run_length, - (int)(mpi_size * (int)run_length), C_DATATYPE_MPI, &vector_type)), - "MPI_Type_vector succeeded"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); - - /* - * Resize the type to allow interleaving, - * so make it "run_length" MPI_LONGs wide - */ - VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, (MPI_Aint)(run_length * sizeof(long)), - &resized_vector_type)), - "MPI_Type_create_resized"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); - } + /* + * Resize the type to allow interleaving, + * so make it only one MPI_LONG wide + */ + VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, sizeof(long), &resized_vector_type)), + "MPI_Type_create_resized"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); VRFY((MPI_SUCCESS == MPI_Allgather(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, 1, resized_vector_type, comm)), - "MPI_Allgatherv succeeded"); + "MPI_Allgather succeeded"); VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -5068,78 +5318,1890 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data to unshared - * chunks using a compound datatype which doesn't - * require a datatype conversion. + * Tests parallel read of transformed and filtered data in the + * case where only one process is reading from a particular + * chunk in the operation. Normally, a data transform function + * will cause the parallel library to break to independent I/O + * and this isn't allowed when there are filters in the pipeline. + * However, in this case the parallel library recognizes that + * the used data transform function "x" is the same as not + * applying the transform function. Therefore it does not apply + * the transform function resulting in not breaking to + * independent I/O. * * The MAINPROCESS rank will first write out all of the * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global - * buffer that is checked for consistency. + * the dataset and contributes its piece to a global buffer + * that is checked for consistency. * - * Programmer: Jordan Henderson - * 05/17/2018 + * Programmer: Jan-Willem Blokland + * 08/20/2021 */ static void -test_read_cmpd_filtered_dataset_no_conversion_unshared(void) +test_read_transformed_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from unshared transformed and filtered chunks"); - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; + dataset_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + correct_buf_size = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS * + (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = + H5Screate_simple(READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY( + (H5Pset_chunk(plist_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Create property list for collective dataset read */ + plist_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((plist_id >= 0), "DXPL creation succeeded"); + + /* Set data transform expression */ + VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, correct_buf) >= 0), + "Dataset write succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + + /* Verify space allocation status */ + plist_id = H5Dget_create_plist(dset_id); + VRFY((plist_id >= 0), "H5Dget_create_plist succeeded"); + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and reads + * it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / + (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + /* Create property list for data transform */ + plist_id = H5Pcopy(dxpl_id); + VRFY((plist_id >= 0), "DXPL copy succeeded"); + + /* Set data transform expression */ + VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + "Dataset read succeeded"); + + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)flat_dims[0]; + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0]); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data in the case where + * the dataset has 3 dimensions and each process reads from + * each "page" in the 3rd dimension. However, no chunk on a + * given "page" is read from by more than one process. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * each "page" of the dataset and contributes its piece to a + * global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/16/2018 + */ +static void +test_read_3d_filtered_dataset_no_overlap_same_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from unshared filtered chunks on the same pages in 3D dataset"); + + dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + + (i / (dataset_dims[0] * dataset_dims[1]))); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = + H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + chunk_dims[2] = 1; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= + 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + sel_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + count[2] = (hsize_t)mpi_size; + stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + block[2] = 1; + start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); + start[1] = 0; + start[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)flat_dims[0]; + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0]); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data in the case where + * the dataset has 3 dimensions and each process reads from + * each "page" in the 3rd dimension. Further, each chunk in + * each "page" is read from equally by all processes. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads part of each + * chunk of each "page" and contributes its pieces to a + * global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/16/2018 + */ +static void +test_read_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) +{ + MPI_Datatype vector_type; + MPI_Datatype resized_vector_type; + C_DATATYPE * read_buf = NULL; + C_DATATYPE * correct_buf = NULL; + C_DATATYPE * global_buf = NULL; + hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t start[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t stride[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t count[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t block[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing read from shared filtered chunks in 3D dataset"); + + dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NROWS; + dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; + dataset_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + /* Add the Column Index */ + correct_buf[i] = (C_DATATYPE)( + (i % (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * + READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / + (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the amount that gets added when a rank moves down to its next + section vertically in the dataset */ + + ((hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS) * + (i / (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * + READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + chunk_dims[2] = 1; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; + sel_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); + count[1] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); + count[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + stride[2] = 1; + block[0] = 1; + block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + block[2] = 1; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + start[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + { + size_t run_length = + (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH); + size_t num_blocks = (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + + /* + * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each + * rank to write to the nth position of the global data buffer, where n is the rank number. + */ + VRFY( + (MPI_SUCCESS == MPI_Type_vector((int)num_blocks, (int)run_length, + (int)(mpi_size * (int)run_length), C_DATATYPE_MPI, &vector_type)), + "MPI_Type_vector succeeded"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); + + /* + * Resize the type to allow interleaving, + * so make it "run_length" MPI_LONGs wide + */ + VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, (MPI_Aint)(run_length * sizeof(long)), + &resized_vector_type)), + "MPI_Type_create_resized"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); + } + + VRFY((MPI_SUCCESS == MPI_Allgather(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, 1, + resized_vector_type, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + VRFY((MPI_SUCCESS == MPI_Type_free(&vector_type)), "MPI_Type_free succeeded"); + VRFY((MPI_SUCCESS == MPI_Type_free(&resized_vector_type)), "MPI_Type_free succeeded"); + + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data to unshared + * chunks using a compound datatype which doesn't + * require a datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * the dataset and contributes its piece to a global + * buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_no_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, + memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data from shared + * chunks using a compound datatype which doesn't + * require a datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * each chunk of the dataset and contributes its piece + * to a global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_no_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from shared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = + (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field2 = + (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field3 = + (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, + memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data from unshared + * chunks using a compound datatype which requires a + * datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * the dataset and contributes its piece to a global + * buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_type_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset with Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data from shared + * chunks using a compound datatype which requires + * a datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * each chunk of the dataset and contributes its pieces + * to a global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_type_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs( + "Testing read from shared filtered chunks in Compound Datatype dataset with Datatype conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = + (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field2 = + (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field3 = + (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests write of filtered data to a dataset + * by a single process. After the write has + * succeeded, the dataset is closed and then + * re-opened in parallel and read by all + * processes to ensure data correctness. + * + * Programmer: Jordan Henderson + * 08/03/2017 + */ +static void +test_write_serial_read_parallel(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write file serially; read file in parallel"); + + dataset_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_DEPTH; + + /* Write the file on the MAINPROCESS rank */ + if (MAINPROCESS) { + /* Set up file access property list */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + chunk_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NCOLS; + chunk_dims[2] = 1; + + filespace = H5Screate_simple(WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + data_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*data); + + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); + + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + if (data) + HDfree(data); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (long)i; + + /* All ranks open the file and verify their "portion" of the dataset is correct */ + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (correct_buf) + HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES +/* + * Tests parallel write of filtered data + * to a dataset. After the write has + * succeeded, the dataset is closed and + * then re-opened and read by a single + * process to ensure data correctness. + * + * Programmer: Jordan Henderson + * 08/03/2017 + */ +static void +test_write_parallel_read_serial(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t sel_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t count[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t stride[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t block[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t offset[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write file in parallel; read serially"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NROWS; + dataset_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; + sel_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; + + filespace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + memspace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = 1; + count[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS / (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + count[2] = (hsize_t)mpi_size; + stride[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + stride[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + block[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + block[2] = 1; + offset[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS * count[0]); + offset[1] = 0; + offset[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], offset[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], offset[0], + offset[1], offset[2], block[0], block[1], block[2]); + HDfflush(stdout); + } + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); - } + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + if (data) + HDfree(data); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -5153,783 +7215,794 @@ test_read_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - /* Create chunked dataset */ - chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + dset_id = H5Dopen2(group_id, WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, - memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + + (i / (dataset_dims[0] * dataset_dims[1]))); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + HDfree(correct_buf); + HDfree(read_buf); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + return; +} - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); +/* + * Tests that causing chunks to continually grow and shrink + * by writing random data followed by zeroed-out data (and + * thus controlling the compression ratio) does not cause + * problems. + * + * Programmer: Jordan Henderson + * 06/04/2018 + */ +static void +test_shrinking_growing_chunks(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) +{ + double *data = NULL; + double *read_buf = NULL; + hsize_t dataset_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t start[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t stride[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t count[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t block[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + if (MAINPROCESS) + HDputs("Testing continually shrinking/growing chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - - dset_id = - H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + filespace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); + memspace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, SHRINKING_GROWING_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, SHRINKING_GROWING_CHUNKS_DATASET_NAME, H5T_NATIVE_DOUBLE, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + * Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); + count[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS / (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + block[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((dset_id >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + data_size = sel_dims[0] * sel_dims[1] * sizeof(double); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + data = (double *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + read_buf = (double *)HDcalloc(1, data_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); - - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + for (i = 0; i < SHRINKING_GROWING_CHUNKS_NLOOPS; i++) { + /* Continually write random float data, followed by zeroed-out data */ + if (i % 2) + HDmemset(data, 0, data_size); + else { + size_t j; + for (j = 0; j < data_size / sizeof(*data); j++) { + data[j] = (rand() / (double)(RAND_MAX / (double)1.0L)); + } + } - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + if (i % 2) { + HDmemset(read_buf, 255, data_size); + } + else { + HDmemset(read_buf, 0, data_size); + } - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "data verification succeeded"); + } - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); if (read_buf) HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); + if (data) + HDfree(data); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data from shared - * chunks using a compound datatype which doesn't - * require a datatype conversion. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each chunk of the dataset and contributes its piece - * to a global buffer that is checked for consistency. + * Tests that filtered and unfiltered partial edge chunks can be + * written to and read from correctly in parallel when only one MPI + * rank writes to a particular partial edge chunk in the dataset. * - * Programmer: Jordan Henderson - * 05/17/2018 + * The dataset contains partial edge chunks in the second dimension. + * Each MPI rank selects a hyperslab in the shape of a single chunk + * that is offset to cover the whole edge chunk and part of the + * full chunk next to the edge chunk. */ static void -test_read_cmpd_filtered_dataset_no_conversion_shared(void) +test_edge_chunks_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id, dset_id, plist_id, memtype; - hid_t filespace, memspace; - int * recvcounts = NULL; - int * displs = NULL; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from shared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; - - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + HDputs("Testing write to unshared filtered edge chunks"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = - (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf[i].field2 = - (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - correct_buf[i].field3 = - (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - } + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + + filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + start[1] = + (hsize_t)(WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, - memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, (mpi_size > 1) ? SOME_CHUNKS_WRITTEN : ALL_CHUNKS_WRITTEN); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - dset_id = - H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + /* Repeat the previous, but set option to not filter partial edge chunks */ + if (MAINPROCESS) + HDputs("Testing write to unshared unfiltered edge chunks"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + H5Pset_chunk_opts(plist_id, H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; + count[1] = 1; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + start[1] = + (hsize_t)(WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); - - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); - - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, (mpi_size > 1) ? SOME_CHUNKS_WRITTEN : ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + HDmemset(read_buf, 255, data_size); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data from unshared - * chunks using a compound datatype which requires a - * datatype conversion. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global - * buffer that is checked for consistency. + * Tests that filtered and unfiltered partial edge chunks can be + * written to and read from correctly in parallel when every MPI + * rank writes to every partial edge chunk in the dataset. * - * Programmer: Jordan Henderson - * 05/17/2018 + * The dataset contains partial edge chunks in the second dimension. + * Each MPI rank selects a hyperslab in the shape of one row of each + * chunk that is offset in the second dimension to cover the whole + * edge chunk and part of the full chunk next to the edge chunk. */ static void -test_read_cmpd_filtered_dataset_type_conversion_unshared(void) +test_edge_chunks_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, filetype = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset with Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; - - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + HDputs("Testing write to shared filtered edge chunks"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); - } + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + filespace = H5Screate_simple(WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS / WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + count[1] = 1; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)1; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - /* Create chunked dataset */ - chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Repeat the previous, but set option to not filter partial edge chunks */ + if (MAINPROCESS) + HDputs("Testing write to shared unfiltered edge chunks"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + H5Pset_chunk_opts(plist_id, H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); + count[0] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS / WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + count[1] = 1; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)1; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); - - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); - - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + HDmemset(read_buf, 255, data_size); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data from shared - * chunks using a compound datatype which requires - * a datatype conversion. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each chunk of the dataset and contributes its pieces - * to a global buffer that is checked for consistency. + * Tests that filtered and unfiltered partial edge chunks can be + * written to and read from correctly in parallel when only one + * MPI rank writes to a particular edge chunk in the dataset and + * only performs a partial write to the edge chunk. * - * Programmer: Jordan Henderson - * 05/17/2018 + * The dataset contains partial edge chunks in the second dimension. + * Each MPI rank selects a hyperslab in the shape of part of a single + * edge chunk and writes to just a portion of the edge chunk. */ static void -test_read_cmpd_filtered_dataset_type_conversion_shared(void) +test_edge_chunks_partial_write(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id, dset_id, plist_id, filetype, memtype; - hid_t filespace, memspace; - int * recvcounts = NULL; - int * displs = NULL; + /* TODO */ +} + +/* + * Tests that the parallel compression feature correctly handles + * writing fill values to a dataset and reading fill values from + * unallocated parts of a dataset. + */ +static void +test_fill_values(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + C_DATATYPE fill_value; + hsize_t dataset_dims[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t chunk_dims[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t sel_dims[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t start[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t stride[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t count[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t block[FILL_VALUES_TEST_DATASET_DIMS]; + size_t i, data_size, read_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs( - "Testing read from shared filtered chunks in Compound Datatype dataset with Datatype conversion"); + HDputs("Testing fill values"); - CHECK_CUR_FILTER_AVAIL(); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)FILL_VALUES_TEST_NROWS; + dataset_dims[1] = (hsize_t)FILL_VALUES_TEST_NCOLS; + chunk_dims[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + chunk_dims[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + filespace = H5Screate_simple(FILL_VALUES_TEST_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = - (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - correct_buf[i].field2 = - (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + VRFY((H5Pset_chunk(plist_id, FILL_VALUES_TEST_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); - correct_buf[i].field3 = - (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - } + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + /* Set a fill value */ + fill_value = FILL_VALUES_TEST_FILL_VAL; + VRFY((H5Pset_fill_value(plist_id, HDF5_DATATYPE_NAME, &fill_value) >= 0), "Fill Value set"); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + dset_id = H5Dcreate2(group_id, FILL_VALUES_TEST_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, + plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Allocate buffer for reading entire dataset */ + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + read_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + correct_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Read entire dataset and verify that the fill value is returned */ + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + correct_buf[i] = FILL_VALUES_TEST_FILL_VAL; - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset and ensure that + * the fill value is returned for the unwritten part of + * the chunk, as well as for the rest of the dataset that + * hasn't been written to yet. + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_VALUES_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; + start[1] = 0; - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - dset_id = - H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* + * Each MPI rank communicates their written piece of data + * into each other rank's correctness-checking buffer + */ + recvcounts = HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + displs = HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)(count[1] * block[1]); + displs[i] = (int)(i * dataset_dims[1]); + } - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + VRFY((MPI_SUCCESS == MPI_Allgatherv(data, recvcounts[mpi_rank], C_DATATYPE_MPI, correct_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + * Write to whole dataset and ensure fill value isn't returned + * after reading whole dataset back */ - count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_VALUES_TEST_NROWS / (hsize_t)FILL_VALUES_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_VALUES_TEST_NCOLS / (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); @@ -5939,304 +8012,290 @@ test_read_cmpd_filtered_dataset_type_conversion_shared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); - - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); - - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); - - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); - - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + VRFY((read_buf[i] != FILL_VALUES_TEST_FILL_VAL), "Data verification succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - - return; -} - -/* - * Tests write of filtered data to a dataset - * by a single process. After the write has - * succeeded, the dataset is closed and then - * re-opened in parallel and read by all - * processes to ensure data correctness. - * - * Programmer: Jordan Henderson - * 08/03/2017 - */ -static void -test_write_serial_read_parallel(void) -{ - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1; - if (MAINPROCESS) - HDputs("Testing write file serially; read file in parallel"); + /******************************************************************** + * Set the fill time to H5D_FILL_TIME_ALLOC and repeat the previous * + ********************************************************************/ - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_DEPTH; + VRFY((H5Pset_fill_time(plist_id, H5D_FILL_TIME_ALLOC) >= 0), "H5Pset_fill_time succeeded"); - /* Write the file on the MAINPROCESS rank */ - if (MAINPROCESS) { - /* Set up file access property list */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + dset_id = H5Dcreate2(group_id, FILL_VALUES_TEST_DATASET_NAME2, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, + plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Read entire dataset and verify that the fill value is returned */ + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - /* Create the dataspace for the dataset */ - chunk_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NCOLS; - chunk_dims[2] = 1; + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + correct_buf[i] = FILL_VALUES_TEST_FILL_VAL; - filespace = H5Screate_simple(WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset and ensure that + * the fill value is returned for the unwritten part of + * the chunk, as well as for the rest of the dataset that + * hasn't been written to yet. + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_VALUES_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; + start[1] = 0; - VRFY((H5Pset_chunk(plist_id, WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - dset_id = H5Dcreate2(file_id, WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - data_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*data); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - data = (C_DATATYPE *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - for (i = 0; i < data_size / sizeof(*data); i++) - data[i] = (C_DATATYPE)GEN_DATA(i); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) >= 0), - "Dataset write succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME2, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - if (data) - HDfree(data); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)(count[1] * block[1]); + displs[i] = (int)(i * dataset_dims[1]); } - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + /* + * Each MPI rank communicates their written piece of data + * into each other rank's correctness-checking buffer + */ + VRFY((MPI_SUCCESS == MPI_Allgatherv(data, recvcounts[mpi_rank], C_DATATYPE_MPI, correct_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + /* + * Write to whole dataset and ensure fill value isn't returned + * after reading whole dataset back + */ - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (long)i; + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_VALUES_TEST_NROWS / (hsize_t)FILL_VALUES_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_VALUES_TEST_NCOLS / (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; - /* All ranks open the file and verify their "portion" of the dataset is correct */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, H5P_DEFAULT); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME2, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + VRFY((read_buf[i] != FILL_VALUES_TEST_FILL_VAL), "Data verification succeeded"); - if (correct_buf) - HDfree(correct_buf); + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } -#if MPI_VERSION >= 3 /* - * Tests parallel write of filtered data - * to a dataset. After the write has - * succeeded, the dataset is closed and - * then re-opened and read by a single - * process to ensure data correctness. - * - * Programmer: Jordan Henderson - * 08/03/2017 + * Tests that the parallel compression feature can handle + * an undefined fill value. Nothing is verified in this + * test since the fill value isn't defined. */ static void -test_write_parallel_read_serial(void) +test_fill_value_undefined(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t sel_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t count[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t stride[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t block[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t offset[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + H5D_alloc_time_t alloc_time; + C_DATATYPE * data = NULL; + C_DATATYPE * read_buf = NULL; + hsize_t dataset_dims[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t chunk_dims[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t sel_dims[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t start[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t stride[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t count[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t block[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + size_t i, data_size, read_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write file in parallel; read serially"); + HDputs("Testing undefined fill value"); - CHECK_CUR_FILTER_AVAIL(); + VRFY((H5Pget_alloc_time(dcpl_id, &alloc_time) >= 0), "H5Pget_alloc_time succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NROWS; - dataset_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; - sel_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; + dataset_dims[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NROWS; + dataset_dims[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NCOLS; + chunk_dims[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + chunk_dims[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + /* Set an undefined fill value */ + VRFY((H5Pset_fill_value(plist_id, HDF5_DATATYPE_NAME, NULL) >= 0), "Fill Value set"); + + dset_id = H5Dcreate2(group_id, FILL_VALUE_UNDEFINED_TEST_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file + /* Allocate buffer for reading entire dataset */ + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); + + read_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + /* + * Read entire dataset - nothing to verify since there's no fill value. + * If not using early space allocation, the read should fail since storage + * isn't allocated yet and no fill value is defined. */ - count[0] = 1; - count[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS / (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - count[2] = (hsize_t)mpi_size; - stride[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - stride[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - block[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - block[2] = 1; - offset[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS * count[0]); - offset[1] = 0; - offset[2] = 0; + if (alloc_time == H5D_ALLOC_TIME_EARLY) { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + } + else { + H5E_BEGIN_TRY + { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) < 0), + "Dataset read succeeded"); + } + H5E_END_TRY; + } + + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset. Don't verify + * anything since there's no fill value defined. + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_VALUE_UNDEFINED_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; + start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], offset[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], offset[0], - offset[1], offset[2], block[0], block[1], block[2]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } @@ -6244,11 +8303,11 @@ test_write_parallel_read_serial(void) filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, stride, count, block) >= 0), + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -6256,150 +8315,204 @@ test_write_parallel_read_serial(void) for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); - if (data) - HDfree(data); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + dset_id = H5Dopen2(group_id, FILL_VALUE_UNDEFINED_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* + * Write to whole dataset and ensure data is correct + * after reading whole dataset back + */ - dset_id = H5Dopen2(file_id, "/" WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NROWS / (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NCOLS / (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + - (i / (dataset_dims[0] * dataset_dims[1]))); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, read_buf) >= 0), - "Dataset read succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUE_UNDEFINED_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - HDfree(correct_buf); + if (data) + HDfree(data); + if (read_buf) HDfree(read_buf); - } + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests that causing chunks to continually grow and shrink - * by writing random data followed by zeroed-out data (and - * thus controlling the compression ratio) does not cause - * problems. - * - * Programmer: Jordan Henderson - * 06/04/2018 + * Tests that the parallel compression feature correctly handles + * avoiding writing fill values to a dataset when the fill time + * is set as H5D_FILL_TIME_NEVER. */ static void -test_shrinking_growing_chunks(void) +test_fill_time_never(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - double *data = NULL; - hsize_t dataset_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t start[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t stride[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t count[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t block[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - size_t i, data_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *fill_buf = NULL; + C_DATATYPE fill_value; + hsize_t dataset_dims[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t chunk_dims[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t sel_dims[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t start[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t stride[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t count[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t block[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + size_t i, data_size, read_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs("Testing continually shrinking/growing chunks"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing fill time H5D_FILL_TIME_NEVER"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* + * Only run this test when incremental file space allocation is + * used, as HDF5's chunk allocation code always writes fill values + * when filters are in the pipeline, but parallel compression does + * incremental file space allocation differently. + */ + { + H5D_alloc_time_t alloc_time; - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pget_alloc_time(dcpl_id, &alloc_time) >= 0), "H5Pget_alloc_time succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + if (alloc_time != H5D_ALLOC_TIME_INCR) { + if (MAINPROCESS) + SKIPPED(); + return; + } + } - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)FILL_TIME_NEVER_TEST_NROWS; + dataset_dims[1] = (hsize_t)FILL_TIME_NEVER_TEST_NCOLS; + chunk_dims[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + chunk_dims[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(FILL_TIME_NEVER_TEST_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, SHRINKING_GROWING_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, FILL_TIME_NEVER_TEST_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + /* Set a fill value */ + fill_value = FILL_VALUES_TEST_FILL_VAL; + VRFY((H5Pset_fill_value(plist_id, HDF5_DATATYPE_NAME, &fill_value) >= 0), "Fill Value set"); - dset_id = H5Dcreate2(file_id, SHRINKING_GROWING_CHUNKS_DATASET_NAME, H5T_NATIVE_DOUBLE, filespace, + /* Set fill time of 'never' */ + VRFY((H5Pset_fill_time(plist_id, H5D_FILL_TIME_NEVER) >= 0), "H5Pset_fill_time succeeded"); + + dset_id = H5Dcreate2(group_id, FILL_TIME_NEVER_TEST_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + /* Allocate buffer for reading entire dataset */ + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); + + read_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + fill_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != fill_buf), "HDcalloc succeeded"); + + /* Read entire dataset and verify that the fill value isn't returned */ + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + fill_buf[i] = FILL_TIME_NEVER_TEST_FILL_VAL; + /* - * Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file + * It should be very unlikely for the dataset's random + * values to all be the fill value, so this should be + * a safe comparison in theory. + */ + VRFY((0 != HDmemcmp(read_buf, fill_buf, read_buf_size)), "Data verification succeeded"); + + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset and ensure that + * the fill value isn't returned for the unwritten part of + * the chunk, as well as for the rest of the dataset that + * hasn't been written to yet. */ count[0] = 1; - count[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS / (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - block[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS * count[0]); + count[1] = 1; + stride[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_TIME_NEVER_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; start[1] = 0; if (VERBOSE_MED) { @@ -6412,44 +8525,122 @@ test_shrinking_growing_chunks(void) /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); - VRFY((dset_id >= 0), "File dataspace retrieval succeeded"); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - data_size = sel_dims[0] * sel_dims[1] * sizeof(double); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - data = (double *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - for (i = 0; i < SHRINKING_GROWING_CHUNKS_NLOOPS; i++) { - /* Continually write random float data, followed by zeroed-out data */ - if ((i % 2)) - HDmemset(data, 0, data_size); - else { - size_t j; - for (j = 0; j < data_size / sizeof(*data); j++) { - data[j] = (float)(rand() / (double)(RAND_MAX / (double)1.0L)); - } - } + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - VRFY((H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_TIME_NEVER_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + /* + * Each MPI rank communicates their written piece of data + * into each other rank's correctness-checking buffer + */ + recvcounts = HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + displs = HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)(count[1] * block[1]); + displs[i] = (int)(i * dataset_dims[1]); + } + + VRFY((MPI_SUCCESS == MPI_Allgatherv(data, recvcounts[mpi_rank], C_DATATYPE_MPI, fill_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + /* + * It should be very unlikely for the dataset's random + * values to all be the fill value, so this should be + * a safe comparison in theory. + */ + VRFY((0 != HDmemcmp(read_buf, fill_buf, read_buf_size)), "Data verification succeeded"); + + /* + * Write to whole dataset and ensure fill value isn't returned + * after reading whole dataset back + */ + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_TIME_NEVER_TEST_NROWS / (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_TIME_NEVER_TEST_NCOLS / (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); } + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_TIME_NEVER_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + VRFY((read_buf[i] != FILL_TIME_NEVER_TEST_FILL_VAL), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); if (data) HDfree(data); + if (read_buf) + HDfree(read_buf); + if (fill_buf) + HDfree(fill_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -6459,8 +8650,14 @@ test_shrinking_growing_chunks(void) int main(int argc, char **argv) { - size_t i; - hid_t file_id = -1, fapl = -1; + size_t cur_filter_idx = 0; + size_t num_filters = 0; + hid_t file_id = H5I_INVALID_HID; + hid_t fcpl_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; int mpi_code; /* Initialize MPI */ @@ -6487,7 +8684,7 @@ main(int argc, char **argv) if (MAINPROCESS) { HDprintf("==========================\n"); - HDprintf("Parallel Filters tests\n"); + HDprintf(" Parallel Filters tests\n"); HDprintf("==========================\n\n"); } @@ -6496,72 +8693,161 @@ main(int argc, char **argv) TestAlarmOn(); - /* Create test file */ - fapl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((fapl >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(fapl, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - VRFY((h5_fixname(FILENAME[0], fapl, filenames[0], sizeof(filenames[0])) != NULL), - "Test file name created"); + num_filters = ARRAY_SIZE(filterIDs); - file_id = H5Fcreate(filenames[0], H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - VRFY((file_id >= 0), "Test file creation succeeded"); + /* Set up file access property list with parallel I/O access, + * collective metadata reads/writes and the latest library + * version bounds */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((fapl_id >= 0), "FAPL creation succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + VRFY((H5Pset_fapl_mpio(fapl_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pset_all_coll_metadata_ops(fapl_id, TRUE) >= 0), "H5Pset_all_coll_metadata_ops succeeded"); + VRFY((H5Pset_coll_metadata_write(fapl_id, TRUE) >= 0), "H5Pset_coll_metadata_write succeeded"); - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (MPI_SUCCESS == (mpi_code = MPI_Barrier(comm))) { - (*tests[i])(); - } - else { - if (MAINPROCESS) - MESG("MPI_Barrier failed"); - nerrors++; - } - } + VRFY((H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); /* - * Increment the filter index to switch to the checksum filter - * and re-run the tests. + * Set up Paged and Persistent Free Space Management */ - cur_filter_idx++; - - h5_clean_files(FILENAME, fapl); - - fapl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((fapl >= 0), "FAPL creation succeeded"); + fcpl_id = H5Pcreate(H5P_FILE_CREATE); + VRFY((fcpl_id >= 0), "FCPL creation succeeded"); - VRFY((H5Pset_fapl_mpio(fapl, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, TRUE, 1) >= 0), + "H5Pset_file_space_strategy succeeded"); - VRFY((H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((h5_fixname(FILENAME[0], fapl_id, filenames[0], sizeof(filenames[0])) != NULL), + "Test file name created"); - file_id = H5Fcreate(filenames[0], H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + file_id = H5Fcreate(filenames[0], H5F_ACC_TRUNC, fcpl_id, fapl_id); VRFY((file_id >= 0), "Test file creation succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + file_id = H5I_INVALID_HID; - if (MAINPROCESS) { - HDprintf("\n=================================================================\n"); - HDprintf("Re-running Parallel Filters tests with Fletcher32 checksum filter\n"); - HDprintf("=================================================================\n\n"); - } - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (MPI_SUCCESS == (mpi_code = MPI_Barrier(comm))) { - (*tests[i])(); - } - else { - if (MAINPROCESS) - MESG("MPI_Barrier failed"); - nerrors++; + /* Create property list for collective dataset write */ + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + VRFY((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) >= 0), "H5Pset_dxpl_mpio succeeded"); + + /* Create DCPL for dataset creation */ + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + VRFY((dcpl_id >= 0), "DCPL creation succeeded"); + + /* Run tests with all available filters */ + for (cur_filter_idx = 0; cur_filter_idx < num_filters; cur_filter_idx++) { + H5FD_mpio_chunk_opt_t chunk_opt; + H5Z_filter_t cur_filter = filterIDs[cur_filter_idx]; + + /* Run tests with both linked-chunk and multi-chunk I/O */ + for (chunk_opt = H5FD_MPIO_CHUNK_ONE_IO; chunk_opt <= H5FD_MPIO_CHUNK_MULTI_IO; chunk_opt++) { + H5D_alloc_time_t space_alloc_time; + + /* Run tests with all available space allocation times */ + for (space_alloc_time = H5D_ALLOC_TIME_EARLY; space_alloc_time <= H5D_ALLOC_TIME_INCR; + space_alloc_time++) { + const char *alloc_time; + unsigned filter_config; + htri_t filter_avail; + size_t i; + char group_name[512]; + + switch (space_alloc_time) { + case H5D_ALLOC_TIME_EARLY: + alloc_time = "Early"; + break; + case H5D_ALLOC_TIME_LATE: + alloc_time = "Late"; + break; + case H5D_ALLOC_TIME_INCR: + alloc_time = "Incremental"; + break; + default: + alloc_time = "Unknown"; + } + + if (MAINPROCESS) + HDprintf("== Running tests with filter '%s' using '%s' and '%s' allocation time ==\n\n", + filterNames[cur_filter_idx], + H5FD_MPIO_CHUNK_ONE_IO == chunk_opt ? "Linked-Chunk I/O" : "Multi-Chunk I/O", + alloc_time); + + /* Make sure current filter is available before testing with it */ + filter_avail = H5Zfilter_avail(cur_filter); + VRFY((filter_avail >= 0), "H5Zfilter_avail succeeded"); + + if (!filter_avail) { + if (MAINPROCESS) + HDprintf(" ** SKIPPED tests with filter '%s' - filter unavailable **\n\n", + filterNames[cur_filter_idx]); + continue; + } + + /* Get the current filter's info */ + VRFY((H5Zget_filter_info(cur_filter, &filter_config) >= 0), "H5Zget_filter_info succeeded"); + + /* Determine if filter is encode-enabled */ + if (0 == (filter_config & H5Z_FILTER_CONFIG_ENCODE_ENABLED)) { + if (MAINPROCESS) + HDprintf(" ** SKIPPED tests with filter '%s' - filter not encode-enabled **\n\n", + filterNames[cur_filter_idx]); + continue; + } + + /* Set space allocation time */ + VRFY((H5Pset_alloc_time(dcpl_id, space_alloc_time) >= 0), "H5Pset_alloc_time succeeded"); + + /* Set chunk I/O optimization method */ + VRFY((H5Pset_dxpl_mpio_chunk_opt(dxpl_id, chunk_opt) >= 0), + "H5Pset_dxpl_mpio_chunk_opt succeeded"); + + /* Create a group to hold all the datasets for this combination + * of filter and chunk optimization mode. Then, close the file + * again since some tests may need to open the file in a special + * way, like on rank 0 only */ + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "H5Fopen succeeded"); + + HDsnprintf(group_name, sizeof(group_name), "%s_%s_%s", filterNames[cur_filter_idx], + H5FD_MPIO_CHUNK_ONE_IO == chunk_opt ? "linked-chunk-io" : "multi-chunk-io", + alloc_time); + + group_id = H5Gcreate2(file_id, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gcreate2 succeeded"); + + VRFY((H5Gclose(group_id) >= 0), "H5Gclose failed"); + group_id = H5I_INVALID_HID; + + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + file_id = H5I_INVALID_HID; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + test_func func = tests[i]; + + if (MPI_SUCCESS == (mpi_code = MPI_Barrier(comm))) { + func(group_name, cur_filter, fapl_id, dcpl_id, dxpl_id); + } + else { + if (MAINPROCESS) + MESG("MPI_Barrier failed"); + nerrors++; + } + } + + if (MAINPROCESS) + HDputs(""); + } } } + VRFY((H5Pclose(dcpl_id) >= 0), "DCPL close succeeded"); + dcpl_id = H5I_INVALID_HID; + + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); + dxpl_id = H5I_INVALID_HID; + if (nerrors) goto exit; @@ -6575,7 +8861,21 @@ exit: TestAlarmOff(); - h5_clean_files(FILENAME, fapl); + h5_clean_files(FILENAME, fapl_id); + fapl_id = H5I_INVALID_HID; + + if (dcpl_id >= 0) + VRFY((H5Pclose(dcpl_id) >= 0), "H5Pclose succeeded"); + if (dxpl_id >= 0) + VRFY((H5Pclose(dxpl_id) >= 0), "H5Pclose succeeded"); + if (fapl_id >= 0) + VRFY((H5Pclose(fapl_id) >= 0), "H5Pclose succeeded"); + if (fcpl_id >= 0) + VRFY((H5Pclose(fcpl_id) >= 0), "H5Pclose succeeded"); + if (group_id >= 0) + VRFY((H5Gclose(group_id) >= 0), "H5Gclose succeeded"); + if (file_id >= 0) + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); H5close(); diff --git a/testpar/t_filters_parallel.h b/testpar/t_filters_parallel.h index 7eb34ed..800604c 100644 --- a/testpar/t_filters_parallel.h +++ b/testpar/t_filters_parallel.h @@ -30,23 +30,23 @@ #include "stdlib.h" #include "testpar.h" +#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0]) + /* Used to load other filters than GZIP */ /* #define DYNAMIC_FILTER */ /* Uncomment and define the fields below to use a dynamically loaded filter */ + +#ifdef DYNAMIC_FILTER #define FILTER_NUM_CDVALUES 1 const unsigned int cd_values[FILTER_NUM_CDVALUES] = {0}; -H5Z_filter_t filter_id; -unsigned int flags = 0; -size_t cd_nelmts = FILTER_NUM_CDVALUES; - -/* Utility Macros */ -#define STRINGIFY(type) #type +unsigned int flags = 0; +size_t cd_nelmts = FILTER_NUM_CDVALUES; +#endif /* Common defines for all tests */ -#define C_DATATYPE long -#define C_DATATYPE_MPI MPI_LONG -#define COMPOUND_C_DATATYPE cmpd_filtered_t -#define C_DATATYPE_STR(type) STRINGIFY(type) -#define HDF5_DATATYPE_NAME H5T_NATIVE_LONG +#define C_DATATYPE long +#define C_DATATYPE_MPI MPI_LONG +#define COMPOUND_C_DATATYPE cmpd_filtered_t +#define HDF5_DATATYPE_NAME H5T_NATIVE_LONG /* Macro used to generate data for datasets for later verification */ #define GEN_DATA(i) INCREMENTAL_DATA(i) @@ -59,7 +59,7 @@ size_t cd_nelmts = FILTER_NUM_CDVALUES; #define RANK_DATA(i) \ (mpi_rank) /* Generates test data to visibly show which rank wrote to which parts of the dataset */ -#define DEFAULT_DEFLATE_LEVEL 6 +#define DEFAULT_DEFLATE_LEVEL 9 #define DIM0_SCALE_FACTOR 4 #define DIM1_SCALE_FACTOR 2 @@ -89,6 +89,14 @@ typedef struct { #define WRITE_UNSHARED_FILTERED_CHUNKS_CH_NROWS (WRITE_UNSHARED_FILTERED_CHUNKS_NROWS / mpi_size) #define WRITE_UNSHARED_FILTERED_CHUNKS_CH_NCOLS (WRITE_UNSHARED_FILTERED_CHUNKS_NCOLS / mpi_size) +/* Defines for the unshared filtered chunks partial write test */ +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_NAME "unshared_filtered_chunks_partial_write" +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS 2 +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS (mpi_size * DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS (DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS (DIM1_SCALE_FACTOR) + /* Defines for the shared filtered chunks write test */ #define WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME "shared_filtered_chunks_write" #define WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS 2 @@ -97,6 +105,42 @@ typedef struct { #define WRITE_SHARED_FILTERED_CHUNKS_NROWS (WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS * DIM0_SCALE_FACTOR) #define WRITE_SHARED_FILTERED_CHUNKS_NCOLS (WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS * DIM1_SCALE_FACTOR) +/* Defines for the unshared filtered chunks w/ single unlim. dimension write test */ +#define WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_NAME "unshared_filtered_chunks_single_unlim_dim_write" +#define WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_UNSHARED_ONE_UNLIM_DIM_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS (mpi_size * DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS (WRITE_UNSHARED_ONE_UNLIM_DIM_NROWS / mpi_size) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS (WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS / mpi_size) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_NLOOPS 5 + +/* Defines for the shared filtered chunks w/ single unlim. dimension write test */ +#define WRITE_SHARED_ONE_UNLIM_DIM_DATASET_NAME "shared_filtered_chunks_single_unlim_dim_write" +#define WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS (mpi_size) +#define WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS (mpi_size) +#define WRITE_SHARED_ONE_UNLIM_DIM_NROWS (WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS * DIM0_SCALE_FACTOR) +#define WRITE_SHARED_ONE_UNLIM_DIM_NCOLS (WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS * DIM1_SCALE_FACTOR) +#define WRITE_SHARED_ONE_UNLIM_DIM_NLOOPS 5 + +/* Defines for the unshared filtered chunks w/ two unlim. dimension write test */ +#define WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_NAME "unshared_filtered_chunks_two_unlim_dim_write" +#define WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_UNSHARED_TWO_UNLIM_DIM_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_NCOLS (DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS (DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS (DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_NLOOPS 5 + +/* Defines for the shared filtered chunks w/ two unlim. dimension write test */ +#define WRITE_SHARED_TWO_UNLIM_DIM_DATASET_NAME "shared_filtered_chunks_two_unlim_dim_write" +#define WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_NROWS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_NCOLS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_NLOOPS 5 + /* Defines for the filtered chunks write test where a process has no selection */ #define WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME "single_no_selection_filtered_chunks_write" #define WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS 2 @@ -403,4 +447,53 @@ typedef struct { #define SHRINKING_GROWING_CHUNKS_CH_NCOLS (SHRINKING_GROWING_CHUNKS_NCOLS / mpi_size) #define SHRINKING_GROWING_CHUNKS_NLOOPS 20 +/* Defines for the unshared filtered edge chunks write test */ +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME "unshared_filtered_edge_chunks_write" +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2 "unshared_filtered_edge_chunks_no_filter_write" +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS 2 +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS (DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS (DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS \ + (mpi_size * DIM1_SCALE_FACTOR) + (WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS - 1) + +/* Defines for the shared filtered edge chunks write test */ +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME "shared_filtered_edge_chunks_write" +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2 "shared_filtered_edge_chunks_no_filter_write" +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS 2 +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS (mpi_size) +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS (mpi_size) +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS \ + (WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS * DIM0_SCALE_FACTOR) +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS \ + ((WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS * DIM1_SCALE_FACTOR) + \ + (WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS - 1)) + +/* Defines for the fill values test */ +#define FILL_VALUES_TEST_DATASET_NAME "fill_value_test" +#define FILL_VALUES_TEST_DATASET_NAME2 "fill_value_alloc_test" +#define FILL_VALUES_TEST_DATASET_DIMS 2 +#define FILL_VALUES_TEST_FILL_VAL (-1) +#define FILL_VALUES_TEST_CH_NROWS (mpi_size) +#define FILL_VALUES_TEST_CH_NCOLS (mpi_size + 1) +#define FILL_VALUES_TEST_NROWS (FILL_VALUES_TEST_CH_NROWS * DIM0_SCALE_FACTOR) +#define FILL_VALUES_TEST_NCOLS (FILL_VALUES_TEST_CH_NCOLS * DIM1_SCALE_FACTOR) + +/* Defines for the undefined fill value test */ +#define FILL_VALUE_UNDEFINED_TEST_DATASET_NAME "fill_value_undefined_test" +#define FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS 2 +#define FILL_VALUE_UNDEFINED_TEST_CH_NROWS (mpi_size) +#define FILL_VALUE_UNDEFINED_TEST_CH_NCOLS (mpi_size + 1) +#define FILL_VALUE_UNDEFINED_TEST_NROWS (FILL_VALUE_UNDEFINED_TEST_CH_NROWS * DIM0_SCALE_FACTOR) +#define FILL_VALUE_UNDEFINED_TEST_NCOLS (FILL_VALUE_UNDEFINED_TEST_CH_NCOLS * DIM1_SCALE_FACTOR) + +/* Defines for the fill time of 'never' test */ +#define FILL_TIME_NEVER_TEST_DATASET_NAME "fill_time_never_test" +#define FILL_TIME_NEVER_TEST_DATASET_DIMS 2 +#define FILL_TIME_NEVER_TEST_FILL_VAL (-1) +#define FILL_TIME_NEVER_TEST_CH_NROWS (mpi_size) +#define FILL_TIME_NEVER_TEST_CH_NCOLS (mpi_size + 1) +#define FILL_TIME_NEVER_TEST_NROWS (FILL_TIME_NEVER_TEST_CH_NROWS * DIM0_SCALE_FACTOR) +#define FILL_TIME_NEVER_TEST_NCOLS (FILL_TIME_NEVER_TEST_CH_NCOLS * DIM1_SCALE_FACTOR) + #endif /* TEST_PARALLEL_FILTERS_H_ */ diff --git a/testpar/t_mdset.c b/testpar/t_mdset.c index 3041e77..97d5966 100644 --- a/testpar/t_mdset.c +++ b/testpar/t_mdset.c @@ -1721,7 +1721,7 @@ io_mode_confusion(void) * Open the each of the data sets in turn. If all opens are successful, * the test passes. Otherwise the test fails. * - * Note that this test will probably become irrelevent shortly, when we + * Note that this test will probably become irrelevant shortly, when we * land the journaling modifications on the trunk -- at which point all * cache clients will have to construct on disk images on demand. * @@ -1782,7 +1782,7 @@ rr_obj_hdr_flush_confusion(void) mrc = MPI_Comm_split(MPI_COMM_WORLD, is_reader, mpi_rank, &comm); VRFY((mrc == MPI_SUCCESS), "MPI_Comm_split"); - /* The reader proocesses branches off to do reading + /* The reader processes branches off to do reading * while the writer processes continues to do writing * Whenever writers finish one writing step, including a H5Fflush, * they inform the readers, via MPI_COMM_WORLD, to verify. @@ -2063,7 +2063,7 @@ rr_obj_hdr_flush_confusion_writer(MPI_Comm comm) /* * flush the metadata cache yet again to clean the object headers. * - * This is an attempt to crate a situation where we have dirty + * This is an attempt to create a situation where we have dirty * object header continuation chunks, but clean opject headers * to verify a speculative bug fix -- it doesn't seem to work, * but I will leave the code in anyway, as the object header diff --git a/testpar/t_mpi.c b/testpar/t_mpi.c index a8a756c..96cbc4b 100644 --- a/testpar/t_mpi.c +++ b/testpar/t_mpi.c @@ -215,7 +215,7 @@ test_mpio_gb_file(char *filename) * sizes. */ if (MAINPROCESS) { /* only process 0 needs to check it*/ - HDprintf("MPI_Offset is %s %d bytes integeral type\n", is_signed ? "signed" : "unsigned", + HDprintf("MPI_Offset is %s %d bytes integral type\n", is_signed ? "signed" : "unsigned", (int)sizeof(MPI_Offset)); if (sizeof_mpi_offset <= 4 && is_signed) { HDprintf("Skipped 2GB range test " @@ -400,7 +400,7 @@ finish: * MPI-IO Test: One writes, Many reads. * Verify if only one process writes some data and then all other * processes can read them back correctly. This tests if the - * underlaying parallel I/O and file system supports parallel I/O + * underlying parallel I/O and file system supports parallel I/O * correctly. * * Algorithm: Only one process (e.g., process 0) writes some data. diff --git a/testpar/t_pflush1.c b/testpar/t_pflush1.c index 2a80f4a..0500a2d 100644 --- a/testpar/t_pflush1.c +++ b/testpar/t_pflush1.c @@ -43,7 +43,7 @@ static int data_g[100][100]; *------------------------------------------------------------------------- */ static hid_t -create_test_file(char *name, hid_t fapl_id) +create_test_file(char *name, size_t name_length, hid_t fapl_id) { hid_t fid = H5I_INVALID_HID; hid_t dcpl_id = H5I_INVALID_HID; @@ -86,7 +86,7 @@ create_test_file(char *name, hid_t fapl_id) if ((top_level_gid = H5Gcreate2(fid, "some_groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; for (i = 0; i < N_GROUPS; i++) { - HDsprintf(name, "grp%02u", (unsigned)i); + HDsnprintf(name, name_length, "grp%02u", (unsigned)i); if ((gid = H5Gcreate2(top_level_gid, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; if (H5Gclose(gid) < 0) @@ -133,7 +133,7 @@ main(int argc, char *argv[]) TESTING("H5Fflush (part1)"); /* Don't run using the split VFD */ - envval = HDgetenv("HDF5_DRIVER"); + envval = HDgetenv(HDF5_DRIVER); if (envval == NULL) envval = "nomatch"; @@ -153,7 +153,7 @@ main(int argc, char *argv[]) /* Create the file */ h5_fixname(FILENAME[0], fapl_id, name, sizeof(name)); - if ((fid1 = create_test_file(name, fapl_id)) < 0) + if ((fid1 = create_test_file(name, sizeof(name), fapl_id)) < 0) goto error; /* Flush and exit without closing the library */ if (H5Fflush(fid1, H5F_SCOPE_GLOBAL) < 0) @@ -161,7 +161,7 @@ main(int argc, char *argv[]) /* Create the other file which will not be flushed */ h5_fixname(FILENAME[1], fapl_id, name, sizeof(name)); - if ((fid2 = create_test_file(name, fapl_id)) < 0) + if ((fid2 = create_test_file(name, sizeof(name), fapl_id)) < 0) goto error; if (mpi_rank == 0) @@ -207,5 +207,7 @@ error: HDfflush(stderr); HDprintf("*** ERROR ***\n"); HDprintf("THERE WAS A REAL ERROR IN t_pflush1.\n"); + HDfflush(stdout); + HD_exit(EXIT_FAILURE); } /* end main() */ diff --git a/testpar/t_pflush2.c b/testpar/t_pflush2.c index c96233a..8cf40d0 100644 --- a/testpar/t_pflush2.c +++ b/testpar/t_pflush2.c @@ -43,7 +43,7 @@ static int data_g[100][100]; *------------------------------------------------------------------------- */ static herr_t -check_test_file(char *name, hid_t fapl_id) +check_test_file(char *name, size_t name_length, hid_t fapl_id) { hid_t fid = H5I_INVALID_HID; hid_t sid = H5I_INVALID_HID; @@ -89,7 +89,7 @@ check_test_file(char *name, hid_t fapl_id) if ((top_level_gid = H5Gopen2(fid, "some_groups", H5P_DEFAULT)) < 0) goto error; for (i = 0; i < N_GROUPS; i++) { - HDsprintf(name, "grp%02u", (unsigned)i); + HDsnprintf(name, name_length, "grp%02u", (unsigned)i); if ((gid = H5Gopen2(top_level_gid, name, H5P_DEFAULT)) < 0) goto error; if (H5Gclose(gid) < 0) @@ -157,7 +157,7 @@ main(int argc, char *argv[]) TESTING("H5Fflush (part2 with flush)"); /* Don't run using the split VFD */ - envval = HDgetenv("HDF5_DRIVER"); + envval = HDgetenv(HDF5_DRIVER); if (envval == NULL) envval = "nomatch"; @@ -167,7 +167,7 @@ main(int argc, char *argv[]) HDputs(" Test not compatible with current Virtual File Driver"); } MPI_Finalize(); - HDexit(EXIT_FAILURE); + HDexit(EXIT_SUCCESS); } if ((fapl_id1 = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -182,7 +182,7 @@ main(int argc, char *argv[]) /* Check the case where the file was flushed */ h5_fixname(FILENAME[0], fapl_id1, name, sizeof(name)); - if (check_test_file(name, fapl_id1)) { + if (check_test_file(name, sizeof(name), fapl_id1)) { H5_FAILED() goto error; } @@ -199,7 +199,7 @@ main(int argc, char *argv[]) H5Eset_auto2(H5E_DEFAULT, NULL, NULL); h5_fixname(FILENAME[1], fapl_id2, name, sizeof(name)); - if (check_test_file(name, fapl_id2)) { + if (check_test_file(name, sizeof(name), fapl_id2)) { if (mpi_rank == 0) PASSED(); } diff --git a/testpar/t_ph5basic.c b/testpar/t_ph5basic.c index 05f2c77..59f14cb 100644 --- a/testpar/t_ph5basic.c +++ b/testpar/t_ph5basic.c @@ -56,7 +56,7 @@ test_fapl_mpio_dup(void) HDprintf("rank/size of MPI_COMM_WORLD are %d/%d\n", mpi_rank, mpi_size); /* Create a new communicator that has the same processes as MPI_COMM_WORLD. - * Use MPI_Comm_split because it is simplier than MPI_Comm_create + * Use MPI_Comm_split because it is simpler than MPI_Comm_create */ mrc = MPI_Comm_split(MPI_COMM_WORLD, 0, 0, &comm); VRFY((mrc == MPI_SUCCESS), "MPI_Comm_split"); diff --git a/testpar/t_pread.c b/testpar/t_pread.c index 17091cb..d79130b 100644 --- a/testpar/t_pread.c +++ b/testpar/t_pread.c @@ -68,7 +68,7 @@ static char *test_argv0 = NULL; * more or less in half. Each sub group will operate * collectively on their assigned file. This split into * subgroups validates that parallel groups can successfully - * open and read data independantly from the other parallel + * open and read data independently from the other parallel * operations taking place. * * Return: Success: 0 @@ -132,7 +132,7 @@ generate_test_file(MPI_Comm comm, int mpi_rank, int group_id) } /* Setup the file names - * The test specfic filenames are stored as consecutive + * The test specific filenames are stored as consecutive * array entries in the global 'FILENAMES' array above. * Here, we simply decide on the starting index for * file construction. The reading portion of the test @@ -1043,11 +1043,11 @@ test_parallel_read(MPI_Comm comm, int mpi_rank, int mpi_size, int group_id) * Purpose: To implement a parallel test which validates whether the * new superblock lookup functionality is working correctly. * - * The test consists of creating two seperate HDF datasets + * The test consists of creating two separate HDF datasets * in which random text is inserted at the start of each * file using the 'j5jam' application. This forces the * HDF5 file superblock to a non-zero offset. - * Having created the two independant files, we create two + * Having created the two independent files, we create two * non-overlapping MPI groups, each of which is then tasked * with the opening and validation of the data contained * therein. @@ -1152,7 +1152,7 @@ main(int argc, char **argv) goto finish; } - /* Now read the generated test file (stil using MPI_COMM_WORLD) */ + /* Now read the generated test file (still using MPI_COMM_WORLD) */ nerrs += test_parallel_read(MPI_COMM_WORLD, mpi_rank, mpi_size, which_group); if (nerrs > 0) { diff --git a/testpar/t_prop.c b/testpar/t_prop.c index a3f1859..606e100 100644 --- a/testpar/t_prop.c +++ b/testpar/t_prop.c @@ -215,7 +215,7 @@ test_plist_ed(void) dxpl = H5Pcreate(H5P_DATASET_XFER); VRFY((dxpl >= 0), "H5Pcreate succeeded"); - ret = H5Pset_btree_ratios(dxpl, 0.2f, 0.6f, 0.2f); + ret = H5Pset_btree_ratios(dxpl, 0.2, 0.6, 0.2); VRFY((ret >= 0), "H5Pset_btree_ratios succeeded"); ret = H5Pset_hyper_vector_size(dxpl, 5); @@ -354,7 +354,7 @@ test_plist_ed(void) ret = H5Pset_alignment(fapl, 2, 1024); VRFY((ret >= 0), "H5Pset_alignment succeeded"); - ret = H5Pset_cache(fapl, 1024, 128, 10485760, 0.3f); + ret = H5Pset_cache(fapl, 1024, 128, 10485760, 0.3); VRFY((ret >= 0), "H5Pset_cache succeeded"); ret = H5Pset_elink_file_cache_size(fapl, 10485760); @@ -556,7 +556,7 @@ external_links(void) /* test opening a group that is to an external link, the external linked file should inherit the source file's access properties */ - HDsprintf(link_path, "%s%s%s", group_path, "/", link_name); + HDsnprintf(link_path, sizeof(link_path), "%s%s%s", group_path, "/", link_name); group = H5Gopen2(fid, link_path, H5P_DEFAULT); VRFY((group >= 0), "H5Gopen succeeded"); ret = H5Gclose(group); diff --git a/testpar/t_shapesame.c b/testpar/t_shapesame.c index 2dd867a..0a18781 100644 --- a/testpar/t_shapesame.c +++ b/testpar/t_shapesame.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - This program will test independant and collective reads and writes between + This program will test independent and collective reads and writes between selections of different rank that non-the-less are deemed as having the same shape by H5Sselect_shape_same(). */ @@ -415,11 +415,11 @@ hs_dr_pio_test__setup(const int test_num, const int edge_size, const int checker /* setup selections for writing initial data to the small data set */ ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded"); if (MAINPROCESS) { /* add an additional slice to the selections */ @@ -427,11 +427,11 @@ hs_dr_pio_test__setup(const int test_num, const int edge_size, const int checker ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, or) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, or) succeeded"); } /* write the initial value of the small data set to file */ @@ -475,21 +475,21 @@ hs_dr_pio_test__setup(const int test_num, const int edge_size, const int checker ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, set) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, set) succeeded"); /* In passing, setup the process slice dataspaces as well */ ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_process_slice_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_process_slice_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_process_slice_sid, set) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_process_slice_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_process_slice_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_process_slice_sid, set) succeeded"); if (MAINPROCESS) { /* add an additional slice to the selections */ @@ -497,11 +497,11 @@ hs_dr_pio_test__setup(const int test_num, const int edge_size, const int checker ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, or) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_OR, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, or) succeeded"); } /* write the initial value of the large data set to file */ @@ -933,7 +933,7 @@ contig_hs_dr_pio_test__d2m_s2l(struct hs_dr_pio_test_vars_t *tv_ptr) ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded"); #if CONTIG_HS_DR_PIO_TEST__D2M_S2L__DEBUG HDfprintf(stdout, "%s reading slices of on disk small data set into slices of big data set.\n", fcnName); @@ -1185,11 +1185,11 @@ contig_hs_dr_pio_test__m2d_l2s(struct hs_dr_pio_test_vars_t *tv_ptr) ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded"); /* set up start, stride, count, and block -- note that we will * change start[] so as to read slices of the large cube. @@ -1462,7 +1462,7 @@ contig_hs_dr_pio_test__m2d_s2l(struct hs_dr_pio_test_vars_t *tv_ptr) ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded"); /* set up start, stride, count, and block -- note that we will * change start[] so as to write slices of the small data set to @@ -1572,7 +1572,7 @@ contig_hs_dr_pio_test__m2d_s2l(struct hs_dr_pio_test_vars_t *tv_ptr) ret = H5Dwrite(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->large_ds_slice_sid, tv_ptr->file_large_ds_process_slice_sid, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_2); - VRFY((ret != FAIL), "H5Dwrite() to zero large ds suceeded"); + VRFY((ret != FAIL), "H5Dwrite() to zero large ds succeeded"); /* select the portion of the in memory large cube to which we * are going to write data. @@ -2008,8 +2008,8 @@ contig_hs_dr_pio_test(ShapeSameTestMethods sstest_type) ** edge_size, and a checker_edge_size, select a checker ** board selection of a sel_rank (sel_rank < tgt_rank) ** dimensional slice through the dataspace parallel to the -** sel_rank fastest changing indicies, with origin (in the -** higher indicies) as indicated by the start array. +** sel_rank fastest changing indices, with origin (in the +** higher indices) as indicated by the start array. ** ** Note that this function, like all its relatives, is ** hard coded to presume a maximum dataspace rank of 5. @@ -2318,7 +2318,7 @@ ckrbrd_hs_dr_pio_test__slct_ckrbrd(const int mpi_rank, const hid_t tgt_sid, cons ** or writing a checker board selection of an m (1 <= m < ** rank) dimensional slice through this processes slice ** of the target data set. Also, this slice must be parallel -** to the fastest changing indicies. +** to the fastest changing indices. ** ** It is further presumed that the buffer was zeroed before ** the read/write, and that the full target data set (i.e. @@ -2353,7 +2353,7 @@ ckrbrd_hs_dr_pio_test__slct_ckrbrd(const int mpi_rank, const hid_t tgt_sid, cons ** of the buffer resides either at the origin of either ** a selected or an unselected checker. (Translation: ** if partial checkers appear in the buffer, they will -** intersect the edges of the n-cube oposite the origin.) +** intersect the edges of the n-cube opposite the origin.) ** ****************************************************************/ @@ -3091,11 +3091,11 @@ ckrbrd_hs_dr_pio_test__m2d_l2s(struct hs_dr_pio_test_vars_t *tv_ptr) ret = H5Sselect_hyperslab(tv_ptr->file_small_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid_0, set) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->mem_small_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded"); sel_start[0] = sel_start[1] = sel_start[2] = sel_start[3] = sel_start[4] = 0; sel_start[tv_ptr->small_ds_offset] = (hsize_t)(tv_ptr->mpi_rank); @@ -3384,11 +3384,11 @@ ckrbrd_hs_dr_pio_test__m2d_s2l(struct hs_dr_pio_test_vars_t *tv_ptr) ret = H5Sselect_hyperslab(tv_ptr->file_large_ds_sid_0, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid_0, set) succeeded"); ret = H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, H5S_SELECT_SET, tv_ptr->start, tv_ptr->stride, tv_ptr->count, tv_ptr->block); - VRFY((ret >= 0), "H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(tv_ptr->mem_large_ds_sid, set) succeeded"); /* setup a checkerboard selection of the slice of the in memory small * data set associated with the process's mpi rank. @@ -3496,7 +3496,7 @@ ckrbrd_hs_dr_pio_test__m2d_s2l(struct hs_dr_pio_test_vars_t *tv_ptr) */ ret = H5Dwrite(tv_ptr->large_dataset, H5T_NATIVE_UINT32, tv_ptr->mem_large_ds_sid, tv_ptr->file_large_ds_sid_0, tv_ptr->xfer_plist, tv_ptr->large_ds_buf_2); - VRFY((ret != FAIL), "H5Dwrite() to zero large ds suceeded"); + VRFY((ret != FAIL), "H5Dwrite() to zero large ds succeeded"); /* select the portion of the in memory large cube to which we * are going to write data. @@ -4210,42 +4210,42 @@ create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type) return (ret_pl); } -/* Shape Same test using contigous hyperslab using independent IO on contigous datasets */ +/* Shape Same test using contiguous hyperslab using independent IO on contiguous datasets */ static void sscontig1(void) { contig_hs_dr_pio_test(IND_CONTIG); } -/* Shape Same test using contigous hyperslab using collective IO on contigous datasets */ +/* Shape Same test using contiguous hyperslab using collective IO on contiguous datasets */ static void sscontig2(void) { contig_hs_dr_pio_test(COL_CONTIG); } -/* Shape Same test using contigous hyperslab using independent IO on chunked datasets */ +/* Shape Same test using contiguous hyperslab using independent IO on chunked datasets */ static void sscontig3(void) { contig_hs_dr_pio_test(IND_CHUNKED); } -/* Shape Same test using contigous hyperslab using collective IO on chunked datasets */ +/* Shape Same test using contiguous hyperslab using collective IO on chunked datasets */ static void sscontig4(void) { contig_hs_dr_pio_test(COL_CHUNKED); } -/* Shape Same test using checker hyperslab using independent IO on contigous datasets */ +/* Shape Same test using checker hyperslab using independent IO on contiguous datasets */ static void sschecker1(void) { ckrbrd_hs_dr_pio_test(IND_CONTIG); } -/* Shape Same test using checker hyperslab using collective IO on contigous datasets */ +/* Shape Same test using checker hyperslab using collective IO on contiguous datasets */ static void sschecker2(void) { @@ -4305,7 +4305,7 @@ main(int argc, char **argv) /* Initialize testing framework */ TestInit(argv[0], usage, parse_options); - /* Shape Same tests using contigous hyperslab */ + /* Shape Same tests using contiguous hyperslab */ AddTest("sscontig1", sscontig1, NULL, "Cntg hslab, ind IO, cntg dsets", PARATESTFILE); AddTest("sscontig2", sscontig2, NULL, "Cntg hslab, col IO, cntg dsets", PARATESTFILE); AddTest("sscontig3", sscontig3, NULL, "Cntg hslab, ind IO, chnk dsets", PARATESTFILE); diff --git a/testpar/t_span_tree.c b/testpar/t_span_tree.c index 518741d..e624a01 100644 --- a/testpar/t_span_tree.c +++ b/testpar/t_span_tree.c @@ -424,7 +424,7 @@ coll_write_test(int chunk_factor) the correctedness of collective write compared with independent write, - In order to throughly test this feature, we choose + In order to thoroughly test this feature, we choose a different selection set for reading the data out. @@ -890,8 +890,8 @@ coll_read_test(void) ** edge_size, and a checker_edge_size, select a checker ** board selection of a sel_rank (sel_rank < tgt_rank) ** dimensional slice through the dataspace parallel to the -** sel_rank fastest changing indicies, with origin (in the -** higher indicies) as indicated by the start array. +** sel_rank fastest changing indices, with origin (in the +** higher indices) as indicated by the start array. ** ** Note that this function, is hard coded to presume a ** maximum dataspace rank of 5. @@ -1230,7 +1230,7 @@ lower_dim_size_comp_test__select_checker_board(const int mpi_rank, const hid_t t ** or writing a checker board selection of an m (1 <= m < ** rank) dimensional slice through this processes slice ** of the target data set. Also, this slice must be parallel -** to the fastest changing indicies. +** to the fastest changing indices. ** ** It is further presumed that the buffer was zeroed before ** the read/write, and that the full target data set (i.e. @@ -1265,7 +1265,7 @@ lower_dim_size_comp_test__select_checker_board(const int mpi_rank, const hid_t t ** of the buffer resides either at the origin of either ** a selected or an unselected checker. (Translation: ** if partial checkers appear in the buffer, they will -** intersect the edges of the n-cube oposite the origin.) +** intersect the edges of the n-cube opposite the origin.) ** ****************************************************************/ @@ -1726,7 +1726,7 @@ lower_dim_size_comp_test__run_test(const int chunk_edge_size, const hbool_t use_ if (!use_collective_io) { ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); - VRFY((ret >= 0), "H5Pset_dxpl_mpio_collective_opt() suceeded"); + VRFY((ret >= 0), "H5Pset_dxpl_mpio_collective_opt() succeeded"); } /* setup selection to write initial data to the small data sets */ @@ -1758,10 +1758,10 @@ lower_dim_size_comp_test__run_test(const int chunk_edge_size, const hbool_t use_ /* setup selections for writing initial data to the small data set */ ret = H5Sselect_hyperslab(mem_small_ds_sid, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, set) succeeded"); ret = H5Sselect_hyperslab(file_small_ds_sid, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid, set) succeeded"); if (MAINPROCESS) { /* add an additional slice to the selections */ @@ -1782,10 +1782,10 @@ lower_dim_size_comp_test__run_test(const int chunk_edge_size, const hbool_t use_ #endif /* LOWER_DIM_SIZE_COMP_TEST__RUN_TEST__DEBUG */ ret = H5Sselect_hyperslab(mem_small_ds_sid, H5S_SELECT_OR, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_small_ds_sid, or) succeeded"); ret = H5Sselect_hyperslab(file_small_ds_sid, H5S_SELECT_OR, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_small_ds_sid, or) succeeded"); } check = H5Sselect_valid(mem_small_ds_sid); @@ -1864,10 +1864,10 @@ lower_dim_size_comp_test__run_test(const int chunk_edge_size, const hbool_t use_ #endif /* LOWER_DIM_SIZE_COMP_TEST__RUN_TEST__DEBUG */ ret = H5Sselect_hyperslab(mem_large_ds_sid, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, set) succeeded"); ret = H5Sselect_hyperslab(file_large_ds_sid, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid, set) succeeded"); #if LOWER_DIM_SIZE_COMP_TEST__RUN_TEST__DEBUG if (mpi_rank == LOWER_DIM_SIZE_COMP_TEST_DEBUG_TARGET_RANK) { @@ -1897,10 +1897,10 @@ lower_dim_size_comp_test__run_test(const int chunk_edge_size, const hbool_t use_ #endif /* LOWER_DIM_SIZE_COMP_TEST__RUN_TEST__DEBUG */ ret = H5Sselect_hyperslab(mem_large_ds_sid, H5S_SELECT_OR, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(mem_large_ds_sid, or) succeeded"); ret = H5Sselect_hyperslab(file_large_ds_sid, H5S_SELECT_OR, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid, or) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_large_ds_sid, or) succeeded"); #if LOWER_DIM_SIZE_COMP_TEST__RUN_TEST__DEBUG if (mpi_rank == LOWER_DIM_SIZE_COMP_TEST_DEBUG_TARGET_RANK) { @@ -2427,7 +2427,7 @@ link_chunk_collective_io_test(void) /* select the file and mem spaces */ start[0] = (hsize_t)(mpi_rank * LINK_CHUNK_COLLECTIVE_IO_TEST_CHUNK_SIZE); ret = H5Sselect_hyperslab(file_ds_sid, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sselect_hyperslab(file_ds_sid, set) suceeded"); + VRFY((ret >= 0), "H5Sselect_hyperslab(file_ds_sid, set) succeeded"); ret = H5Sselect_all(write_mem_ds_sid); VRFY((ret != FAIL), "H5Sselect_all(mem_ds_sid) succeeded"); diff --git a/testpar/testpflush.sh.in b/testpar/testpflush.sh.in index be0dcb3..b400447 100644 --- a/testpar/testpflush.sh.in +++ b/testpar/testpflush.sh.in @@ -50,7 +50,7 @@ RUNPARALLELSCRIPT=`echo "@RUNPARALLEL@" | sed "s/$$/\$/g"` echo "*** NOTE ***********************************************************" echo "You may see complaints from mpiexec et al. that not all processes" echo "called MPI_Finalize(). This is an intended characteristic of the" -echo "test and should not be considered an error." +echo "test and should not be considered an error." echo "********************************************************************" eval ${RUNPARALLELSCRIPT} ./t_pflush1 diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index e3075a3..e6c36bf 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -186,10 +186,6 @@ enum H5TEST_COLL_CHUNK_API { #define TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES 0x010 #define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT 0x020 #define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL 0x040 -#define TEST_FILTERS 0x080 -/* TEST_FILTERS will take place of this after supporting mpio + filter for - * H5Dcreate and H5Dwrite */ -#define TEST_FILTERS_READ 0x100 /* Don't erase these lines, they are put here for debugging purposes */ /* @@ -218,8 +214,8 @@ typedef int DATATYPE; /* Shape Same Tests Definitions */ typedef enum { - IND_CONTIG, /* Independent IO on contigous datasets */ - COL_CONTIG, /* Collective IO on contigous datasets */ + IND_CONTIG, /* Independent IO on contiguous datasets */ + COL_CONTIG, /* Collective IO on contiguous datasets */ IND_CHUNKED, /* Independent IO on chunked datasets */ COL_CHUNKED /* Collective IO on chunked datasets */ } ShapeSameTestMethods; diff --git a/tools/lib/h5diff.c b/tools/lib/h5diff.c index 0ad319e..2275c72 100644 --- a/tools/lib/h5diff.c +++ b/tools/lib/h5diff.c @@ -1228,7 +1228,7 @@ diff_match(hid_t file1_id, const char *grp1, trav_info_t *info1, hid_t file2_id, H5TOOLS_DEBUG("Beginning of big else block"); /* We're in parallel mode */ /* Since the data type of diff value is hsize_t which can - * be arbitary large such that there is no MPI type that + * be arbitrary large such that there is no MPI type that * matches it, the value is passed between processes as * an array of bytes in order to be portable. But this * may not work in non-homogeneous MPI environments. diff --git a/tools/lib/h5diff.h b/tools/lib/h5diff.h index 7b41538..87eff98 100644 --- a/tools/lib/h5diff.h +++ b/tools/lib/h5diff.h @@ -74,7 +74,7 @@ typedef struct { struct exclude_path_list *exclude_attr; /* keep exclude attribute list */ int count_bool; /* count, compare up to count */ hsize_t count; /* count value */ - diff_err_t err_stat; /* an error ocurred (2, error, 1, differences, 0, no error) */ + diff_err_t err_stat; /* an error occurred (2, error, 1, differences, 0, no error) */ hsize_t nelmts; /* total number of elements */ hsize_t hs_nelmts; /* number of elements to read at a time*/ int rank; /* dimensionality */ diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c index 1b0d36f..a7156b5 100644 --- a/tools/lib/h5diff_array.c +++ b/tools/lib/h5diff_array.c @@ -204,7 +204,8 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ mcomp_t members; H5T_class_t type_class; - H5TOOLS_START_DEBUG(" - rank:%d hs_nelmts:%lld errstat:%d", opts->rank, opts->hs_nelmts, opts->err_stat); + H5TOOLS_START_DEBUG(" - rank:%d hs_nelmts:%" PRIuHSIZE " errstat:%d", opts->rank, opts->hs_nelmts, + opts->err_stat); opts->print_header = 1; /* enable print header */ /* get the size. */ @@ -388,7 +389,8 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ HDmemset(&members, 0, sizeof(mcomp_t)); get_member_types(opts->m_tid, &members); for (i = 0; i < opts->hs_nelmts; i++) { - H5TOOLS_DEBUG("opts->pos[%lld]:%lld - nelmts:%lld", i, opts->pos[i], opts->hs_nelmts); + H5TOOLS_DEBUG("opts->pos[%" PRIuHSIZE "]:%" PRIuHSIZE " - nelmts:%" PRIuHSIZE, i, + opts->pos[i], opts->hs_nelmts); nfound += diff_datum(mem1 + i * size, mem2 + i * size, i, opts, container1_id, container2_id, &members); if (opts->count_bool && nfound >= opts->count) @@ -396,7 +398,7 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ } /* i */ close_member_types(&members); } /* switch */ - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -452,7 +454,8 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co hsize_t nfound = 0; /* differences found */ diff_err_t ret_value = opts->err_stat; - H5TOOLS_START_DEBUG("ph:%d elemtno:%lld - errstat:%d", opts->print_header, elemtno, opts->err_stat); + H5TOOLS_START_DEBUG("ph:%d elemtno:%" PRIuHSIZE " - errstat:%d", opts->print_header, elemtno, + opts->err_stat); type_size = H5Tget_size(opts->m_tid); type_class = H5Tget_class(opts->m_tid); @@ -702,8 +705,8 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co H5TOOLS_DEBUG("H5T_ARRAY ph=%d", opts->print_header); arr_opts = *opts; - H5TOOLS_DEBUG("Check opts: hs_nelmts:%lld to %lld rank:%d to %d", opts->hs_nelmts, - arr_opts.hs_nelmts, opts->rank, arr_opts.rank); + H5TOOLS_DEBUG("Check opts: hs_nelmts:%" PRIuHSIZE " to %" PRIuHSIZE " rank:%d to %d", + opts->hs_nelmts, arr_opts.hs_nelmts, opts->rank, arr_opts.rank); /* get the array's base datatype for each element */ arr_opts.m_tid = H5Tget_super(opts->m_tid); size = H5Tget_size(arr_opts.m_tid); @@ -1051,8 +1054,8 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co H5TOOLS_INFO("H5Rdestroy H5R_OBJECT1 failed"); if (H5Rdestroy(ref1_buf) < 0) H5TOOLS_INFO("H5Rdestroy H5R_OBJECT1 failed"); - H5TOOLS_DEBUG("H5T_REFERENCE - H5T_STD_REF complete nfound:%lld - errstat:%d", nfound, - ref_opts.err_stat); + H5TOOLS_DEBUG("H5T_REFERENCE - H5T_STD_REF complete nfound:%" PRIuHSIZE " - errstat:%d", + nfound, ref_opts.err_stat); } /*------------------------------------------------------------------------- * H5T_STD_REF_DSETREG @@ -1263,7 +1266,7 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co done: opts->err_stat = opts->err_stat | ret_value; - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -1374,8 +1377,8 @@ diff_region(hid_t obj1_id, hid_t obj2_id, hid_t region1_id, hid_t region2_id, di npoints2 = H5Sget_select_elem_npoints(region2_id); } H5E_END_TRY; - H5TOOLS_DEBUG("blocks: 1=%lld-2=%lld", nblocks1, nblocks2); - H5TOOLS_DEBUG("points: 1=%lld-2=%lld", npoints1, npoints2); + H5TOOLS_DEBUG("blocks: 1=%" PRIdHSIZE "-2=%" PRIdHSIZE, nblocks1, nblocks2); + H5TOOLS_DEBUG("points: 1=%" PRIdHSIZE "-2=%" PRIdHSIZE, npoints1, npoints2); if (nblocks1 != nblocks2 || npoints1 != npoints2 || ndims1 != ndims2) { opts->not_cmp = 1; @@ -1518,10 +1521,10 @@ diff_region(hid_t obj1_id, hid_t obj2_id, hid_t region1_id, hid_t region2_id, di #if defined(H5DIFF_DEBUG) for (i = 0; i < npoints1; i++) { - parallel_print("%sPt%lu: ", i ? "," : "", (unsigned long)i); + parallel_print("%sPt%d: ", i ? "," : "", i); for (j = 0; j < ndims1; j++) - parallel_print("%s%lu", j ? "," : "(", (unsigned long)(ptdata1[i * ndims1 + j])); + parallel_print("%s%" PRIuHSIZE, j ? "," : "(", ptdata1[i * ndims1 + j]); parallel_print(")"); } @@ -1538,7 +1541,7 @@ diff_region(hid_t obj1_id, hid_t obj2_id, hid_t region1_id, hid_t region2_id, di ret_value = nfound_p + nfound_b; done: - H5TOOLS_ENDDEBUG(" with diffs:%lld", ret_value); + H5TOOLS_ENDDEBUG(" with diffs:%" PRIuHSIZE, ret_value); return ret_value; } @@ -1575,7 +1578,7 @@ character_compare(char *mem1, char *mem2, hsize_t elemtno, size_t u, diff_opt_t } nfound++; } - H5TOOLS_ENDDEBUG(": %lld", nfound); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE, nfound); return nfound; } @@ -1649,7 +1652,7 @@ character_compare_opt(unsigned char *mem1, unsigned char *mem2, hsize_t elemtno, nfound++; } - H5TOOLS_ENDDEBUG(": %lld zero:%d", nfound, both_zero); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE " zero:%d", nfound, both_zero); return nfound; } @@ -1830,7 +1833,7 @@ diff_float_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d } } - H5TOOLS_ENDDEBUG(": %lld zero:%d", nfound, both_zero); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE " zero:%d", nfound, both_zero); return nfound; } @@ -2001,7 +2004,7 @@ diff_double_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, nfound++; } } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2063,7 +2066,7 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, opts->print_percentage = 0; print_pos(opts, elem_idx, 0); if (print_data(opts)) { - parallel_print(F_FORMAT, temp1_double, temp2_double, ABS(temp1_double - temp2_double)); + parallel_print(LD_FORMAT, temp1_double, temp2_double, ABS(temp1_double - temp2_double)); } nfound++; } @@ -2110,7 +2113,7 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, opts->print_percentage = 0; print_pos(opts, elem_idx, 0); if (print_data(opts)) { - parallel_print(F_FORMAT, temp1_double, temp2_double, ABS(temp1_double - temp2_double)); + parallel_print(LD_FORMAT, temp1_double, temp2_double, ABS(temp1_double - temp2_double)); } nfound++; } @@ -2157,7 +2160,7 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, opts->print_percentage = 0; print_pos(opts, elem_idx, 0); if (print_data(opts)) { - parallel_print(F_FORMAT, temp1_double, temp2_double, ABS(temp1_double - temp2_double)); + parallel_print(LD_FORMAT, temp1_double, temp2_double, ABS(temp1_double - temp2_double)); } nfound++; } @@ -2175,7 +2178,7 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2263,7 +2266,7 @@ diff_schar_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2351,7 +2354,7 @@ diff_uchar_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2439,7 +2442,7 @@ diff_short_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2531,7 +2534,7 @@ diff_ushort_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2619,7 +2622,7 @@ diff_int_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, dif nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2707,7 +2710,7 @@ diff_uint_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, di nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2795,7 +2798,7 @@ diff_long_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, di nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2885,7 +2888,7 @@ diff_ulong_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2978,7 +2981,7 @@ diff_llong_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d } } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -3078,7 +3081,7 @@ diff_ullong_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, } } - H5TOOLS_ENDDEBUG(": %lld zero:%d", nfound, both_zero); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE " zero:%d", nfound, both_zero); return nfound; } @@ -3323,7 +3326,7 @@ print_header(diff_opt_t *opts) static void print_pos(diff_opt_t *opts, hsize_t idx, size_t u) { - H5TOOLS_START_DEBUG(" -- idx:%lld", idx); + H5TOOLS_START_DEBUG(" -- idx:%" PRIuHSIZE, idx); if (print_data(opts)) { hsize_t curr_pos = idx; @@ -3336,8 +3339,8 @@ print_pos(diff_opt_t *opts, hsize_t idx, size_t u) H5TOOLS_DEBUG("rank=%d", opts->rank); if (opts->rank > 0) { parallel_print("[ "); - H5TOOLS_DEBUG("do calc_acc_pos[%lld] nelmts:%lld - errstat:%d", idx, opts->hs_nelmts, - opts->err_stat); + H5TOOLS_DEBUG("do calc_acc_pos[%" PRIuHSIZE "] nelmts:%" PRIuHSIZE " - errstat:%d", idx, + opts->hs_nelmts, opts->err_stat); if (opts->sset[0] != NULL) { /* Subsetting is used - calculate total position */ hsize_t curr_idx = 0; /* current pos in the selection space for each dimension */ @@ -3364,33 +3367,36 @@ print_pos(diff_opt_t *opts, hsize_t idx, size_t u) j = opts->rank - i - 1; prev_total_dim_size *= prev_dim_size; dim_size = opts->dims[j]; - H5TOOLS_DEBUG("j=%d, dim_size=%lld, prev_dim_size=%lld, total_dim_size=%lld, " - "prev_total_dim_size=%lld", + H5TOOLS_DEBUG("j=%d, dim_size=%" PRIuHSIZE ", prev_dim_size=%" PRIuHSIZE + ", total_dim_size=%" PRIuHSIZE ", " + "prev_total_dim_size=%" PRIuHSIZE, j, dim_size, prev_dim_size, total_dim_size, prev_total_dim_size); count = opts->sset[0]->count.data[j]; block = opts->sset[0]->block.data[j]; stride = opts->sset[0]->stride.data[j]; - H5TOOLS_DEBUG("stride=%lld, count=%lld, block=%lld", stride, count, block); + H5TOOLS_DEBUG("stride=%" PRIuHSIZE ", count=%" PRIuHSIZE ", block=%" PRIuHSIZE, + stride, count, block); tmp = count * block; k0 = curr_idx / tmp; k1 = curr_idx % tmp; curr_pos += k1 * stride * prev_total_dim_size; - H5TOOLS_DEBUG("curr_idx=%lld, k0=%lld, k1=%lld, curr_pos=%lld", curr_idx, k0, k1, - curr_pos); + H5TOOLS_DEBUG("curr_idx=%" PRIuHSIZE ", k0=%" PRIuHSIZE ", k1=%" PRIuHSIZE + ", curr_pos=%" PRIuHSIZE, + curr_idx, k0, k1, curr_pos); if (k0 > 0) curr_idx = k0 * total_dim_size; - H5TOOLS_DEBUG("curr_idx=%lld, tmp=%lld", curr_idx, tmp); + H5TOOLS_DEBUG("curr_idx=%" PRIuHSIZE ", tmp=%" PRIuHSIZE, curr_idx, tmp); total_dim_size *= dim_size; /* if last calculation exists within in current dimension */ if (k0 == 0) break; - H5TOOLS_DEBUG("j=%d, curr_pos=%lld", j, curr_pos); + H5TOOLS_DEBUG("j=%d, curr_pos=%" PRIuHSIZE, j, curr_pos); prev_dim_size = dim_size; } /* check if there is a final calculation needed for slowest changing dimension */ if (k0 > 0) curr_pos += k0 * stride * prev_total_dim_size; - H5TOOLS_DEBUG("4:curr_idx=%lld, curr_pos=%lld", curr_idx, curr_pos); + H5TOOLS_DEBUG("4:curr_idx=%" PRIuHSIZE ", curr_pos=%" PRIuHSIZE, curr_idx, curr_pos); } } /* @@ -3400,11 +3406,11 @@ print_pos(diff_opt_t *opts, hsize_t idx, size_t u) calc_acc_pos((unsigned)opts->rank, curr_pos, opts->acc, opts->pos); for (int i = 0; i < opts->rank; i++) { - H5TOOLS_DEBUG("pos loop:%d with opts->pos=%lld opts->sm_pos=%lld", i, opts->pos[i], - opts->sm_pos[i]); + H5TOOLS_DEBUG("pos loop:%d with opts->pos=%" PRIuHSIZE " opts->sm_pos=%" PRIuHSIZE, i, + opts->pos[i], opts->sm_pos[i]); opts->pos[i] += (unsigned long)opts->sm_pos[i]; - H5TOOLS_DEBUG("pos loop:%d with opts->pos=%lld", i, opts->pos[i]); - parallel_print(HSIZE_T_FORMAT, (unsigned long long)opts->pos[i]); + H5TOOLS_DEBUG("pos loop:%d with opts->pos=%" PRIuHSIZE, i, opts->pos[i]); + parallel_print("%" PRIuHSIZE, opts->pos[i]); parallel_print(" "); } parallel_print("]"); diff --git a/tools/lib/h5diff_attr.c b/tools/lib/h5diff_attr.c index 5c1777a..63b6d8f 100644 --- a/tools/lib/h5diff_attr.c +++ b/tools/lib/h5diff_attr.c @@ -292,7 +292,7 @@ build_match_list_attrs(hid_t loc1_id, hid_t loc2_id, table_attrs_t **table_out, } if (opts->mode_verbose_level >= 1) - parallel_print("Attributes status: %d common, %d only in obj1, %d only in obj2\n", + parallel_print("Attributes status: %zu common, %zu only in obj1, %zu only in obj2\n", table_lp->nattrs - table_lp->nattrs_only1 - table_lp->nattrs_only2, table_lp->nattrs_only1, table_lp->nattrs_only2); diff --git a/tools/lib/h5diff_util.c b/tools/lib/h5diff_util.c index c40de9d..05f380a 100644 --- a/tools/lib/h5diff_util.c +++ b/tools/lib/h5diff_util.c @@ -39,11 +39,11 @@ print_dimensions(int rank, hsize_t *dims) else { parallel_print("["); for (i = 0; i < rank - 1; i++) { - parallel_print(HSIZE_T_FORMAT, dims[i]); + parallel_print("%" PRIuHSIZE, dims[i]); parallel_print("x"); } - parallel_print(HSIZE_T_FORMAT, dims[rank - 1]); + parallel_print("%" PRIuHSIZE, dims[rank - 1]); parallel_print("]"); } } diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 78e5c48..45fc78c 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -624,6 +624,12 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info) H5TOOLS_GOTO_ERROR(FAIL, "invalid VFD name"); done: + if (ret_value < 0) { + /* Clear error message unless asked for */ + if (enable_error_stack <= 1) + H5Epop(H5tools_ERR_STACK_g, 1); + } + return ret_value; } @@ -723,6 +729,10 @@ done: if (ret_value < 0) { if (connector_id >= 0 && H5Idec_ref(connector_id) < 0) H5TOOLS_ERROR(FAIL, "failed to decrement refcount on VOL connector ID"); + + /* Clear error message unless asked for */ + if (enable_error_stack <= 1) + H5Epop(H5tools_ERR_STACK_g, 1); } return ret_value; @@ -771,9 +781,15 @@ h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info, h5tools_vfd_i ret_value = new_fapl_id; done: - if ((new_fapl_id >= 0) && (ret_value < 0)) { - H5Pclose(new_fapl_id); - new_fapl_id = H5I_INVALID_HID; + if (ret_value < 0) { + if (new_fapl_id >= 0) { + H5Pclose(new_fapl_id); + new_fapl_id = H5I_INVALID_HID; + } + + /* Clear error message unless asked for */ + if (enable_error_stack <= 1) + H5Epop(H5tools_ERR_STACK_g, 1); } return ret_value; @@ -1049,6 +1065,10 @@ done: if (tmp_fapl_id >= 0) H5Pclose(tmp_fapl_id); + /* Clear error message unless asked for */ + if (ret_value < 0 && enable_error_stack <= 1) + H5Epop(H5tools_ERR_STACK_g, 1); + return ret_value; } diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index b4c1e49..8f64dd3 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -29,9 +29,6 @@ #define START_OF_DATA 0x0001 #define END_OF_DATA 0x0002 -/* format for hsize_t */ -#define HSIZE_T_FORMAT "%" H5_PRINTF_LL_WIDTH "u" - #define H5TOOLS_DUMP_MAX_RANK H5S_MAX_RANK /* Stream macros */ @@ -240,7 +237,7 @@ typedef struct h5tool_format_t { * * fmt_schar: The printf() format to use when rendering data which is * typed `signed char'. The default is `%d'. This format is - * used ony if the `ascii' field is zero. + * used only if the `ascii' field is zero. * * fmt_uchar: The printf() format to use when rendering data which is * typed `unsigned char'. The default is `%u'. This format @@ -411,9 +408,9 @@ typedef struct h5tool_format_t { * sep: Each integer in the index list will be separated from the * others by this string, which defaults to a comma. * - * fmt: After the index values are formated individually and + * fmt: After the index values are formatted individually and * separated from one another by some string, the entire - * resulting string will be formated according to this + * resulting string will be formatted according to this * printf(3c) format which should include a format for a * character string. The default is "%s". */ @@ -483,7 +480,7 @@ typedef struct h5tool_format_t { const char *line_suf; /*string to append to each line */ const char *line_sep; /*separates lines */ int line_multi_new; /*split multi-line outputs? */ - const char *line_indent; /*for extra identation if we need it*/ + const char *line_indent; /*for extra indentation if we need it*/ /*used to skip the first set of checks for line length*/ int skip_first; @@ -605,7 +602,7 @@ typedef enum { } driver_idx; /* The following include, h5tools_str.h, must be after the - * above stucts are defined. There is a dependency in the following + * above structs are defined. There is a dependency in the following * include that hasn't been identified yet. */ #include "h5tools_str.h" diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c index cb4f9d2..78dd3ff 100644 --- a/tools/lib/h5tools_dump.c +++ b/tools/lib/h5tools_dump.c @@ -478,7 +478,7 @@ done: * This is a special case subfunction to print the data in a region reference of type blocks. * * Return: - * The function returns FAIL if there was an error, otherwise SUCEED + * The function returns FAIL if there was an error, otherwise SUCCEED * * Parameters Description: * h5tools_str_t *buffer is the string into which to render @@ -500,7 +500,7 @@ h5tools_print_region_data_blocks(hid_t region_id, FILE *stream, const h5tool_for hsize_t * count = NULL; hsize_t blkndx; hsize_t total_size[H5S_MAX_RANK]; - hsize_t elmtno; /* elemnt index */ + hsize_t elmtno; /* element index */ hsize_t curr_pos = 0; unsigned int region_flags; /* buffer extent flags */ hsize_t numelem; @@ -745,11 +745,11 @@ h5tools_dump_region_data_blocks(hid_t region_space, hid_t region_id, FILE *strea /* Start coordinates and opposite corner */ for (loop_indx = 0; loop_indx < ndims; loop_indx++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, loop_indx ? "," : "(", + h5tools_str_append(buffer, "%s%" PRIuHSIZE, loop_indx ? "," : "(", ptdata[indx * 2 * ndims + loop_indx]); for (loop_indx = 0; loop_indx < ndims; loop_indx++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, loop_indx ? "," : ")-(", + h5tools_str_append(buffer, "%s%" PRIuHSIZE, loop_indx ? "," : ")-(", ptdata[indx * 2 * ndims + loop_indx + ndims]); h5tools_str_append(buffer, ")"); @@ -896,7 +896,7 @@ h5tools_print_region_data_points(hid_t region_space, hid_t region_id, FILE *stre { hbool_t dimension_break = TRUE; hsize_t * dims1 = NULL; - hsize_t elmtno; /* elemnt index */ + hsize_t elmtno; /* element index */ hsize_t curr_pos = 0; hsize_t total_size[H5S_MAX_RANK]; hsize_t jndx; @@ -1101,7 +1101,7 @@ h5tools_dump_region_data_points(hid_t region_space, hid_t region_id, FILE *strea (unsigned long)indx); for (loop_indx = 0; loop_indx < ndims; loop_indx++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, loop_indx ? "," : "(", + h5tools_str_append(buffer, "%s%" PRIuHSIZE, loop_indx ? "," : "(", ptdata[indx * ndims + loop_indx]); h5tools_str_append(buffer, ")"); @@ -1286,7 +1286,7 @@ h5tools_print_simple_subset(FILE *stream, const h5tool_format_t *info, h5tools_c H5TOOLS_START_DEBUG(" "); if ((size_t)ctx->ndims > NELMTS(sm_size)) - H5TOOLS_THROW(FAIL, "ndims and sm_size comparision failed"); + H5TOOLS_THROW(FAIL, "ndims and sm_size comparison failed"); size_row_block = ctx->sset->block.data[row_dim]; @@ -2486,7 +2486,7 @@ found_string_type: ctx->need_prefix = TRUE; h5tools_str_reset(buffer); - h5tools_str_append(buffer, "OPAQUE_SIZE \"%s\";", size); + h5tools_str_append(buffer, "OPAQUE_SIZE \"%zu\";", size); h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); } @@ -2608,7 +2608,7 @@ found_string_type: if (H5Tget_array_dims2(type, dims) >= 0) { /* Print array dimensions */ for (i = 0; i < ndims; i++) - h5tools_str_append(buffer, "[" HSIZE_T_FORMAT "]", dims[i]); + h5tools_str_append(buffer, "[%" PRIuHSIZE "]", dims[i]); h5tools_str_append(buffer, " "); } @@ -2683,12 +2683,12 @@ h5tools_print_dataspace(h5tools_str_t *buffer, hid_t space) case H5S_SIMPLE: /* simple dataspace */ - h5tools_str_append(buffer, "%s %s { %s " HSIZE_T_FORMAT, + h5tools_str_append(buffer, "%s %s { %s %" PRIuHSIZE, h5tools_dump_header_format->dataspacedescriptionbegin, S_SIMPLE, h5tools_dump_header_format->dataspacedimbegin, size[0]); for (i = 1; i < ndims; i++) - h5tools_str_append(buffer, ", " HSIZE_T_FORMAT, size[i]); + h5tools_str_append(buffer, ", %" PRIuHSIZE, size[i]); h5tools_str_append(buffer, " %s / ", h5tools_dump_header_format->dataspacedimend); @@ -2696,14 +2696,14 @@ h5tools_print_dataspace(h5tools_str_t *buffer, hid_t space) h5tools_str_append(buffer, "%s %s", h5tools_dump_header_format->dataspacedimbegin, "H5S_UNLIMITED"); else - h5tools_str_append(buffer, "%s " HSIZE_T_FORMAT, - h5tools_dump_header_format->dataspacedimbegin, maxsize[0]); + h5tools_str_append(buffer, "%s %" PRIuHSIZE, h5tools_dump_header_format->dataspacedimbegin, + maxsize[0]); for (i = 1; i < ndims; i++) if (maxsize[i] == H5S_UNLIMITED) h5tools_str_append(buffer, ", %s", "H5S_UNLIMITED"); else - h5tools_str_append(buffer, ", " HSIZE_T_FORMAT, maxsize[i]); + h5tools_str_append(buffer, ", %" PRIuHSIZE, maxsize[i]); h5tools_str_append(buffer, " %s }", h5tools_dump_header_format->dataspacedimend); break; @@ -2742,7 +2742,6 @@ h5tools_print_enum(FILE *stream, h5tools_str_t *buffer, const h5tool_format_t *i { char ** name = NULL; /*member names */ unsigned char *value = NULL; /*value array */ - unsigned char *copy = NULL; /*a pointer to value array */ unsigned i; unsigned nmembs = 0; /*number of members */ int snmembs; @@ -2834,16 +2833,16 @@ h5tools_print_enum(FILE *stream, h5tools_str_t *buffer, const h5tool_format_t *i h5tools_str_append(buffer, "%02x", value[i * dst_size + j]); } else if (H5T_SGN_NONE == H5Tget_sign(native)) { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, HSIZE_T_FORMAT, *((unsigned long long *)((void *)copy))); + unsigned long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%llu", copy); } else { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, "%" H5_PRINTF_LL_WIDTH "d", *((long long *)((void *)copy))); + long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%lld", copy); } h5tools_str_append(buffer, ";"); @@ -2996,7 +2995,7 @@ h5tools_dump_oid(FILE *stream, const h5tool_format_t *info, h5tools_context_t *c ctx->need_prefix = TRUE; h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "%s %s %d %s", OBJID, BEGIN, oid, END); + h5tools_str_append(&buffer, "%s %s %" PRId64 " %s", OBJID, BEGIN, oid, END); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); h5tools_str_close(&buffer); @@ -3174,10 +3173,10 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * h5tools_str_append(&buffer, "%s ", CHUNKED); rank = H5Pget_chunk(dcpl_id, (int)NELMTS(chsize), chsize); - h5tools_str_append(&buffer, "%s " HSIZE_T_FORMAT, h5tools_dump_header_format->dataspacedimbegin, + h5tools_str_append(&buffer, "%s %" PRIuHSIZE, h5tools_dump_header_format->dataspacedimbegin, chsize[0]); for (i = 1; i < rank; i++) - h5tools_str_append(&buffer, ", " HSIZE_T_FORMAT, chsize[i]); + h5tools_str_append(&buffer, ", %" PRIuHSIZE, chsize[i]); h5tools_str_append(&buffer, " %s", h5tools_dump_header_format->dataspacedimend); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3219,17 +3218,17 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * if (storage_size != 0) ratio = (double)uncomp_size / (double)storage_size; - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT " (%.3f:1 COMPRESSION)", storage_size, + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE " (%.3f:1 COMPRESSION)", storage_size, ratio); } else - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); H5Sclose(sid); H5Tclose(tid); } else { - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); } h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3247,7 +3246,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * ctx->need_prefix = TRUE; h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); ctx->indent_level--; @@ -3283,7 +3282,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * ctx->need_prefix = TRUE; h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "FILENAME %s SIZE " HSIZE_T_FORMAT, name, size); + h5tools_str_append(&buffer, "FILENAME %s SIZE %" PRIuHSIZE, name, size); h5tools_str_append(&buffer, " OFFSET %ld", offset); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3313,7 +3312,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * ctx->need_prefix = TRUE; h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3340,7 +3339,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * } break; case H5D_VIRTUAL: { - char dsetname[256]; /* virtual datset name */ + char dsetname[256]; /* virtual dataset name */ size_t n_vmaps; H5Pget_virtual_count(dcpl_id, &n_vmaps); @@ -3909,7 +3908,7 @@ h5tools_print_dims(h5tools_str_t *buffer, hsize_t *s, int dims) int i; for (i = 0; i < dims; i++) { - h5tools_str_append(buffer, HSIZE_T_FORMAT, s[i]); + h5tools_str_append(buffer, "%" PRIuHSIZE, s[i]); if (i + 1 != dims) h5tools_str_append(buffer, ", "); @@ -4330,7 +4329,7 @@ h5tools_dump_data(FILE *stream, const h5tool_format_t *info, h5tools_context_t * if (string_dataformat.pindex) { string_dataformat.idx_fmt = "(%s): "; - string_dataformat.idx_n_fmt = HSIZE_T_FORMAT; + string_dataformat.idx_n_fmt = "%" PRIuHSIZE; string_dataformat.idx_sep = ","; string_dataformat.line_pre = "%s"; } diff --git a/tools/lib/h5tools_ref.c b/tools/lib/h5tools_ref.c index 50f283d..1c97a52 100644 --- a/tools/lib/h5tools_ref.c +++ b/tools/lib/h5tools_ref.c @@ -70,7 +70,7 @@ free_ref_path_info(void *item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED *op /*------------------------------------------------------------------------- * Function: init_ref_path_cb * - * Purpose: Called by interator to create references for + * Purpose: Called by iterator to create references for * all objects and enter them in the table. * * Return: Error status. @@ -123,7 +123,7 @@ ref_path_table_cmp(const void *key1, const void *key2) /*------------------------------------------------------------------------- * Function: init_ref_path_table * - * Purpose: Initalize the reference path table + * Purpose: Initialize the reference path table * * Return: Non-negative on success, negative on failure * @@ -377,7 +377,7 @@ lookup_ref_path(H5R_ref_t refbuf) /*------------------------------------------------------------------------- * Function: fill_ref_path_table * - * Purpose: Called by interator to create references for + * Purpose: Called by iterator to create references for * all objects and enter them in the table. * * Return: Error status. diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index 3cd12bb..1fb2c68 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -129,7 +129,7 @@ h5tools_str_append(h5tools_str_t *str /*in,out*/, const char *fmt, ...) /* Note: HDvsnprintf() behaves differently on Windows as Unix, when * buffer is smaller than source string. On Unix, this function - * returns length of the source string and copy string upto the + * returns length of the source string and copy string up to the * buffer size with NULL at the end of the buffer. However on * Windows with the same condition, this function returns -1 and * doesn't add NULL at the end of the buffer. @@ -147,7 +147,7 @@ h5tools_str_append(h5tools_str_t *str /*in,out*/, const char *fmt, ...) * following conditions, each of which indicates that the proper C99 return value probably * should have been positive when the format string is * something other than "%s" - * Alocate at least twice as much space and try again. + * Allocate at least twice as much space and try again. */ size_t newsize = MAX(str->len + (size_t)nchars + 1, 2 * str->nalloc); HDassert(newsize > str->nalloc); /*overflow*/ @@ -300,11 +300,11 @@ h5tools_str_prefix(h5tools_str_t *str /*in,out*/, const h5tool_format_t *info, h if (i) h5tools_str_append(str, "%s", OPT(info->idx_sep, ",")); - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)ctx->pos[i]); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)ctx->pos[i]); } } else /* Scalar */ - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)elmtno); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)elmtno); H5TOOLS_DEBUG("str=%s", str->s); H5TOOLS_ENDDEBUG(" "); @@ -341,11 +341,11 @@ h5tools_str_region_prefix(h5tools_str_t *str /*in,out*/, const h5tool_format_t * if (i) h5tools_str_append(str, "%s", OPT(info->idx_sep, ",")); - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)ctx->pos[i]); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)ctx->pos[i]); } } else /* Scalar */ - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)0); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)0); H5TOOLS_DEBUG("str=%s", str->s); H5TOOLS_ENDDEBUG(" "); @@ -385,7 +385,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form /* Start coordinates */ h5tools_str_append(str, "%s%s ", info->line_indent, START); for (j = 0; j < ndims; j++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", start[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", start[j]); h5tools_str_append(str, ")"); h5tools_str_append(str, "%s", "\n"); h5tools_str_indent(str, info, ctx); @@ -393,7 +393,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form /* Stride coordinates */ h5tools_str_append(str, "%s ", STRIDE); for (j = 0; j < ndims; j++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", stride[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", stride[j]); h5tools_str_append(str, ")"); h5tools_str_append(str, "%s", "\n"); h5tools_str_indent(str, info, ctx); @@ -404,7 +404,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form if (count[j] == H5S_UNLIMITED) h5tools_str_append(str, "%s%s", j ? "," : "(", "H5S_UNLIMITED"); else - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", count[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", count[j]); } h5tools_str_append(str, ")"); h5tools_str_append(str, "%s", "\n"); @@ -416,7 +416,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form if (block[j] == H5S_UNLIMITED) h5tools_str_append(str, "%s%s", j ? "," : "(", "H5S_UNLIMITED"); else - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", block[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", block[j]); } h5tools_str_append(str, ")"); } @@ -468,11 +468,10 @@ h5tools_str_dump_space_blocks(h5tools_str_t *str, hid_t rspace, const h5tool_for /* Start coordinates and opposite corner */ for (v = 0; v < ndims; v++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, v ? "," : "(", ptdata[u * 2 * ndims + v]); + h5tools_str_append(str, "%s%" PRIuHSIZE, v ? "," : "(", ptdata[u * 2 * ndims + v]); for (v = 0; v < ndims; v++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, v ? "," : ")-(", - ptdata[u * 2 * ndims + v + ndims]); + h5tools_str_append(str, "%s%" PRIuHSIZE, v ? "," : ")-(", ptdata[u * 2 * ndims + v + ndims]); h5tools_str_append(str, ")"); } @@ -527,7 +526,7 @@ h5tools_str_dump_space_points(h5tools_str_t *str, hid_t rspace, const h5tool_for (unsigned long)u); for (v = 0; v < ndims; v++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, v ? "," : "(", (ptdata[u * ndims + v])); + h5tools_str_append(str, "%s%" PRIuHSIZE, v ? "," : "(", (ptdata[u * ndims + v])); h5tools_str_append(str, ")"); } @@ -1246,7 +1245,7 @@ h5tools_str_sprint(h5tools_str_t *str, const h5tool_format_t *info, hid_t contai /* Print the closing bracket */ h5tools_str_append(str, "%s", OPT(info->arr_suf, "]")); - is_next_arry_elmt = 1; /* set for begining of next array element */ + is_next_arry_elmt = 1; /* set for beginning of next array element */ H5Tclose(memb); } break; @@ -1504,7 +1503,7 @@ h5tools_str_is_zero(const void *_mem, size_t size) * March 8, 2012 * * Notes: - * Applications need to call free() to free the memoery allocated for + * Applications need to call free() to free the memory allocated for * the return string * *------------------------------------------------------------------------- diff --git a/tools/lib/h5tools_str.h b/tools/lib/h5tools_str.h index eee87c9..f4610f0 100644 --- a/tools/lib/h5tools_str.h +++ b/tools/lib/h5tools_str.h @@ -26,7 +26,7 @@ typedef struct h5tools_str_t { H5TOOLS_DLL void h5tools_str_close(h5tools_str_t *str); H5TOOLS_DLL size_t h5tools_str_len(h5tools_str_t *str); -H5TOOLS_DLL char * h5tools_str_append(h5tools_str_t *str, const char *fmt, ...); +H5TOOLS_DLL char * h5tools_str_append(h5tools_str_t *str, const char *fmt, ...) H5_ATTR_FORMAT(printf, 2, 3); H5TOOLS_DLL char * h5tools_str_reset(h5tools_str_t *str); H5TOOLS_DLL char * h5tools_str_trunc(h5tools_str_t *str, size_t size); H5TOOLS_DLL char * h5tools_str_fmt(h5tools_str_t *str, size_t start, const char *fmt); diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index ea9812b..66d3ef6 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -180,7 +180,7 @@ help_ref_msg(FILE *output) * * Last two characters in the string MUST be ")\0". * * * Generates a copy of the input string `start`, (src..")\0"), replacing - * separators and close-paren with null charaters. + * separators and close-paren with null characters. * * This string is allocated at runtime and should be freed when done. * * Generates array of char pointers, and directs start of each element * (each pointer) into this copy. @@ -227,7 +227,7 @@ parse_tuple(const char *start, int sep, char **cpy_out, unsigned *nelems, char * char * elem_ptr = NULL; char * dest_ptr = NULL; unsigned elems_count = 0; - char ** elems = NULL; /* more like *elems[], but complier... */ + char ** elems = NULL; /* more like *elems[], but compiler... */ char ** elems_re = NULL; /* temporary pointer, for realloc */ char * cpy = NULL; herr_t ret_value = SUCCEED; @@ -337,7 +337,7 @@ parse_tuple(const char *start, int sep, char **cpy_out, unsigned *nelems, char * /* Found terminal, non-escaped close-paren. Last element. * Write null terminator to copy. * Advance source pointer to gently break from loop. - * Requred to prevent ")" from always being added to last element. + * Required to prevent ")" from always being added to last element. */ start++; } diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index 095ad6c..4cf4e40 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -70,11 +70,11 @@ H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for outputting * /* Definitions of useful routines */ H5TOOLS_DLL void indentation(unsigned); H5TOOLS_DLL void print_version(const char *progname); -H5TOOLS_DLL void parallel_print(const char *format, ...); +H5TOOLS_DLL void parallel_print(const char *format, ...) H5_ATTR_FORMAT(printf, 1, 2); H5TOOLS_DLL herr_t parse_tuple(const char *start, int sep, char **cpy_out, unsigned *nelems, char ***ptrs_out); -H5TOOLS_DLL void error_msg(const char *fmt, ...); -H5TOOLS_DLL void warn_msg(const char *fmt, ...); +H5TOOLS_DLL void error_msg(const char *fmt, ...) H5_ATTR_FORMAT(printf, 1, 2); +H5TOOLS_DLL void warn_msg(const char *fmt, ...) H5_ATTR_FORMAT(printf, 1, 2); H5TOOLS_DLL void help_ref_msg(FILE *output); H5TOOLS_DLL void free_table(table_t *table); #ifdef H5DUMP_DEBUG @@ -100,7 +100,7 @@ typedef enum toolname_t { TOOL__H5DUMP /* add as necessary */ } h5tool_toolname_t; -/* this struct can be used to differntiate among tools */ +/* this struct can be used to differentiate among tools */ typedef struct { h5tool_toolname_t toolname; int msg_mode; diff --git a/tools/lib/h5trav.c b/tools/lib/h5trav.c index d0c299f..cca36e4 100644 --- a/tools/lib/h5trav.c +++ b/tools/lib/h5trav.c @@ -53,11 +53,6 @@ typedef struct trav_path_op_data_t { const char *path; } trav_path_op_data_t; -/* format for hsize_t */ -#ifdef H5TRAV_PRINT_SPACE -#define HSIZE_T_FORMAT "%" H5_PRINTF_LL_WIDTH "u" -#endif /* H5TRAV_PRINT_SPACE */ - /*------------------------------------------------------------------------- * local functions *------------------------------------------------------------------------- @@ -534,7 +529,7 @@ trav_info_free(trav_info_t *info) /*------------------------------------------------------------------------- * Function: trav_table_visit_obj * - * Purpose: Callback for visiting object, with 'table' sructure + * Purpose: Callback for visiting object, with 'table' structure * * Return: 0 on success, * -1 on failure @@ -559,7 +554,7 @@ trav_table_visit_obj(const char *path, const H5O_info2_t *oinfo, const char *alr /*------------------------------------------------------------------------- * Function: trav_table_visit_lnk * - * Purpose: Callback for visiting link, with 'table' sructure + * Purpose: Callback for visiting link, with 'table' structure * * Return: 0 on success, * -1 on failure @@ -856,7 +851,7 @@ trav_attr(hid_t /* simple dataspace */ HDprintf(" {"); for (i = 0; i < ndims; i++) { - HDprintf("%s" HSIZE_T_FORMAT, i ? ", " : "", size[i]); + HDprintf("%s%" PRIuHSIZE, i ? ", " : "", size[i]); } HDprintf("}\n"); break; @@ -1127,7 +1122,7 @@ symlink_is_visited(symlink_trav_t *visited, H5L_type_t type, const char *file, c /* Look for symlink */ for (u = 0; u < visited->nused; u++) { /* Check for symlink values already in array */ - /* check type and path pair to distingush between symbolic links */ + /* check type and path pair to distinguish between symbolic links */ if ((visited->objs[u].type == type) && !HDstrcmp(visited->objs[u].path, path)) { /* if external link, file need to be matched as well */ if (visited->objs[u].type == H5L_TYPE_EXTERNAL) diff --git a/tools/lib/h5trav.h b/tools/lib/h5trav.h index 7a8a63d..58d347d 100644 --- a/tools/lib/h5trav.h +++ b/tools/lib/h5trav.h @@ -87,7 +87,7 @@ typedef struct trav_link_t { } trav_link_t; /*------------------------------------------------------------------------- - * struct to store basic info needed for the h5trav table traversal algorythm + * struct to store basic info needed for the h5trav table traversal algorithm *------------------------------------------------------------------------- */ diff --git a/tools/lib/io_timer.c b/tools/lib/io_timer.c index aec7e9b..ce21b3e 100644 --- a/tools/lib/io_timer.c +++ b/tools/lib/io_timer.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* changes: * rename pio_timer.c as io_timer.c; - * Removed pio_perf.h so that it is not dependant on it; + * Removed pio_perf.h so that it is not dependent on it; * Removed set_timer_type() and get_timer_type() since no one calls them; * Merged sio_timer.c into io_timer.c; */ diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c index 66d7ac7..8cc147a 100644 --- a/tools/libtest/h5tools_test_utils.c +++ b/tools/libtest/h5tools_test_utils.c @@ -431,7 +431,7 @@ test_parse_tuple(void) {"stuff keeps on going"}, }, { - "4-ple, escaped seperator", + "4-ple, escaped separator", "(elem0,elem1,el\\,em2,elem3)", /* "el\,em" */ ',', SUCCEED, @@ -701,7 +701,7 @@ test_populate_ros3_fa(void) } /* NULL region - * yeilds default fapl + * yields default fapl */ { H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; @@ -720,7 +720,7 @@ test_populate_ros3_fa(void) } /* empty region - * yeilds default fapl + * yields default fapl */ { H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; @@ -739,7 +739,7 @@ test_populate_ros3_fa(void) } /* region overflow - * yeilds default fapl + * yields default fapl */ { H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; @@ -781,7 +781,7 @@ test_populate_ros3_fa(void) } /* empty id (non-empty region, key) - * yeilds default fapl + * yields default fapl */ { H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; @@ -851,7 +851,7 @@ test_populate_ros3_fa(void) } /* empty key (non-empty region, id) - * yeilds authenticating fapl + * yields authenticating fapl */ { H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; @@ -870,7 +870,7 @@ test_populate_ros3_fa(void) } /* empty key, region (non-empty id) - * yeilds default fapl + * yields default fapl */ { H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; @@ -889,7 +889,7 @@ test_populate_ros3_fa(void) } /* empty key, id (non-empty region) - * yeilds default fapl + * yields default fapl */ { H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; diff --git a/tools/src/h5copy/h5copy.c b/tools/src/h5copy/h5copy.c index 4e9a25e..6f10fee 100644 --- a/tools/src/h5copy/h5copy.c +++ b/tools/src/h5copy/h5copy.c @@ -212,7 +212,7 @@ parse_flag(const char *s_flag, unsigned *flag) */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid_src = H5I_INVALID_HID; hid_t fid_dst = H5I_INVALID_HID; @@ -242,7 +242,7 @@ main(int argc, const char *argv[]) } /* end if */ /* parse command line options */ - while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, (const char *const *)argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'd': oname_dst = HDstrdup(H5_optarg); diff --git a/tools/src/h5diff/h5diff_common.c b/tools/src/h5diff/h5diff_common.c index b669087..1bf3994 100644 --- a/tools/src/h5diff/h5diff_common.c +++ b/tools/src/h5diff/h5diff_common.c @@ -200,7 +200,7 @@ parse_subset_params(const char *dset) */ void -parse_command_line(int argc, const char *argv[], const char **fname1, const char **fname2, +parse_command_line(int argc, const char *const *argv, const char **fname1, const char **fname2, const char **objname1, const char **objname2, diff_opt_t *opts) { int i; @@ -277,7 +277,7 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char break; case 'q': - /* use quiet mode; supress the message "0 differences found" */ + /* use quiet mode; suppress the message "0 differences found" */ opts->mode_quiet = 1; break; @@ -657,6 +657,24 @@ usage(void) PRINTVALSTREAM(rawoutstream, " --vol-info-2 VOL-specific info to pass to the VOL connector used for\n"); PRINTVALSTREAM(rawoutstream, " opening the second HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-value-1 Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " first HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-name-1 Name of the VFL driver to use for opening the first\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-info-1 VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the first HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-value-2 Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " second HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-name-2 Name of the VFL driver to use for opening the second\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-info-2 VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the second HDF5 file specified\n"); PRINTVALSTREAM(rawoutstream, " --follow-symlinks\n"); PRINTVALSTREAM(rawoutstream, " Follow symbolic links (soft links and external links and compare the)\n"); @@ -778,7 +796,7 @@ usage(void) PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " Modes of output:\n"); PRINTVALSTREAM(rawoutstream, - " Default mode: print the number of differences found and where they occured\n"); + " Default mode: print the number of differences found and where they occurred\n"); PRINTVALSTREAM(rawoutstream, " -r Report mode: print the above plus the differences\n"); PRINTVALSTREAM(rawoutstream, " -v Verbose mode: print the above plus a list of objects and warnings\n"); PRINTVALSTREAM(rawoutstream, " -q Quiet mode: do not print output\n"); diff --git a/tools/src/h5diff/h5diff_common.h b/tools/src/h5diff/h5diff_common.h index 83f4255..35e5dfb 100644 --- a/tools/src/h5diff/h5diff_common.h +++ b/tools/src/h5diff/h5diff_common.h @@ -23,7 +23,7 @@ extern "C" { #endif void usage(void); -void parse_command_line(int argc, const char *argv[], const char **fname1, const char **fname2, +void parse_command_line(int argc, const char *const *argv, const char **fname1, const char **fname2, const char **objname1, const char **objname2, diff_opt_t *opts); void h5diff_exit(int status); void print_info(diff_opt_t *opts); diff --git a/tools/src/h5diff/h5diff_main.c b/tools/src/h5diff/h5diff_main.c index a835953..2a43b56 100644 --- a/tools/src/h5diff/h5diff_main.c +++ b/tools/src/h5diff/h5diff_main.c @@ -33,7 +33,7 @@ * * Modifications: July 2004 * Introduced the four modes: - * Normal mode: print the number of differences found and where they occured + * Normal mode: print the number of differences found and where they occurred * Report mode: print the above plus the differences * Verbose mode: print the above plus a list of objects and warnings * Quiet mode: do not print output @@ -65,7 +65,7 @@ */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { int ret; int i; @@ -86,7 +86,7 @@ main(int argc, const char *argv[]) * process the command-line *------------------------------------------------------------------------- */ - parse_command_line(argc, argv, &fname1, &fname2, &objname1, &objname2, &opts); + parse_command_line(argc, (const char *const *)argv, &fname1, &fname2, &objname1, &objname2, &opts); /* enable error reporting if command line option */ h5tools_error_report(); diff --git a/tools/src/h5diff/ph5diff_main.c b/tools/src/h5diff/ph5diff_main.c index ec78e49..8e5c3e8 100644 --- a/tools/src/h5diff/ph5diff_main.c +++ b/tools/src/h5diff/ph5diff_main.c @@ -44,7 +44,7 @@ static void ph5diff_worker(int); */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { int nID = 0; const char *fname1 = NULL; @@ -72,7 +72,7 @@ main(int argc, const char *argv[]) g_Parallel = 0; - parse_command_line(argc, argv, &fname1, &fname2, &objname1, &objname2, &opts); + parse_command_line(argc, (const char *const *)argv, &fname1, &fname2, &objname1, &objname2, &opts); h5diff(fname1, fname2, objname1, objname2, &opts); @@ -83,7 +83,8 @@ main(int argc, const char *argv[]) /* Have the manager process the command-line */ if (nID == 0) { - parse_command_line(argc, argv, &fname1, &fname2, &objname1, &objname2, &opts); + parse_command_line(argc, (const char *const *)argv, &fname1, &fname2, &objname1, &objname2, + &opts); h5diff(fname1, fname2, objname1, objname2, &opts); diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index 2901241..20ff29e 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -196,6 +196,14 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " --vol-info VOL-specific info to pass to the VOL connector used for\n"); PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-value Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, " --vfd-name Name of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-info VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n"); PRINTVALSTREAM(rawoutstream, "--------------- Object Options ---------------\n"); PRINTVALSTREAM(rawoutstream, " -a P, --attribute=P Print the specified attribute\n"); PRINTVALSTREAM(rawoutstream, @@ -816,7 +824,7 @@ free_handler(struct handler_t *hand, int len) *------------------------------------------------------------------------- */ static struct handler_t * -parse_command_line(int argc, const char *argv[]) +parse_command_line(int argc, const char *const *argv) { struct handler_t *hand = NULL; struct handler_t *last_dset = NULL; @@ -1298,7 +1306,7 @@ error: *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid = H5I_INVALID_HID; hid_t gid = H5I_INVALID_HID; @@ -1318,7 +1326,7 @@ main(int argc, const char *argv[]) /* Initialize h5tools lib */ h5tools_init(); - if ((hand = parse_command_line(argc, argv)) == NULL) { + if ((hand = parse_command_line(argc, (const char *const *)argv)) == NULL) { goto done; } @@ -1474,7 +1482,7 @@ main(int argc, const char *argv[]) dset_table = table_list.tables[0].dset_table; type_table = table_list.tables[0].type_table; - /* does there exist unamed committed datatype */ + /* does there exist unnamed committed datatype */ for (u = 0; u < type_table->nobjs; u++) if (!type_table->objs[u].recorded) { unamedtype = 1; diff --git a/tools/src/h5dump/h5dump_ddl.c b/tools/src/h5dump/h5dump_ddl.c index 638a738..b0bce0c 100644 --- a/tools/src/h5dump/h5dump_ddl.c +++ b/tools/src/h5dump/h5dump_ddl.c @@ -829,7 +829,7 @@ dump_group(hid_t gid, const char *name) if (!HDstrcmp(name, "/") && unamedtype) { unsigned u; /* Local index variable */ - /* dump unamed type in root group */ + /* dump unnamed type in root group */ for (u = 0; u < type_table->nobjs; u++) if (!type_table->objs[u].recorded) { char *obj_tok_str = NULL; @@ -838,7 +838,7 @@ dump_group(hid_t gid, const char *name) type = H5Dget_type(dset); H5Otoken_to_str(dset, &type_table->objs[u].obj_token, &obj_tok_str); - HDsprintf(type_name, "#%s", obj_tok_str); + HDsnprintf(type_name, sizeof(type_name), "#%s", obj_tok_str); H5free_memory(obj_tok_str); dump_function_table->dump_named_datatype_function(type, type_name); @@ -1283,7 +1283,7 @@ dump_fcontents(hid_t fid) { PRINTSTREAM(rawoutstream, "%s %s\n", FILE_CONTENTS, BEGIN); - /* special case of unamed types in root group */ + /* special case of unnamed types in root group */ if (unamedtype) { unsigned u; @@ -1318,7 +1318,7 @@ attr_search(hid_t oid, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *a j = (int)HDstrlen(op_name) - 1; /* find the last / */ while (j >= 0) { - if (op_name[j] == '/' && (j == 0 || (j > 0 && op_name[j - 1] != '\\'))) + if (op_name[j] == '/' && (j == 0 || (op_name[j - 1] != '\\'))) break; j--; } @@ -1541,7 +1541,7 @@ handle_attributes(hid_t fid, const char *attr, void H5_ATTR_UNUSED *data, int H5 /* find the last / */ while (j >= 0) { - if (attr[j] == '/' && (j == 0 || (j > 0 && attr[j - 1] != '\\'))) + if (attr[j] == '/' && (j == 0 || (attr[j - 1] != '\\'))) break; j--; } @@ -1959,7 +1959,7 @@ handle_datatypes(hid_t fid, const char *type, void H5_ATTR_UNUSED *data, int pe, const char *real_name = display_name ? display_name : type; if ((type_id = H5Topen2(fid, type, H5P_DEFAULT)) < 0) { - /* check if type is unamed datatype */ + /* check if type is unnamed datatype */ unsigned idx = 0; while (idx < type_table->nobjs) { @@ -1968,9 +1968,9 @@ handle_datatypes(hid_t fid, const char *type, void H5_ATTR_UNUSED *data, int pe, if (!type_table->objs[idx].recorded) { char *obj_tok_str = NULL; - /* unamed datatype */ + /* unnamed datatype */ H5Otoken_to_str(fid, &type_table->objs[idx].obj_token, &obj_tok_str); - HDsprintf(name, "/#%s", obj_tok_str); + HDsnprintf(name, sizeof(name), "/#%s", obj_tok_str); H5free_memory(obj_tok_str); if (!HDstrcmp(name, real_name)) diff --git a/tools/src/h5dump/h5dump_xml.c b/tools/src/h5dump/h5dump_xml.c index 0e881df..827daf3 100644 --- a/tools/src/h5dump/h5dump_xml.c +++ b/tools/src/h5dump/h5dump_xml.c @@ -23,7 +23,7 @@ const char *xmlnsprefix = "hdf5:"; /* - * Alternative formating for data dumped to XML + * Alternative formatting for data dumped to XML * In general, the numbers are the same, but separators * except spaces are not used. * @@ -591,6 +591,8 @@ xml_name_to_XID(hid_t loc_id, const char *str, char *outstr, int outlen, int gen if (outlen < 22) return 1; + H5_CHECK_OVERFLOW(outlen, int, size_t); + lookup_ret = ref_path_table_lookup(str, &obj_token); if (lookup_ret < 0) { if (HDstrlen(str) == 0) { @@ -600,7 +602,7 @@ xml_name_to_XID(hid_t loc_id, const char *str, char *outstr, int outlen, int gen ref_path_table_gen_fake(str, &obj_token); H5Otoken_to_str(loc_id, &obj_token, &obj_tok_str); - HDsprintf(outstr, "xid_%s", obj_tok_str); + HDsnprintf(outstr, (size_t)outlen, "xid_%s", obj_tok_str); H5free_memory(obj_tok_str); return 0; @@ -615,7 +617,7 @@ xml_name_to_XID(hid_t loc_id, const char *str, char *outstr, int outlen, int gen ref_path_table_gen_fake(str, &obj_token); H5Otoken_to_str(loc_id, &obj_token, &obj_tok_str); - HDsprintf(outstr, "xid_%s", obj_tok_str); + HDsnprintf(outstr, (size_t)outlen, "xid_%s", obj_tok_str); H5free_memory(obj_tok_str); return 0; @@ -627,7 +629,7 @@ xml_name_to_XID(hid_t loc_id, const char *str, char *outstr, int outlen, int gen } H5Otoken_to_str(loc_id, &obj_token, &obj_tok_str); - HDsprintf(outstr, "xid_%s", obj_tok_str); + HDsnprintf(outstr, (size_t)outlen, "xid_%s", obj_tok_str); H5free_memory(obj_tok_str); return 0; @@ -1777,8 +1779,7 @@ xml_dump_dataspace(hid_t space) /* Render the element */ h5tools_str_reset(&buffer); h5tools_str_append(&buffer, - "<%sDimension DimSize=\"%" H5_PRINTF_LL_WIDTH - "u\" MaxDimSize=\"UNLIMITED\"/>", + "<%sDimension DimSize=\"%" PRIuHSIZE "\" MaxDimSize=\"UNLIMITED\"/>", xmlnsprefix, size[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); @@ -1788,10 +1789,9 @@ xml_dump_dataspace(hid_t space) /* Render the element */ h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, - "<%sDimension DimSize=\"%" H5_PRINTF_LL_WIDTH - "u\" MaxDimSize=\"%" H5_PRINTF_LL_WIDTH "u\"/>", - xmlnsprefix, size[i], size[i]); + h5tools_str_append( + &buffer, "<%sDimension DimSize=\"%" PRIuHSIZE "\" MaxDimSize=\"%" PRIuHSIZE "\"/>", + xmlnsprefix, size[i], size[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); } @@ -1800,10 +1800,9 @@ xml_dump_dataspace(hid_t space) /* Render the element */ h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, - "<%sDimension DimSize=\"%" H5_PRINTF_LL_WIDTH - "u\" MaxDimSize=\"%" H5_PRINTF_LL_WIDTH "u\"/>", - xmlnsprefix, size[i], maxsize[i]); + h5tools_str_append( + &buffer, "<%sDimension DimSize=\"%" PRIuHSIZE "\" MaxDimSize=\"%" PRIuHSIZE "\"/>", + xmlnsprefix, size[i], maxsize[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); } @@ -2793,7 +2792,7 @@ xml_dump_group(hid_t gid, const char *name) if (isRoot && unamedtype) { unsigned u; - /* Very special case: dump unamed type in root group */ + /* Very special case: dump unnamed type in root group */ for (u = 0; u < type_table->nobjs; u++) { if (!type_table->objs[u].recorded) { char *obj_tok_str = NULL; @@ -2802,7 +2801,7 @@ xml_dump_group(hid_t gid, const char *name) type = H5Dget_type(dset); H5Otoken_to_str(dset, &type_table->objs[u].obj_token, &obj_tok_str); - HDsprintf(type_name, "#%s", obj_tok_str); + HDsnprintf(type_name, sizeof(type_name), "#%s", obj_tok_str); H5free_memory(obj_tok_str); dump_function_table->dump_named_datatype_function(type, type_name); @@ -2886,7 +2885,7 @@ xml_dump_group(hid_t gid, const char *name) if (isRoot && unamedtype) { unsigned u; - /* Very special case: dump unamed type in root group */ + /* Very special case: dump unnamed type in root group */ for (u = 0; u < type_table->nobjs; u++) { if (!type_table->objs[u].recorded) { char *obj_tok_str = NULL; @@ -2895,7 +2894,7 @@ xml_dump_group(hid_t gid, const char *name) type = H5Dget_type(dset); H5Otoken_to_str(dset, &type_table->objs[u].obj_token, &obj_tok_str); - HDsprintf(type_name, "#%s", obj_tok_str); + HDsnprintf(type_name, sizeof(type_name), "#%s", obj_tok_str); H5free_memory(obj_tok_str); dump_function_table->dump_named_datatype_function(type, type_name); @@ -2938,7 +2937,7 @@ xml_dump_group(hid_t gid, const char *name) /*------------------------------------------------------------------------- * Function: xml_print_refs * - * Purpose: Print a path to the objects referenced by HDF5 Referneces. + * Purpose: Print a path to the objects referenced by HDF5 References. * * Return: void * @@ -3598,7 +3597,10 @@ xml_dump_fill_value(hid_t dcpl, hid_t type) h5tools_str_reset(&buffer); h5tools_str_append(&buffer, "\""); for (i = 0; i < sz; i++) { - h5tools_str_append(&buffer, "%x ", *(unsigned int *)buf + (i * sizeof(unsigned int))); + unsigned long val = *(unsigned int *)buf + (i * sizeof(unsigned int)); + + H5_CHECK_OVERFLOW(val, unsigned long, unsigned); + h5tools_str_append(&buffer, "%x ", (unsigned)val); } h5tools_str_append(&buffer, "\""); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, @@ -3892,8 +3894,8 @@ xml_dump_dataset(hid_t did, const char *name, struct subset_t H5_ATTR_UNUSED *ss /* Render the element */ h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "<%sChunkDimension DimSize=\"%" H5_PRINTF_LL_WIDTH "u\" />", - xmlnsprefix, chsize[i]); + h5tools_str_append(&buffer, "<%sChunkDimension DimSize=\"%" PRIuHSIZE "\" />", xmlnsprefix, + chsize[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); } @@ -4524,12 +4526,16 @@ xml_print_enum(hid_t type) h5tools_str_append(&buffer, "%02x", value[i * dst_size + j]); } else if (H5T_SGN_NONE == H5Tget_sign(native)) { - h5tools_str_append(&buffer, "%" H5_PRINTF_LL_WIDTH "u", - *((unsigned long long *)((void *)(value + i * dst_size)))); + unsigned long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(&buffer, "%llu", copy); } else { - h5tools_str_append(&buffer, "%" H5_PRINTF_LL_WIDTH "d", - *((long long *)((void *)(value + i * dst_size)))); + long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(&buffer, "%lld", copy); } h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); diff --git a/tools/src/h5format_convert/h5format_convert.c b/tools/src/h5format_convert/h5format_convert.c index ddf129c..50d0f5d 100644 --- a/tools/src/h5format_convert/h5format_convert.c +++ b/tools/src/h5format_convert/h5format_convert.c @@ -94,7 +94,7 @@ usage(const char *prog) *------------------------------------------------------------------------- */ static int -parse_command_line(int argc, const char **argv) +parse_command_line(int argc, const char *const *argv) { int opt; @@ -127,7 +127,7 @@ parse_command_line(int argc, const char **argv) dname_g = HDstrdup(H5_optarg); if (dname_g == NULL) { h5tools_setstatus(EXIT_FAILURE); - error_msg("No dataset name\n", H5_optarg); + error_msg("No dataset name `%s`\n", H5_optarg); usage(h5tools_getprogname()); goto error; } @@ -383,7 +383,7 @@ error: *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid = H5I_INVALID_HID; @@ -394,7 +394,7 @@ main(int argc, const char *argv[]) h5tools_init(); /* Parse command line options */ - if (parse_command_line(argc, argv) < 0) + if (parse_command_line(argc, (const char *const *)argv) < 0) goto done; else if (verbose_g) HDfprintf(stdout, "Process command line options\n"); diff --git a/tools/src/h5import/h5import.c b/tools/src/h5import/h5import.c index 7272dff..317db81 100644 --- a/tools/src/h5import/h5import.c +++ b/tools/src/h5import/h5import.c @@ -2126,7 +2126,7 @@ processConfigurationFile(char *infile, struct Input *in) } } /* while (get_next_prop) */ } /* else if(!HDstrcmp("SUBSET", key)) */ - else if (!HDstrcmp("DATA", key)) { /* FINSHED */ + else if (!HDstrcmp("DATA", key)) { /* FINISHED */ #ifdef H5DEBUGIMPORT HDprintf("h5dump DATA key\n"); #endif @@ -2320,7 +2320,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err12a, infile); goto error; } - /* cant appear before dimension sizes have been provided */ + /* can't appear before dimension sizes have been provided */ if (in->configOptionVector[DIM] == 0) { (void)HDfprintf(stderr, err12b, infile); goto error; @@ -2387,7 +2387,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err16a, infile); goto error; } - /* cant appear before dimension sizes have been provided */ + /* can't appear before dimension sizes have been provided */ if (in->configOptionVector[DIM] == 0) { (void)HDfprintf(stderr, err16b, infile); goto error; @@ -2474,14 +2474,14 @@ validateConfigurationParameters(struct Input *in) } } - /* Arch cant be STD if O/p class is FP */ + /* Arch can't be STD if O/p class is FP */ if (in->outputArchitecture == 1) if (in->outputClass == 1) { (void)HDfprintf(stderr, "%s", err4a); return (-1); } - /* Arch cant be IEEE if O/p class is IN */ + /* Arch can't be IEEE if O/p class is IN */ if (in->outputArchitecture == 2) if (in->outputClass == 0) { (void)HDfprintf(stderr, "%s", err4b); @@ -3725,7 +3725,7 @@ getCompressionParameter(struct Input *in, FILE *strm) int ival; const char *err1 = "Unable to get integer value.\n"; - const char *err2 = "Invalid value for compression paramter.\n"; + const char *err2 = "Invalid value for compression parameter.\n"; const char *err3 = "Unsupported Compression Type.\n"; switch (in->compressionType) { @@ -3784,7 +3784,7 @@ setDefaultValues(struct Input *in, int count) in->path.count = 1; HDstrcpy(temp, "dataset"); - HDsprintf(num, "%d", count); + HDsnprintf(num, sizeof(num), "%d", count); HDstrcat(temp, num); HDstrcpy(in->path.group[0], temp); @@ -4689,7 +4689,7 @@ process(struct Options *opt) } if (in->configOptionVector[EXTERNALSTORE] == 1) { - /* creating the external file if it doesnt exist */ + /* creating the external file if it doesn't exist */ if ((extfile = HDfopen(in->externFilename, "ab")) == NULL) { (void)HDfprintf(stderr, "%s", err4); H5Pclose(proplist); diff --git a/tools/src/h5jam/h5jam.c b/tools/src/h5jam/h5jam.c index 7f3385c..45978bb 100644 --- a/tools/src/h5jam/h5jam.c +++ b/tools/src/h5jam/h5jam.c @@ -22,7 +22,7 @@ herr_t write_pad(int ofile, hsize_t old_where, hsize_t *new_where); hsize_t compute_user_block_size(hsize_t); hsize_t copy_some_to_file(int, int, hsize_t, hsize_t, ssize_t); -void parse_command_line(int, const char *[]); +void parse_command_line(int, const char *const *); int do_clobber = FALSE; char *output_file = NULL; @@ -104,7 +104,7 @@ leave(int ret) */ void -parse_command_line(int argc, const char *argv[]) +parse_command_line(int argc, const char *const *argv) { int opt = FALSE; @@ -149,7 +149,7 @@ parse_command_line(int argc, const char *argv[]) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { int ufid = -1; int h5fid = -1; @@ -174,7 +174,7 @@ main(int argc, const char *argv[]) /* Initialize h5tools lib */ h5tools_init(); - parse_command_line(argc, argv); + parse_command_line(argc, (const char *const *)argv); /* enable error reporting if command line option */ h5tools_error_report(); @@ -358,7 +358,7 @@ done: * Purpose: Copy part of the input file to output. * infid: fd of file to read * outfid: fd of file to write - * startin: offset of where to read from infid + * starting: offset of where to read from infid * startout: offset of where to write to outfid * limit: bytes to read/write * @@ -373,7 +373,7 @@ done: *------------------------------------------------------------------------- */ hsize_t -copy_some_to_file(int infid, int outfid, hsize_t startin, hsize_t startout, ssize_t limit) +copy_some_to_file(int infid, int outfid, hsize_t starting, hsize_t startout, ssize_t limit) { char buf[1024]; h5_stat_t sbuf; @@ -386,9 +386,9 @@ copy_some_to_file(int infid, int outfid, hsize_t startin, hsize_t startout, ssiz ssize_t toend; ssize_t fromend; - if (startin > startout) { + if (starting > startout) { /* this case is prohibited */ - error_msg("copy_some_to_file: panic: startin > startout?\n"); + error_msg("copy_some_to_file: panic: starting > startout?\n"); exit(EXIT_FAILURE); } /* end if */ @@ -409,7 +409,7 @@ copy_some_to_file(int infid, int outfid, hsize_t startin, hsize_t startout, ssiz return 0; toend = (ssize_t)startout + howmuch; - fromend = (ssize_t)startin + howmuch; + fromend = (ssize_t)starting + howmuch; if (howmuch > 512) { to = toend - 512; diff --git a/tools/src/h5jam/h5unjam.c b/tools/src/h5jam/h5unjam.c index fa23b06..8ff354e 100644 --- a/tools/src/h5jam/h5unjam.c +++ b/tools/src/h5jam/h5unjam.c @@ -92,7 +92,7 @@ usage(const char *prog) *------------------------------------------------------------------------- */ static int -parse_command_line(int argc, const char *argv[]) +parse_command_line(int argc, const char *const *argv) { int opt = FALSE; @@ -172,7 +172,7 @@ leave(int ret) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t ifile = H5I_INVALID_HID; hid_t plist = H5I_INVALID_HID; @@ -189,7 +189,7 @@ main(int argc, const char *argv[]) /* Initialize h5tools lib */ h5tools_init(); - if (EXIT_FAILURE == parse_command_line(argc, argv)) + if (EXIT_FAILURE == parse_command_line(argc, (const char *const *)argv)) goto done; /* enable error reporting if command line option */ diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index bffd12b..06f537d 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -32,7 +32,7 @@ #define NAME_BUF_SIZE 2048 /* - * Alternative formating for data dumped by H5LS + * Alternative formatting for data dumped by H5LS * * This table only affects H5LS output. */ @@ -78,9 +78,9 @@ static h5tool_format_t ls_dataformat = { ",", /*elmt_suf1 */ " ", /*elmt_suf2 */ - HSIZE_T_FORMAT, /*idx_n_fmt */ - ",", /*idx_sep */ - "(%s)", /*idx_fmt */ + "%" PRIuHSIZE, /*idx_n_fmt */ + ",", /*idx_sep */ + "(%s)", /*idx_fmt */ 65535, /*line_ncols */ /*standard default columns */ @@ -244,6 +244,12 @@ usage(void) PRINTVALSTREAM(rawoutstream, " --vol-info VOL-specific info to pass to the VOL connector used for\n"); PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, " --vfd-value Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, " --vfd-name Name of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, " --vfd-info VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n"); PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " file/OBJECT\n"); PRINTVALSTREAM(rawoutstream, " Each object consists of an HDF5 file name optionally followed by a\n"); @@ -940,8 +946,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) /* Print members */ for (i = 0; i < (unsigned)nmembs; i++) { - unsigned char *copy; /* a pointer to value array */ - int nchars; /* number of output characters */ + int nchars; /* number of output characters */ h5tools_str_append(buffer, "\n%*s", ind + 4, ""); nchars = print_string(buffer, name[i], TRUE); @@ -955,16 +960,16 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) h5tools_str_append(buffer, "%02x", value[i * dst_size + j]); } else if (H5T_SGN_NONE == H5Tget_sign(native)) { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, HSIZE_T_FORMAT, *((unsigned long long *)((void *)copy))); + unsigned long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%llu", copy); } else { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, "%lld", *((long long *)((void *)copy))); + long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%lld", copy); } } @@ -1183,13 +1188,13 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind) /* Print dimensions */ for (i = 0; i < ndims; i++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, i ? "," : "[", dims[i]); + h5tools_str_append(buffer, "%s%" PRIuHSIZE, i ? "," : "[", dims[i]); h5tools_str_append(buffer, "]"); HDfree(dims); } else - h5tools_str_append(buffer, " [SCALAR]\n", rawoutstream); + h5tools_str_append(buffer, " [SCALAR]\n"); /* Print parent type */ h5tools_str_append(buffer, " "); @@ -1701,7 +1706,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain /* simple dataspace */ h5tools_str_append(&buffer, " {"); for (i = 0; i < ndims; i++) { - h5tools_str_append(&buffer, "%s" HSIZE_T_FORMAT, i ? ", " : "", size[i]); + h5tools_str_append(&buffer, "%s%" PRIuHSIZE, i ? ", " : "", size[i]); nelmts *= size[i]; } h5tools_str_append(&buffer, "}\n"); @@ -1789,12 +1794,12 @@ dataset_list1(hid_t dset) ndims = H5Sget_simple_extent_dims(space, cur_size, max_size); h5tools_str_append(&buffer, " {"); for (i = 0; i < ndims; i++) { - h5tools_str_append(&buffer, "%s" HSIZE_T_FORMAT, i ? ", " : "", cur_size[i]); + h5tools_str_append(&buffer, "%s%" PRIuHSIZE, i ? ", " : "", cur_size[i]); if (max_size[i] == H5S_UNLIMITED) { h5tools_str_append(&buffer, "/%s", "Inf"); } else if (max_size[i] != cur_size[i] || verbose_g > 0) { - h5tools_str_append(&buffer, "/" HSIZE_T_FORMAT, max_size[i]); + h5tools_str_append(&buffer, "/%" PRIuHSIZE, max_size[i]); } } if (space_type == H5S_SCALAR) @@ -1868,10 +1873,10 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) h5tools_str_append(&buffer, " %-10s {", "Chunks:"); total = H5Tget_size(type); for (i = 0; i < ndims; i++) { - h5tools_str_append(&buffer, "%s" HSIZE_T_FORMAT, i ? ", " : "", chsize[i]); + h5tools_str_append(&buffer, "%s%" PRIuHSIZE, i ? ", " : "", chsize[i]); total *= chsize[i]; } - h5tools_str_append(&buffer, "} " HSIZE_T_FORMAT " bytes\n", total); + h5tools_str_append(&buffer, "} %" PRIuHSIZE " bytes\n", total); } break; case H5D_COMPACT: break; @@ -1897,15 +1902,13 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) if (H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, &f_offset, &f_size) < 0) { h5tools_str_append( - &buffer, - " #%03d %10" H5_PRINTF_LL_WIDTH "u %10s %10s ***ERROR*** %s\n", i, + &buffer, " #%03d %10" PRIuHSIZE " %10s %10s ***ERROR*** %s\n", i, total, "", "", i + 1 < nf ? "Following addresses are incorrect" : ""); } else if (H5S_UNLIMITED == f_size) { h5tools_str_append(&buffer, - " #%03d %10" H5_PRINTF_LL_WIDTH - "u %10" H5_PRINTF_LL_WIDTH "u %10s ", - i, total, (hsize_t)f_offset, "INF"); + " #%03d %10" PRIuHSIZE " %10" PRIuHSIZE " %10s ", i, + total, (hsize_t)f_offset, "INF"); print_string(&buffer, f_name, TRUE); } else { @@ -1985,8 +1988,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) case H5T_ARRAY: case H5T_NCLASSES: default: - h5tools_str_append(&buffer, - HSIZE_T_FORMAT " logical byte%s, " HSIZE_T_FORMAT " allocated byte%s", + h5tools_str_append(&buffer, "%" PRIuHSIZE " logical byte%s, %" PRIuHSIZE " allocated byte%s", total, 1 == total ? "" : "s", used, 1 == used ? "" : "s"); if (used > 0) { utilization = ((double)total * 100.0) / (double)used; @@ -2646,7 +2648,7 @@ leave(int ret) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t file_id = H5I_INVALID_HID; char * fname = NULL, *oname = NULL, *x = NULL; diff --git a/tools/src/h5perf/perf.c b/tools/src/h5perf/perf.c index 83d4ab0..84be7e8 100644 --- a/tools/src/h5perf/perf.c +++ b/tools/src/h5perf/perf.c @@ -119,7 +119,7 @@ static char *h5_fixname_real(const char *base_name, hid_t fapl, const char *_suf int main(int argc, char **argv) { - char * buf, *tmp, *buf2 = NULL, *tmp2 = NULL, *check; + char * buf = NULL, *tmp = NULL, *buf2 = NULL, *tmp2 = NULL, *check = NULL; int i, j, mynod = 0, nprocs = 1, my_correct = 1, correct, myerrno; double stim, etim; double write_tim = 0; @@ -605,7 +605,7 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fu * we are using the split driver since both of those * use the multi VFD under the hood. */ - env = HDgetenv("HDF5_DRIVER"); + env = HDgetenv(HDF5_DRIVER); #ifdef HDF5_DRIVER /* Use the environment variable, then the compile-time constant */ if (!env) diff --git a/tools/src/h5perf/pio_engine.c b/tools/src/h5perf/pio_engine.c index cac36d7..e316245 100644 --- a/tools/src/h5perf/pio_engine.c +++ b/tools/src/h5perf/pio_engine.c @@ -244,7 +244,7 @@ do_pio(parameters param) if (((size_t)(snbytes / pio_mpi_nprocs_g) % buf_size) != 0) { HDfprintf(stderr, "Dataset size/process (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size (%zu)\n", + "transfer buffer size (%zu)\n", (long long)(snbytes / pio_mpi_nprocs_g), buf_size); GOTOERROR(FAIL); } @@ -253,7 +253,7 @@ do_pio(parameters param) if (((size_t)snbytes % buf_size) != 0) { HDfprintf(stderr, "Dataset side size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size (%zu)\n", + "transfer buffer size (%zu)\n", (long long)snbytes, buf_size); GOTOERROR(FAIL); } @@ -282,7 +282,7 @@ do_pio(parameters param) /* Open file for write */ char base_name[256]; - HDsprintf(base_name, "#pio_tmp_%lu", nf); + HDsnprintf(base_name, sizeof(base_name), "#pio_tmp_%lu", nf); pio_create_filename(iot, base_name, fname, sizeof(fname)); if (pio_debug_level > 0) HDfprintf(output, "rank %d: data filename=%s\n", pio_mpi_rank_g, fname); @@ -898,7 +898,7 @@ do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nby } /* end if */ } /* end else */ - HDsprintf(dname, "Dataset_%ld", ndset); + HDsnprintf(dname, sizeof(dname), "Dataset_%ld", ndset); h5ds_id = H5DCREATE(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, h5dcpl); if (h5ds_id < 0) { @@ -1183,7 +1183,7 @@ do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nby /* Set the file view */ mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, - (char *)"native", h5_io_info_g); + "native", h5_io_info_g); VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); /* Perform write */ @@ -1320,7 +1320,7 @@ do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nby else { /* Set the file view */ mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, - (char *)"native", h5_io_info_g); + "native", h5_io_info_g); VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); /* Perform write */ @@ -1879,7 +1879,7 @@ do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbyt break; case PHDF5: - HDsprintf(dname, "Dataset_%ld", ndset); + HDsnprintf(dname, sizeof(dname), "Dataset_%ld", ndset); h5ds_id = H5DOPEN(fd->h5fd, dname); if (h5ds_id < 0) { HDfprintf(stderr, "HDF5 Dataset open failed\n"); @@ -2154,7 +2154,7 @@ do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbyt /* Set the file view */ mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, - (char *)"native", h5_io_info_g); + "native", h5_io_info_g); VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); /* Perform collective read */ @@ -2291,7 +2291,7 @@ do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbyt else { /* Set the file view */ mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, - (char *)"native", h5_io_info_g); + "native", h5_io_info_g); VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); /* Perform read */ @@ -2676,7 +2676,7 @@ do_cleanupfile(iotype iot, char *fname) return; if (clean_file_g == -1) - clean_file_g = (getenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; + clean_file_g = (getenv(HDF5_NOCLEANUP) == NULL) ? 1 : 0; if (clean_file_g) { switch (iot) { diff --git a/tools/src/h5perf/pio_perf.c b/tools/src/h5perf/pio_perf.c index 41245d9..7e12bed 100644 --- a/tools/src/h5perf/pio_perf.c +++ b/tools/src/h5perf/pio_perf.c @@ -188,7 +188,7 @@ typedef struct _minmax { /* local functions */ static off_t parse_size_directive(const char *size); -static struct options *parse_command_line(int argc, char *argv[]); +static struct options *parse_command_line(int argc, const char *const *argv); static void run_test_loop(struct options *options); static int run_test(iotype iot, parameters parms, struct options *opts); static void output_all_info(minmax *mm, int count, int indent_level); @@ -199,7 +199,7 @@ static int destroy_comm_world(void); static void output_results(const struct options *options, const char *name, minmax *table, int table_size, off_t data_size); static void output_times(const struct options *options, const char *name, minmax *table, int table_size); -static void output_report(const char *fmt, ...); +static void output_report(const char *fmt, ...) H5_ATTR_FORMAT(printf, 1, 2); static void print_indent(register int indent); static void usage(const char *prog); static void report_parameters(struct options *opts); @@ -260,7 +260,7 @@ main(int argc, char *argv[]) pio_comm_g = MPI_COMM_WORLD; h5_set_info_object(); - opts = parse_command_line(argc, argv); + opts = parse_command_line(argc, (const char *const *)argv); if (!opts) { exit_value = EXIT_FAILURE; @@ -579,7 +579,7 @@ run_test(iotype iot, parameters parms, struct options *opts) output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); } /* end if */ - /* show mpi write statics */ + /* show mpi write statistics */ if (pio_debug_level >= 3) { /* output all of the times for all iterations */ print_indent(3); @@ -645,7 +645,7 @@ run_test(iotype iot, parameters parms, struct options *opts) output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); } /* end if */ - /* show mpi read statics */ + /* show mpi read statistics */ if (pio_debug_level >= 3) { /* output all of the times for all iterations */ print_indent(3); @@ -904,7 +904,7 @@ accumulate_minmax_stuff(minmax *mm, int count) int i; minmax total_mm; - total_mm.sum = 0.0f; + total_mm.sum = 0.0; total_mm.max = -DBL_MAX; total_mm.min = DBL_MAX; total_mm.num = count; @@ -1276,7 +1276,7 @@ report_parameters(struct options *opts) * Added 2D testing (Christian Chilan, 10. August 2005) */ static struct options * -parse_command_line(int argc, char *argv[]) +parse_command_line(int argc, const char *const *argv) { register int opt; struct options *cl_opts; @@ -1305,7 +1305,7 @@ parse_command_line(int argc, char *argv[]) cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ cl_opts->verify = FALSE; /* No Verify data correctness by default */ - while ((opt = H5_get_option(argc, (const char **)argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'a': cl_opts->h5_alignment = parse_size_directive(H5_optarg); diff --git a/tools/src/h5perf/sio_engine.c b/tools/src/h5perf/sio_engine.c index e69a7cd..95ec3ce 100644 --- a/tools/src/h5perf/sio_engine.c +++ b/tools/src/h5perf/sio_engine.c @@ -179,7 +179,7 @@ do_sio(parameters param, results *res) if ((param.dset_size[i] % param.buf_size[i]) != 0) { HDfprintf(stderr, "Dataset size[%d] (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size[%d] (%zu)\n", + "transfer buffer size[%d] (%zu)\n", param.rank, (long long)param.dset_size[i], param.rank, param.buf_size[i]); GOTOERROR(FAIL); } @@ -1267,12 +1267,12 @@ done: static void do_cleanupfile(iotype iot, char *filename) { - char temp[2048]; + char temp[4096 + sizeof("-?.h5")]; int j; hid_t driver; if (clean_file_g == -1) - clean_file_g = (HDgetenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; + clean_file_g = (HDgetenv(HDF5_NOCLEANUP) == NULL) ? 1 : 0; if (clean_file_g) { diff --git a/tools/src/h5perf/sio_perf.c b/tools/src/h5perf/sio_perf.c index 1b200b0..87b9d65 100644 --- a/tools/src/h5perf/sio_perf.c +++ b/tools/src/h5perf/sio_perf.c @@ -164,7 +164,7 @@ typedef struct { /* local functions */ static hsize_t parse_size_directive(const char *size); -static struct options *parse_command_line(int argc, const char *argv[]); +static struct options *parse_command_line(int argc, const char *const *argv); static void run_test_loop(struct options *options); static int run_test(iotype iot, parameters parms, struct options *opts); static void output_all_info(minmax *mm, int count, int indent_level); @@ -185,7 +185,7 @@ static void report_parameters(struct options *opts); * Modifications: */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { int exit_value = EXIT_SUCCESS; struct options *opts = NULL; @@ -197,7 +197,7 @@ main(int argc, const char *argv[]) output = stdout; - opts = parse_command_line(argc, argv); + opts = parse_command_line(argc, (const char *const *)argv); if (!opts) { exit_value = EXIT_FAILURE; @@ -277,7 +277,7 @@ run_test_loop(struct options *opts) } /* print size information */ - output_report("Transfer Buffer Size (bytes): %d\n", buf_bytes); + output_report("Transfer Buffer Size (bytes): %zu\n", buf_bytes); output_report("File Size(MB): %.2f\n", ((double)parms.num_bytes) / ONE_MB); print_indent(0); @@ -310,14 +310,14 @@ run_test(iotype iot, parameters parms, struct options *opts) minmax * read_mm_table = NULL; minmax * read_gross_mm_table = NULL; minmax * read_raw_mm_table = NULL; - minmax write_sys_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_gross_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_raw_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_sys_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_gross_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_raw_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax write_sys_mm = {0.0, 0.0, 0.0, 0}; + minmax write_mm = {0.0, 0.0, 0.0, 0}; + minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_sys_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mm = {0.0, 0.0, 0.0, 0}; + minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; raw_size = (off_t)parms.num_bytes; parms.io_type = iot; @@ -427,7 +427,7 @@ run_test(iotype iot, parameters parms, struct options *opts) output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); } /* end if */ - /* show sys write statics */ + /* show sys write statistics */ #if 0 if (sio_debug_level >= 3) { /* output all of the times for all iterations */ @@ -473,7 +473,7 @@ run_test(iotype iot, parameters parms, struct options *opts) output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); } /* end if */ - /* show mpi read statics */ + /* show mpi read statistics */ #if 0 if (sio_debug_level >= 3) { /* output all of the times for all iterations */ @@ -574,7 +574,7 @@ accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm) { int i; - total_mm->sum = 0.0F; + total_mm->sum = 0.0; total_mm->max = -DBL_MAX; total_mm->min = DBL_MAX; total_mm->num = count; @@ -817,7 +817,7 @@ report_parameters(struct options *opts) * Added multidimensional testing (Christian Chilan, April, 2008) */ static struct options * -parse_command_line(int argc, const char *argv[]) +parse_command_line(int argc, const char *const *argv) { int opt; struct options *cl_opts; @@ -1278,7 +1278,7 @@ usage(const char *prog) HDprintf("\n"); HDprintf(" Example: 2K,2K,3K\n"); HDprintf("\n"); - HDprintf(" The example defines an object (dataset, tranfer buffer) with three\n"); + HDprintf(" The example defines an object (dataset, transfer buffer) with three\n"); HDprintf(" dimensions. Be aware that as the number of dimensions increases, the\n"); HDprintf(" the total size of the object increases exponentially.\n"); HDprintf("\n"); diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index 7cad36b..d0aad79 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -221,6 +221,7 @@ h5repack_addlayout(const char *str, pack_opt_t *options) * to free the stack. *------------------------------------------------------------------------- */ + hid_t copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options) @@ -354,24 +355,28 @@ done: int copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options) { - hid_t attr_id = H5I_INVALID_HID; /* attr ID */ - hid_t attr_out = H5I_INVALID_HID; /* attr ID */ - hid_t space_id = H5I_INVALID_HID; /* space ID */ - hid_t ftype_id = H5I_INVALID_HID; /* file type ID */ - hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ - size_t msize; /* size of type */ - void * buf = NULL; /* data buffer */ - hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ - htri_t is_named; /* Whether the datatype is named */ - hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ - char name[255]; - H5O_info2_t oinfo; /* object info */ - int j; - unsigned u; - hbool_t is_ref = 0; - H5T_class_t type_class = -1; - int ret_value = 0; + hid_t attr_id = H5I_INVALID_HID; /* attr ID */ + hid_t attr_out = H5I_INVALID_HID; /* attr ID */ + hid_t space_id = H5I_INVALID_HID; /* space ID */ + hid_t ftype_id = H5I_INVALID_HID; /* file type ID */ + hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ + size_t msize; /* size of type */ + void * buf = NULL; /* data buffer */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + htri_t is_named; /* Whether the datatype is named */ + hsize_t dims[H5S_MAX_RANK]; /* dimensions of dataset */ + H5_timer_t timer; /* Timer for read/write operations */ + H5_timevals_t times; /* Elapsed time for each operation */ + static double read_time = 0; + static double write_time = 0; + char name[255]; + H5O_info2_t oinfo; /* object info */ + int j; + unsigned u; + hbool_t is_ref = 0; + H5T_class_t type_class = -1; + int ret_value = 0; if (H5Oget_info3(loc_in, &oinfo, H5O_INFO_NUM_ATTRS) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Oget_info failed"); @@ -468,6 +473,9 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ } /* end for each member */ } /* end if type_class is H5T_COMPOUND */ + read_time = 0; + write_time = 0; + if (!is_ref) { /*----------------------------------------------------------------- * read to memory @@ -478,8 +486,17 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ if (buf == NULL) { H5TOOLS_GOTO_ERROR((-1), "HDmalloc failed"); } /* end if */ + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Aread(attr_id, wtype_id, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Aread failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + read_time += times.elapsed; + } /*----------------------------------------------------------------- * copy @@ -488,8 +505,18 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ if ((attr_out = H5Acreate2(loc_out, name, wtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Acreate2 failed on ,%s>", name); + + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Awrite(attr_out, wtype_id, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Awrite failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } /*close*/ if (H5Aclose(attr_out) < 0) @@ -499,11 +526,14 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_ * be reclaimed */ if (TRUE == h5tools_detect_vlen(wtype_id)) H5Treclaim(wtype_id, space_id, H5P_DEFAULT, buf); + HDfree(buf); buf = NULL; } /*H5T_REFERENCE*/ - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_ATTR_TIME, "attr", read_time, write_time, name); + else HDprintf(FORMAT_OBJ_ATTR, "attr", name); /*--------------------------------------------------------------------- @@ -569,7 +599,7 @@ check_options(pack_opt_t *options) * Objects to layout *------------------------------------------------------------------------- */ - if (options->verbose && have_request(options)) { + if (options->verbose > 0 && have_request(options)) { if (options->all_layout == 1) { HDprintf("All objects to modify layout are...\n"); switch (options->layout_g) { @@ -610,7 +640,7 @@ check_options(pack_opt_t *options) char *name = options->op_tbl->objs[i].path; if (options->op_tbl->objs[i].chunk.rank > 0) { - if (options->verbose) { + if (options->verbose > 0) { HDprintf(" <%s> with chunk size ", name); for (k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) HDprintf("%d ", (int)options->op_tbl->objs[i].chunk.chunk_lengths[k]); @@ -619,7 +649,7 @@ check_options(pack_opt_t *options) has_ck = 1; } else if (options->op_tbl->objs[i].chunk.rank == -2) { /* TODO: replace 'magic number' */ - if (options->verbose) + if (options->verbose > 0) HDprintf(" <%s> %s\n", name, "NONE (contiguous)"); has_ck = 1; } @@ -633,7 +663,7 @@ check_options(pack_opt_t *options) *------------------------------------------------------------------------- */ - if (options->verbose && have_request(options)) { + if (options->verbose > 0 && have_request(options)) { if (options->all_filter == 1) { HDprintf("All objects to apply filter are...\n"); for (k = 0; k < options->n_filter_g; k++) { @@ -670,7 +700,7 @@ check_options(pack_opt_t *options) char * name = pack.path; for (j = 0; j < pack.nfilters; j++) { - if (options->verbose) { + if (options->verbose > 0) { if (pack.filter[j].filtn >= 0) { if (pack.filter[j].filtn > H5Z_FILTER_SCALEOFFSET) { HDprintf(" <%s> with %s filter %d\n", name, get_sfilter(pack.filter[j].filtn), @@ -708,7 +738,7 @@ check_options(pack_opt_t *options) *------------------------------------------------------------------------ */ if (options->ublock_filename != NULL && options->ublock_size == 0) { - if (options->verbose) { + if (options->verbose > 0) { HDprintf("Warning: user block size missing for file %s. Assigning a default size of 1024...\n", options->ublock_filename); options->ublock_size = 1024; @@ -758,8 +788,8 @@ check_objects(const char *fname, pack_opt_t *options) * open the file *------------------------------------------------------------------------- */ - if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl, - (options->fin_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0) + if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), + NULL, 0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname, H5FOPENERROR); /*------------------------------------------------------------------------- @@ -781,21 +811,21 @@ check_objects(const char *fname, pack_opt_t *options) *------------------------------------------------------------------------- */ - if (options->verbose) + if (options->verbose > 0) HDprintf("Opening file. Searching %zu objects to modify ...\n", travt->nobjs); for (i = 0; i < options->op_tbl->nelems; i++) { pack_info_t obj = options->op_tbl->objs[i]; char * name = obj.path; - if (options->verbose) + if (options->verbose > 0) HDprintf(" <%s>", name); /* the input object names are present in the file and are valid */ if (h5trav_getindext(name, travt) < 0) H5TOOLS_GOTO_ERROR((-1), "%s Could not find <%s> in file <%s>. Exiting...\n", - (options->verbose ? "\n" : ""), name, fname); - if (options->verbose) + (options->verbose > 0 ? "\n" : ""), name, fname); + if (options->verbose > 0) HDprintf("...Found\n"); for (ifil = 0; ifil < obj.nfilters; ifil++) { diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h index 74525f4..65e5d40 100644 --- a/tools/src/h5repack/h5repack.h +++ b/tools/src/h5repack/h5repack.h @@ -27,6 +27,12 @@ #define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */ #define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */ +/* timing formats */ +#define FORMAT_OBJ_TIME " %-27s %e/%e %s\n" /* obj type, name */ +#define FORMAT_OBJ_ATTR_TIME " %-27s %e/%e %s\n" /* obj type, name */ +#define FORMAT_OBJ_NOTIME " %-27s %s\n" /* obj type, name */ +#define FORMAT_OBJ_ATTR_NOTIME " %-27s %s\n" /* obj type, name */ + /* File space default information */ #define FS_PAGESIZE_DEF 4096 #define FS_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR @@ -118,7 +124,7 @@ typedef struct { int grp_compact; /* Set the maximum number of links to store as header messages in the group */ int grp_indexed; /* Set the minimum number of links to store in the indexed format */ int msg_size[8]; /* Minimum size of shared messages: dataspace, - datatype, fill value, filter pipleline, attribute */ + datatype, fill value, filter pipeline, attribute */ const char * ublock_filename; /* user block file name */ hsize_t ublock_size; /* user block size */ hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 934b4d1..c38a431 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -39,7 +39,8 @@ */ static int get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t size_datum, hsize_t dims_hslab[], hsize_t *hslab_nbytes_p); -static void print_dataset_info(hid_t dcpl_id, char *objname, double per, int pr); +static void print_dataset_info(hid_t dcpl_id, char *objname, double per, int pr, pack_opt_t *options, + double read_time, double write_time); static int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *options); static int copy_user_block(const char *infile, const char *outfile, hsize_t size); #if defined(H5REPACK_DEBUG_USER_BLOCK) @@ -66,7 +67,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) hid_t fcpl = H5P_DEFAULT; /* file creation property list ID */ trav_table_t * travt = NULL; hsize_t ub_size = 0; /* size of user block */ - H5F_fspace_strategy_t set_strategy; /* Strategy to be set in outupt file */ + H5F_fspace_strategy_t set_strategy; /* Strategy to be set in output file */ hbool_t set_persist; /* Persist free-space status to be set in output file */ hsize_t set_threshold; /* Free-space section threshold to be set in output file */ hsize_t set_pagesize; /* File space page size to be set in output file */ @@ -81,8 +82,8 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) * open input file *------------------------------------------------------------------------- */ - if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, - (options->fin_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, (size_t)0)) < 0) + if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT), + NULL, (size_t)0)) < 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR); /* get user block size and file space strategy/persist/threshold */ @@ -187,7 +188,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) H5TOOLS_GOTO_ERROR((-1), "H5Pset_shared_mesg_nindexes failed to set the number of shared " "object header message indexes"); - /* msg_size[0]=dataspace, 1=datatype, 2=file value, 3=filter pipleline, 4=attribute */ + /* msg_size[0]=dataspace, 1=datatype, 2=file value, 3=filter pipeline, 4=attribute */ for (i = 0; i < (nindex - 1); i++) if (H5Pset_shared_mesg_index(fcpl, i, mesg_type_flags[i], min_mesg_sizes[i]) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pset_shared_mesg_index failed to configure the specified " @@ -298,7 +299,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) * create the output file *------------------------------------------------------------------------- */ - if (options->verbose) + if (options->verbose > 0) HDprintf("Making new file ...\n"); if ((fidout = H5Fcreate(fnameout, H5F_ACC_TRUNC, fcpl, options->fout_fapl)) < 0) @@ -401,18 +402,18 @@ done: /*------------------------------------------------------------------------- * Function: get_hyperslab * - * Purpose: Calulate a hyperslab from a dataset for higher performance. - * The size of hyperslab is limitted by H5TOOLS_BUFSIZE. - * Return the hyperslab dimentions and size in byte. + * Purpose: Calculate a hyperslab from a dataset for higher performance. + * The size of hyperslab is limited by H5TOOLS_BUFSIZE. + * Return the hyperslab dimensions and size in byte. * * Return: 0 - SUCCEED, -1 FAILED * * Parameters: * dcpl_id : [IN] dataset creation property. * rank_dset : [IN] dataset rank - * dims_dset[] : [IN] dataset dimentions + * dims_dset[] : [IN] dataset dimensions * size_datum : [IN] size of a data element in byte - * dims_hslab[] : [OUT] calculated hyperslab dimentions + * dims_hslab[] : [OUT] calculated hyperslab dimensions * * hslab_nbytes_p : [OUT] total byte of the hyperslab * * Update: @@ -427,7 +428,7 @@ done: * 3. If not chunked, each data element would be a unit of collection and * the boundary would be dataset's dims. * - * The calulation starts from the last dimention (h5dump dims output). + * The calculation starts from the last dimension (h5dump dims output). *-----------------------------------------*/ int @@ -441,8 +442,8 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si hsize_t size_chunk = 1; hsize_t nchunk_fit; /* number of chunks that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ hsize_t ndatum_fit; /* number of dataum that fits in hyperslab buffer (H5TOOLS_BUFSIZE) */ - hsize_t chunk_dims_map[H5S_MAX_RANK]; /* mapped chunk dimentions */ - hsize_t hs_dims_map[H5S_MAX_RANK]; /* mapped hyperslab dimentions */ + hsize_t chunk_dims_map[H5S_MAX_RANK]; /* mapped chunk dimensions */ + hsize_t hs_dims_map[H5S_MAX_RANK]; /* mapped hyperslab dimensions */ hsize_t hslab_nbytes; /* size of hyperslab in byte */ int ret_value = 0; @@ -467,18 +468,18 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si /* 1. if a chunk fit in hyperslab buffer */ if (nchunk_fit >= 1) { - /* Calulate a hyperslab that contains as many chunks that can fit + /* Calculate a hyperslab that contains as many chunks that can fit * in hyperslab buffer. Hyperslab will be increased starting from - * the last dimention of the dataset (see h5dump's dims output). + * the last dimension of the dataset (see h5dump's dims output). * The calculation boundary is dataset dims. * In the loop, used mapping from a datum to a chunk to figure out * chunk based hyperslab. */ for (k = rank_dset; k > 0; --k) { - /* map dataset dimentions with a chunk dims */ + /* map dataset dimensions with a chunk dims */ chunk_dims_map[k - 1] = dims_dset[k - 1] / dims_chunk[k - 1]; - /* if reminder exist, increse by 1 to cover partial edge chunks */ + /* if reminder exist, increase by 1 to cover partial edge chunks */ if (dims_dset[k - 1] % dims_chunk[k - 1] > 0) chunk_dims_map[k - 1]++; @@ -491,7 +492,7 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si if (nchunk_fit == 0) nchunk_fit = 1; - /* get hyperslab dimentions as unmapping to actual size */ + /* get hyperslab dimensions as unmapping to actual size */ dims_hslab[k - 1] = MIN((hs_dims_map[k - 1] * dims_chunk[k - 1]), dims_dset[k - 1]); /* calculate total size for the hyperslab */ @@ -500,9 +501,9 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si } /* 2. if a chunk is bigger than hyperslab buffer */ else { - /* Calulate a hyperslab that contains as many data elements that + /* Calculate a hyperslab that contains as many data elements that * can fit in hyperslab buffer. Hyperslab will be increased - * starting from the last dimention of the chunk (see h5dump's dims + * starting from the last dimension of the chunk (see h5dump's dims * output). * The calculation boundary is a chunk dims. */ @@ -512,7 +513,7 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si /* if a datum is bigger than rest of buffer */ if (ndatum_fit == 0) ndatum_fit = 1; - /* get hyperslab dimentions within a chunk boundary */ + /* get hyperslab dimensions within a chunk boundary */ dims_hslab[k - 1] = MIN(dims_chunk[k - 1], ndatum_fit); /* calculate total size for the hyperslab */ @@ -525,9 +526,9 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si } /* 3. if dataset is not chunked */ else { - /* Calulate a hyperslab that contains as many data elements that can + /* Calculate a hyperslab that contains as many data elements that can * fit in hyperslab buffer. Hyperslab will be increased starting from - * the last dimention of the dataset (see h5dump's dims output). + * the last dimension of the dataset (see h5dump's dims output). * The calculation boundary is dataset dims. */ for (k = rank_dset; k > 0; --k) { @@ -536,7 +537,7 @@ get_hyperslab(hid_t dcpl_id, int rank_dset, const hsize_t dims_dset[], size_t si /* if a datum is bigger than rest of buffer */ if (ndatum_fit == 0) ndatum_fit = 1; - /* get hyperslab dimentions within dataset boundary */ + /* get hyperslab dimensions within dataset boundary */ dims_hslab[k - 1] = MIN(dims_dset[k - 1], ndatum_fit); /* calculate total size for the hyperslab */ @@ -600,7 +601,7 @@ done: * in (2) is that, when using the strip mine size, it assures that the "remaining" part * of the dataset that does not fill an entire strip mine is processed. * - * 1. figure out a hyperslab (dimentions) and size (refer to get_hyperslab()). + * 1. figure out a hyperslab (dimensions) and size (refer to get_hyperslab()). * 2. Calculate the hyperslab selections as the selection is moving forward. * Selection would be same as the hyperslab except for the remaining edge portion * of the dataset. The code take care of the remaining portion if exist. @@ -643,6 +644,10 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti int req_filter; /* there was a request for a filter */ int req_obj_layout = 0; /* request layout to current object */ unsigned crt_order_flags; /* group creation order flag */ + H5_timer_t timer; /* Timer for read/write operations */ + H5_timevals_t times; /* Elapsed time for each operation */ + static double read_time = 0; + static double write_time = 0; h5tool_link_info_t linkinfo; unsigned i; unsigned u; @@ -661,7 +666,12 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti *------------------------------------------------------------------------- */ - if (options->verbose) { + if (options->verbose == 2) { + HDprintf("-----------------------------------------------------------------\n"); + HDprintf(" Type Filter (Compression) Timing read/write Name\n"); + HDprintf("-----------------------------------------------------------------\n"); + } + else { HDprintf("-----------------------------------------\n"); HDprintf(" Type Filter (Compression) Name\n"); HDprintf("-----------------------------------------\n"); @@ -682,7 +692,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti *------------------------------------------------------------------------- */ case H5TRAV_TYPE_GROUP: - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "group", travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "group", travt->objs[i].name); /* open input group */ @@ -749,6 +761,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti case H5TRAV_TYPE_DATASET: { hbool_t use_h5ocopy; + read_time = 0.0; + write_time = 0.0; + has_filter = 0; req_filter = 0; @@ -972,7 +987,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti H5P_DEFAULT, dcpl_out, H5P_DEFAULT); if (dset_out == H5I_INVALID_HID) { H5TOOLS_INFO("H5Dcreate2 failed"); - if (options->verbose) + if (options->verbose > 0) HDprintf(" warning: could not create dataset <%s>. Applying original " "settings\n", travt->objs[i].name); @@ -1013,11 +1028,27 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti } if (buf != NULL) { + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dread(dset_in, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dread failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + read_time += times.elapsed; + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dwrite(dset_out, wtype_id, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dwrite failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } /* Check if we have VL data in the dataset's * datatype that must be reclaimed */ @@ -1115,12 +1146,28 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti hs_select_nelmts = 1; } /* end (else) rank == 0 */ + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dread(dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dread failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + read_time += times.elapsed; + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Dwrite(dset_out, wtype_id, hslab_space, f_space_id, dxpl_id, hslab_buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dwrite failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } /* reclaim any VL memory, if necessary */ if (vl_data) @@ -1149,7 +1196,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti * print amount of compression used *------------------------------------------------------------------------- */ - if (options->verbose) { + if (options->verbose > 0) { double ratio = 0; /* only print the compression ration if there was a filter request */ @@ -1160,10 +1207,12 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti /* compression ratio = uncompressed size / compressed size */ if (dsize_out != 0) ratio = (double)dsize_in / (double)dsize_out; - print_dataset_info(dcpl_out, travt->objs[i].name, ratio, 1); + print_dataset_info(dcpl_out, travt->objs[i].name, ratio, 1, options, + read_time, write_time); } else - print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0); + print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0, options, + read_time, write_time); /* print a message that the filter was not applied * (in case there was a filter) @@ -1220,6 +1269,10 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti if (H5Pset_copy_object(ocpl_id, H5O_COPY_WITHOUT_ATTR_FLAG) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed"); + if (options->verbose == 2) { + H5_timer_init(&timer); + H5_timer_start(&timer); + } if (H5Ocopy(fidin, /* Source file or group identifier */ travt->objs[i].name, /* Name of the source object to be copied */ fidout, /* Destination file or group identifier */ @@ -1227,6 +1280,11 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti ocpl_id, /* Properties which apply to the copy */ H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */ H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed"); + if (options->verbose == 2) { + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + write_time += times.elapsed; + } if (H5Pclose(ocpl_id) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed"); @@ -1247,7 +1305,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti if (H5Dclose(dset_out) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dclose failed"); - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_TIME, "dset", 0.0, write_time, travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); } /* end whether we have request for filter/chunking */ @@ -1260,7 +1320,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti *------------------------------------------------------------------------- */ case H5TRAV_TYPE_NAMED_DATATYPE: - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "type", travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "type", travt->objs[i].name); if ((type_in = H5Topen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) @@ -1300,7 +1362,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti */ case H5TRAV_TYPE_LINK: case H5TRAV_TYPE_UDLINK: - if (options->verbose) + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "link", travt->objs[i].name); + else HDprintf(FORMAT_OBJ, "link", travt->objs[i].name); /* Check -X option. */ @@ -1425,7 +1489,8 @@ done: *------------------------------------------------------------------------- */ static void -print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) +print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr, pack_opt_t *options, double read_time, + double write_time) { char strfilter[255]; #if defined(PRINT_DEBUG) @@ -1514,7 +1579,10 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) } /* end for each filter */ if (!pr) - HDprintf(FORMAT_OBJ, "dset", objname); + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_TIME, "dset", read_time, write_time, objname); + else + HDprintf(FORMAT_OBJ, "dset", objname); else { char str[512], temp[512]; @@ -1522,7 +1590,10 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) HDstrcat(str, strfilter); HDsprintf(temp, " (%.3f:1)", ratio); HDstrcat(str, temp); - HDprintf(FORMAT_OBJ, str, objname); + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_TIME, str, read_time, write_time, objname); + else + HDprintf(FORMAT_OBJ, str, objname); } } /* end print_dataset_info() */ @@ -1550,7 +1621,7 @@ copy_user_block(const char *infile, const char *outfile, hsize_t size) /* Copy the userblock from the input file to the output file */ while (size > 0) { - ssize_t nread, nbytes; /* # of bytes transfered, etc. */ + ssize_t nread, nbytes; /* # of bytes transferred, etc. */ char rbuf[USERBLOCK_XFER_SIZE]; /* Buffer for reading */ const char *wbuf; /* Pointer into buffer, for writing */ diff --git a/tools/src/h5repack/h5repack_filters.c b/tools/src/h5repack/h5repack_filters.c index 32476ce..76bac5e 100644 --- a/tools/src/h5repack/h5repack_filters.c +++ b/tools/src/h5repack/h5repack_filters.c @@ -231,7 +231,7 @@ aux_assign_obj(const char * name, /* object name from traverse list */ * do extra checking in the case of SZIP; delete all filters in the case * of H5Z_FILTER_NONE present in the PACK_INFO_T filter array * - * Return: 0 success, -1 an error occured + * Return: 0 success, -1 an error occurred *------------------------------------------------------------------------- */ @@ -258,7 +258,7 @@ apply_filters(const char * name, /* object name from traverse list */ H5TOOLS_GOTO_DONE(0); /*------------------------------------------------------------------------- - * initialize the assigment object + * initialize the assignment object *------------------------------------------------------------------------- */ init_packobject(&obj); diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index 3526268..0241d62 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -18,7 +18,7 @@ /* Name of tool */ #define PROGRAMNAME "h5repack" -static int parse_command_line(int argc, const char **argv, pack_opt_t *options); +static int parse_command_line(int argc, const char *const *argv, pack_opt_t *options); static void leave(int ret) H5_ATTR_NORETURN; /* module-scoped variables */ @@ -31,7 +31,7 @@ const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char * s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:"; +static const char * s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:v*z:EG:LM:P:S:T:VXW1:2:3:4:5:6:"; static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"block", require_arg, 'b'}, {"compact", require_arg, 'c'}, @@ -39,18 +39,18 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"file", require_arg, 'e'}, {"filter", require_arg, 'f'}, {"help", no_arg, 'h'}, - {"infile", require_arg, 'i'}, /* for backward compability */ + {"infile", require_arg, 'i'}, /* for backward compatibility */ {"low", require_arg, 'j'}, {"high", require_arg, 'k'}, {"layout", require_arg, 'l'}, {"minimum", require_arg, 'm'}, {"native", no_arg, 'n'}, - {"outfile", require_arg, 'o'}, /* for backward compability */ + {"outfile", require_arg, 'o'}, /* for backward compatibility */ {"sort_by", require_arg, 'q'}, {"ssize", require_arg, 's'}, {"threshold", require_arg, 't'}, {"ublock", require_arg, 'u'}, - {"verbose", no_arg, 'v'}, + {"verbose", optional_arg, 'v'}, {"sort_order", require_arg, 'z'}, {"enable-error-stack", no_arg, 'E'}, {"fs_pagesize", require_arg, 'G'}, @@ -88,7 +88,8 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " file2 Output HDF5 File\n"); PRINTVALSTREAM(rawoutstream, " OPTIONS\n"); PRINTVALSTREAM(rawoutstream, " -h, --help Print a usage message and exit\n"); - PRINTVALSTREAM(rawoutstream, " -v, --verbose Verbose mode, print object information\n"); + PRINTVALSTREAM(rawoutstream, " -v N, --verbose=N Verbose mode, print object information.\n"); + PRINTVALSTREAM(rawoutstream, " N - is an integer greater than 1, 2 displays read/write timing\n"); PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n"); PRINTVALSTREAM(rawoutstream, " -n, --native Use a native HDF5 type when repacking\n"); PRINTVALSTREAM(rawoutstream, @@ -112,6 +113,24 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " --dst-vol-info VOL-specific info to pass to the VOL connector used for\n"); PRINTVALSTREAM(rawoutstream, " opening the output HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --src-vfd-value Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " input HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --src-vfd-name Name of the VFL driver to use for opening the input\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --src-vfd-info VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the input HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --dst-vfd-value Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " output HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --dst-vfd-name Name of the VFL driver to use for opening the output\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --dst-vfd-info VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the output HDF5 file specified\n"); PRINTVALSTREAM(rawoutstream, " -L, --latest Use latest version of file format\n"); PRINTVALSTREAM(rawoutstream, " This option will take precedence over the options\n"); @@ -374,7 +393,7 @@ read_info(const char *filename, pack_opt_t *options) goto done; } - /* find begining of info */ + /* find beginning of info */ i = 0; c = '0'; while (c != ' ') { @@ -482,7 +501,7 @@ set_sort_order(const char *form) *------------------------------------------------------------------------- */ static int -parse_command_line(int argc, const char **argv, pack_opt_t *options) +parse_command_line(int argc, const char *const *argv, pack_opt_t *options) { h5tools_vol_info_t in_vol_info; h5tools_vol_info_t out_vol_info; @@ -525,7 +544,12 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) goto done; case 'v': - options->verbose = 1; + if (H5_optarg != NULL) { + if (2 == HDatoi(H5_optarg)) + options->verbose = 2; + } + else + options->verbose = 1; break; case 'f': @@ -665,7 +689,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case 'a': options->alignment = HDstrtoull(H5_optarg, NULL, 0); if (options->alignment < 1) { - error_msg("invalid alignment size\n", H5_optarg); + error_msg("invalid alignment size `%s`\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -685,7 +709,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) else if (!HDstrcmp(strategy, "NONE")) options->fs_strategy = H5F_FSPACE_STRATEGY_NONE; else { - error_msg("invalid file space management strategy\n", H5_optarg); + error_msg("invalid file space management strategy `%s`\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -858,7 +882,7 @@ done: *------------------------------------------------------------------------- */ int -main(int argc, const char **argv) +main(int argc, char **argv) { pack_opt_t options; /*the global options */ int parse_ret; @@ -888,7 +912,7 @@ main(int argc, const char **argv) /* Initialize default indexing options */ sort_by = H5_INDEX_CRT_ORDER; - parse_ret = parse_command_line(argc, argv, &options); + parse_ret = parse_command_line(argc, (const char *const *)argv, &options); if (parse_ret < 0) { HDprintf("Error occurred while parsing command-line options\n"); h5tools_setstatus(EXIT_FAILURE); diff --git a/tools/src/h5repack/h5repack_refs.c b/tools/src/h5repack/h5repack_refs.c index 6191624..3259b89 100644 --- a/tools/src/h5repack/h5repack_refs.c +++ b/tools/src/h5repack/h5repack_refs.c @@ -182,8 +182,11 @@ do_copy_refobjs(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti /* create the reference, -1 parameter for objects */ if (H5Rcreate(&refbuf[u], fidout, refname, H5R_OBJECT, (hid_t)-1) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Rcreate failed"); - if (options->verbose) { - HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); + if (options->verbose > 0) { + if (options->verbose == 2) + HDprintf(FORMAT_OBJ_NOTIME, "dset", travt->objs[i].name); + else + HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); HDprintf("object <%s> object reference created to <%s>\n", travt->objs[i].name, refname); } @@ -276,7 +279,7 @@ do_copy_refobjs(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti H5TOOLS_GOTO_ERROR((-1), "H5Rcreate failed"); if (H5Sclose(region_id) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Sclose failed"); - if (options->verbose) { + if (options->verbose > 0) { HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); HDprintf("object <%s> region reference created to <%s>\n", travt->objs[i].name, refname); @@ -536,7 +539,7 @@ copy_refs_attr(hid_t loc_in, hid_t loc_out, trav_table_t *travt, hid_t fidout) / } /* This line below needs to be moved in this loop instead of inserting outside. Otherwise, ref_comp_field_n may be >0 for the next attribute, which may not be - the reference type and will be accidently treated as the reference type. + the reference type and will be accidentally treated as the reference type. It will then cause the H5Acreate2 failed since that attribute is already created. KY 2020-02-07 */ diff --git a/tools/src/h5repack/h5repack_verify.c b/tools/src/h5repack/h5repack_verify.c index 0542d03..71ee72e 100644 --- a/tools/src/h5repack/h5repack_verify.c +++ b/tools/src/h5repack/h5repack_verify.c @@ -378,11 +378,11 @@ h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t *------------------------------------------------------------------------- */ /* Open the files */ - if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl, - (fname1_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0) + if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl, (fname1_fapl != H5P_DEFAULT), NULL, 0)) < + 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname1, H5FOPENERROR); - if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl, - (fname2_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0) + if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl, (fname2_fapl != H5P_DEFAULT), NULL, 0)) < + 0) H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname2, H5FOPENERROR); /*------------------------------------------------------------------------- @@ -397,7 +397,7 @@ h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t H5TOOLS_GOTO_ERROR((-1), "h5trav_gettable failed"); /*------------------------------------------------------------------------- - * traverse the suppplied object list + * traverse the supplied object list *------------------------------------------------------------------------- */ for (i = 0; i < trav->nobjs; i++) { diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index ea4e314..9ad9203 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -33,7 +33,7 @@ #define H5_NFILTERS_IMPL \ 8 /* Number of currently implemented filters + one to \ accommodate for user-define filters + one \ - to accomodate datasets whithout any filters */ + to accommodate datasets without any filters */ /* File space management strategies: see H5Fpublic.h for declarations */ const char *FS_STRATEGY_NAME[] = {"H5F_FSPACE_STRATEGY_FSM_AGGR", @@ -830,7 +830,7 @@ hand_free(struct handler_t *hand) *------------------------------------------------------------------------- */ static int -parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) +parse_command_line(int argc, const char *const *argv, struct handler_t **hand_ret) { int opt; unsigned u; @@ -1677,7 +1677,7 @@ print_statistics(const char *name, const iter_t *iter) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { iter_t iter; const char * fname = NULL; @@ -1693,7 +1693,7 @@ main(int argc, const char *argv[]) HDmemset(&iter, 0, sizeof(iter)); - if (parse_command_line(argc, argv, &hand) < 0) + if (parse_command_line(argc, (const char *const *)argv, &hand) < 0) goto done; fname = argv[H5_optind]; diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c index face2f0..15c170d 100644 --- a/tools/src/misc/h5clear.c +++ b/tools/src/misc/h5clear.c @@ -109,7 +109,7 @@ usage(const char *prog) *------------------------------------------------------------------------- */ static int -parse_command_line(int argc, const char **argv) +parse_command_line(int argc, const char *const *argv) { int opt; @@ -224,7 +224,7 @@ leave(int ret) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { char * fname = NULL; /* File name */ hid_t fapl = H5I_INVALID_HID; /* File access property list */ @@ -240,7 +240,7 @@ main(int argc, const char *argv[]) h5tools_init(); /* Parse command line options */ - if (parse_command_line(argc, argv) < 0) + if (parse_command_line(argc, (const char *const *)argv) < 0) goto done; if (fname_g == NULL) diff --git a/tools/src/misc/h5debug.c b/tools/src/misc/h5debug.c index 72f47cb..e03c72d 100644 --- a/tools/src/misc/h5debug.c +++ b/tools/src/misc/h5debug.c @@ -647,7 +647,7 @@ main(int argc, char *argv[]) } else if (!HDmemcmp(sig, H5EA_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* - * Debug an extensible aray header. + * Debug an extensible array header. */ const H5EA_class_t *cls = get_H5EA_class(sig); HDassert(cls); @@ -668,7 +668,7 @@ main(int argc, char *argv[]) } else if (!HDmemcmp(sig, H5EA_IBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* - * Debug an extensible aray index block. + * Debug an extensible array index block. */ const H5EA_class_t *cls = get_H5EA_class(sig); HDassert(cls); @@ -689,7 +689,7 @@ main(int argc, char *argv[]) } else if (!HDmemcmp(sig, H5EA_SBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* - * Debug an extensible aray super block. + * Debug an extensible array super block. */ const H5EA_class_t *cls = get_H5EA_class(sig); HDassert(cls); @@ -709,7 +709,7 @@ main(int argc, char *argv[]) } else if (!HDmemcmp(sig, H5EA_DBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* - * Debug an extensible aray data block. + * Debug an extensible array data block. */ const H5EA_class_t *cls = get_H5EA_class(sig); HDassert(cls); diff --git a/tools/src/misc/h5delete.c b/tools/src/misc/h5delete.c index 3c4f8d5..20e81a1 100644 --- a/tools/src/misc/h5delete.c +++ b/tools/src/misc/h5delete.c @@ -29,7 +29,7 @@ usage(void) } int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hbool_t quiet = FALSE; const char *name = NULL; diff --git a/tools/src/misc/h5mkgrp.c b/tools/src/misc/h5mkgrp.c index a85ee4d..5c2396c 100644 --- a/tools/src/misc/h5mkgrp.c +++ b/tools/src/misc/h5mkgrp.c @@ -105,6 +105,14 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " --vol-info VOL-specific info to pass to the VOL connector used for\n"); PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-value Value (ID) of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, " --vfd-name Name of the VFL driver to use for opening the\n"); + PRINTVALSTREAM(rawoutstream, " HDF5 file specified\n"); + PRINTVALSTREAM(rawoutstream, + " --vfd-info VFD-specific info to pass to the VFL driver used for\n"); + PRINTVALSTREAM(rawoutstream, " opening the HDF5 file specified\n"); PRINTVALSTREAM(rawoutstream, "\n"); } /* end usage() */ @@ -122,7 +130,7 @@ usage(const char *prog) *------------------------------------------------------------------------- */ static int -parse_command_line(int argc, const char *argv[], mkgrp_opt_t *options) +parse_command_line(int argc, const char *const *argv, mkgrp_opt_t *options) { int opt; /* Option from command line */ size_t curr_group; /* Current group name to copy */ @@ -252,7 +260,7 @@ parse_command_line(int argc, const char *argv[], mkgrp_opt_t *options) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { hid_t fid = H5I_INVALID_HID; /* HDF5 file ID */ hid_t lcpl_id = H5I_INVALID_HID; /* Link creation property list ID */ @@ -274,7 +282,7 @@ main(int argc, const char *argv[]) } /* Parse command line */ - if (parse_command_line(argc, argv, ¶ms_g) < 0) { + if (parse_command_line(argc, (const char *const *)argv, ¶ms_g) < 0) { error_msg("unable to parse command line arguments\n"); leave(EXIT_FAILURE); } diff --git a/tools/test/h5copy/CMakeTests.cmake b/tools/test/h5copy/CMakeTests.cmake index e9b36fe..154ca15 100644 --- a/tools/test/h5copy/CMakeTests.cmake +++ b/tools/test/h5copy/CMakeTests.cmake @@ -211,7 +211,7 @@ endmacro () # - # Similiar to ADD_H5_TEST macro. Compare to outputs from source & target + # Similar to ADD_H5_TEST macro. Compare to outputs from source & target # files instead of checking with h5ls. # macro (ADD_H5_CMP_TEST testname resultcode infile vparam sparam srcname dparam dstname) diff --git a/tools/test/h5copy/h5copygentest.c b/tools/test/h5copy/h5copygentest.c index 6898839..c1f8349 100644 --- a/tools/test/h5copy/h5copygentest.c +++ b/tools/test/h5copy/h5copygentest.c @@ -618,7 +618,7 @@ gen_region_ref(hid_t loc_id) goto out; } - /* create normal dataset which is refered */ + /* create normal dataset which is referred */ oid2 = H5Dcreate2(loc_id, REG_REF_DS2, H5T_STD_I8LE, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (oid2 < 0) { HDfprintf(stderr, "Error: %s %d> H5Dcreate2 failed.\n", __func__, __LINE__); diff --git a/tools/test/h5copy/testh5copy.sh.in b/tools/test/h5copy/testh5copy.sh.in index 79b6fd5..6349ef4 100644 --- a/tools/test/h5copy/testh5copy.sh.in +++ b/tools/test/h5copy/testh5copy.sh.in @@ -88,7 +88,7 @@ test -d $TESTDIR || mkdir $TESTDIR # RUNSERIAL is used. Check if it can return exit code from executalbe correctly. if [ -n "$RUNSERIAL_NOEXITCODE" ]; then - echo "***Warning*** Serial Exit Code is not passed back to shell corretly." + echo "***Warning*** Serial Exit Code is not passed back to shell correctly." echo "***Warning*** Exit code checking is skipped." h5haveexitcode=no fi diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake index 4fcfa26..8e01605 100644 --- a/tools/test/h5diff/CMakeTests.cmake +++ b/tools/test/h5diff/CMakeTests.cmake @@ -1278,7 +1278,7 @@ ADD_H5_TEST (h5diff_208 0 -c ${FILE19} ${FILE20}) ADD_H5_TEST (h5diff_220 1 -c non_comparables1.h5 non_comparables2.h5 /g1) # comparable dataset with non-comparable attribute and other comparable attributes. -# Also test non-compatible attributes with different type, dimention, rank. +# Also test non-compatible attributes with different type, dimension, rank. # All the rest comparables should display differences. ADD_H5_TEST (h5diff_221 1 -c non_comparables1.h5 non_comparables2.h5 /g2) @@ -1438,7 +1438,7 @@ ADD_H5_TEST (h5diff_475 1 -v h5diff_danglelinks1.h5 h5diff_danglelinks2.h5 /ext_ # ############################################################################## -# # test for group diff recursivly +# # test for group diff recursively # ############################################################################## # root ADD_H5_TEST (h5diff_500 1 -v ${GRP_RECURSE_FILE1} ${GRP_RECURSE_FILE2} / /) diff --git a/tools/test/h5diff/h5diff_plugin.sh.in b/tools/test/h5diff/h5diff_plugin.sh.in index 0ab169a..28f723e 100644 --- a/tools/test/h5diff/h5diff_plugin.sh.in +++ b/tools/test/h5diff/h5diff_plugin.sh.in @@ -158,7 +158,7 @@ done # RUNSERIAL is used. Check if it can return exit code from executalbe correctly. if [ -n "$RUNSERIAL_NOEXITCODE" ]; then - echo "***Warning*** Serial Exit Code is not passed back to shell corretly." + echo "***Warning*** Serial Exit Code is not passed back to shell correctly." echo "***Warning*** Exit code checking is skipped." h5haveexitcode=no fi @@ -236,7 +236,7 @@ TOOLTEST() { # don't add exit code check in pmode, as it causes failure. (exit code # is from mpirun not tool) # if any problem occurs relate to an exit code, it will be caught in - # serial mode, so the test is fullfilled. + # serial mode, so the test is fulfilled. if test $h5haveexitcode = 'yes' -a -z "$pmode"; then echo "EXIT CODE: $EXIT_CODE" >> $actual fi @@ -327,7 +327,7 @@ TOOLTEST_ERR() { # don't add exit code check in pmode, as it causes failure. (exit code # is from mpirun not tool) # if any problem occurs relate to an exit code, it will be caught in - # serial mode, so the test is fullfilled. + # serial mode, so the test is fulfilled. if test $h5haveexitcode = 'yes' -a -z "$pmode"; then echo "EXIT CODE: $EXIT_CODE" >> $actual fi diff --git a/tools/test/h5diff/h5diffgentest.c b/tools/test/h5diff/h5diffgentest.c index 7cefeea..5e0f830 100644 --- a/tools/test/h5diff/h5diffgentest.c +++ b/tools/test/h5diff/h5diffgentest.c @@ -82,7 +82,7 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024); #define EXCLUDE_FILE3_2 "h5diff_exclude3-2.h5" /* compound type with multiple vlen string types */ #define COMP_VL_STRS_FILE "h5diff_comp_vl_strs.h5" -/* attribute compre with verbose level */ +/* attribute compare with verbose level */ #define ATTR_VERBOSE_LEVEL_FILE1 "h5diff_attr_v_level1.h5" #define ATTR_VERBOSE_LEVEL_FILE2 "h5diff_attr_v_level2.h5" /* file containing valid/invalid enum value mix */ @@ -358,10 +358,10 @@ test_basic(const char *fname1, const char *fname2, const char *fname3) */ { - double data1[3][2] = {{1.0F, 1.0F}, {1.00F, 1.000F}, {0.0F, 0.0F}}; - double data2[3][2] = {{0.0F, 1.1F}, {1.01F, 1.001F}, {0.0F, 1.0F}}; - double data3[3][2] = {{100.0F, 100.0F}, {100.00F, 100.000F}, {100.0F, 100.0F}}; - double data4[3][2] = {{105.0F, 120.0F}, {160.00F, 95.000F}, {80.0F, 40.0F}}; + double data1[3][2] = {{1.0, 1.0}, {1.00, 1.000}, {0.0, 0.0}}; + double data2[3][2] = {{0.0, 1.1}, {1.01, 1.001}, {0.0, 1.0}}; + double data3[3][2] = {{100.0, 100.0}, {100.00, 100.000}, {100.0, 100.0}}; + double data4[3][2] = {{105.0, 120.0}, {160.00, 95.000}, {80.0, 40.0}}; write_dset(gid1, 2, dims2, "dset1", H5T_NATIVE_DOUBLE, data1); write_dset(gid2, 2, dims2, "dset2", H5T_NATIVE_DOUBLE, data2); @@ -409,8 +409,8 @@ test_basic(const char *fname1, const char *fname2, const char *fname3) *------------------------------------------------------------------------- */ { - double data9[3][2] = {{100.0F, 100.0F}, {100.0F, 0.0F}, {0.0F, 100.0F}}; - double data10[3][2] = {{120.0F, 80.0F}, {0.0F, 100.0F}, {0.0F, 50.0F}}; + double data9[3][2] = {{100.0, 100.0}, {100.0, 0.0}, {0.0, 100.0}}; + double data10[3][2] = {{120.0, 80.0}, {0.0, 100.0}, {0.0, 50.0}}; write_dset(gid1, 2, dims2, "dset9", H5T_NATIVE_DOUBLE, data9); write_dset(gid1, 2, dims2, "dset10", H5T_NATIVE_DOUBLE, data10); @@ -464,18 +464,18 @@ test_basic(const char *fname1, const char *fname2, const char *fname3) float data16[6]; data15[0] = (float)HDsqrt(-1.0); - data15[1] = 1.0F; + data15[1] = 1.0; data15[2] = (float)HDsqrt(-1.0); - data15[3] = 1.0F; - data15[4] = 1.0F; - data15[5] = 1.0F; + data15[3] = 1.0; + data15[4] = 1.0; + data15[5] = 1.0; data16[0] = (float)HDsqrt(-1.0); data16[1] = (float)HDsqrt(-1.0); - data16[2] = 1.0F; - data16[3] = 1.0F; - data16[4] = 1.0F; - data16[5] = 1.0F; + data16[2] = 1.0; + data16[3] = 1.0; + data16[4] = 1.0; + data16[5] = 1.0; write_dset(gid1, 1, dims1, "fp15", H5T_NATIVE_FLOAT, data15); write_dset(gid1, 1, dims1, "fp16", H5T_NATIVE_FLOAT, data16); @@ -552,8 +552,8 @@ test_basic(const char *fname1, const char *fname2, const char *fname3) buf1[1].d = HDsqrt(-1.0); buf1[1].f = (float)HDsqrt(-1.0); - buf2[1].d = 0.0F; - buf2[1].f = 0.0F; + buf2[1].d = 0.0; + buf2[1].f = 0.0; type_size = sizeof(cmp1_t); tid = H5Tcreate(H5T_COMPOUND, type_size); @@ -581,13 +581,13 @@ test_basic(const char *fname1, const char *fname2, const char *fname3) int i; } cmp3_t; - double data2[6] = {0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F}; + double data2[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; int data3[6] = {0, 0, 0, 0, 0, 0}; int data4[3][2] = {{0, 0}, {0, 0}, {0, 0}}; int data5[2][2] = {{0, 0}, {0, 0}}; unsigned int data6[3][2] = {{0, 0}, {0, 0}, {0, 0}}; - cmp1_t data7[1] = {{1.0f, 2}}; - cmp2_t data8[1] = {{1, 2.0f}}; + cmp1_t data7[1] = {{1.0, 2}}; + cmp2_t data8[1] = {{1, 2.0}}; hsize_t dims3[2] = {2, 2}; hsize_t dims4[1] = {1}; size_t type_size; @@ -787,10 +787,10 @@ test_datatypes(const char *fname) int buf3b[3][2] = {{1, 1}, {3, 4}, {5, 6}}; long buf4a[3][2] = {{1, 1}, {1, 1}, {1, 1}}; long buf4b[3][2] = {{1, 1}, {3, 4}, {5, 6}}; - float buf5a[3][2] = {{1.0F, 1.0F}, {1.0F, 1.0F}, {1.0F, 1.0F}}; - float buf5b[3][2] = {{1.0F, 1.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; - double buf6a[3][2] = {{1.0F, 1.0F}, {1.0F, 1.0F}, {1.0F, 1.0F}}; - double buf6b[3][2] = {{1.0F, 1.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; + float buf5a[3][2] = {{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}; + float buf5b[3][2] = {{1.0, 1.0}, {3.0, 4.0}, {5.0, 6.0}}; + double buf6a[3][2] = {{1.0, 1.0}, {1.0, 1.0}, {1.0, 1.0}}; + double buf6b[3][2] = {{1.0, 1.0}, {3.0, 4.0}, {5.0, 6.0}}; /*unsigned/signed test signed char -128 to 127 @@ -2904,7 +2904,7 @@ test_group_recurse2(void) * FILE 2-3 *------------------------------------------------------------------------*/ - /* crate target file */ + /* create target file */ fileid4 = H5Fcreate(GRP_RECURSE2_EXT3, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); /*----------------------------------------------- @@ -2938,7 +2938,7 @@ test_group_recurse2(void) * FILE 2-2 *------------------------------------------------------------------------*/ - /* crate target file */ + /* create target file */ fileid3 = H5Fcreate(GRP_RECURSE2_EXT2, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); /*----------------------------------------------- @@ -2990,7 +2990,7 @@ test_group_recurse2(void) * FILE 2-1 *------------------------------------------------------------------------*/ - /* crate target file */ + /* create target file */ fileid2 = H5Fcreate(GRP_RECURSE2_EXT1, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); /*----------------------------------------------- @@ -4856,7 +4856,7 @@ test_data_nocomparables(const char *fname, int make_diffs) /* attr1 - non-compatible : different type */ write_attr(did2, 1, dims1_1, "attr1", tid_attr1, attr_data_ptr2); - /* attr2 - non-compatible : same rank, different dimention */ + /* attr2 - non-compatible : same rank, different dimension */ write_attr(did2, 1, (hsize_t *)attr2_dim_ptr, "attr2", H5T_NATIVE_INT, data3); /* attr3 - non-compatible : different rank */ @@ -5210,27 +5210,27 @@ write_attr_strings(hid_t loc_id, const char *dset_name, hid_t fid, char buf1[2][STR_SIZE] = {"ab", "de"}; /* string */ char * buf1a[2]; /* VL string */ char buf2[2] = {1, 2}; /* bitfield, opaque */ - s_t buf3[2] = {{1, 2.0F}, {3, 4.0F}}; /* compound */ + s_t buf3[2] = {{1, 2.0}, {3, 4.0}}; /* compound */ hobj_ref_t buf4[2]; /* reference */ e_t buf45[2] = {RED, RED}; /* enum */ hvl_t buf5[2]; /* vlen */ hsize_t dimarray[1] = {3}; /* array dimension */ int buf6[2][3] = {{1, 2, 3}, {4, 5, 6}}; /* array */ int buf7[2] = {1, 2}; /* integer */ - float buf8[2] = {1.0F, 2.0F}; /* float */ + float buf8[2] = {1.0, 2.0}; /* float */ /* create 2D attributes with dimension [3][2], 6 elements */ - hsize_t dims2[2] = {3, 2}; - char buf12[3][2][STR_SIZE] = {{"ab", "cd"}, {"ef", "gh"}, {"ij", "kl"}}; /* string */ - char * buf12a[3][2]; /* VL string */ - char buf22[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* bitfield, opaque */ - s_t buf32[6] = {{1, 2.0F}, {3, 4.0F}, {5, 6.0F}, {7, 8.0F}, {9, 10.0F}, {11, 12.0F}}; /* compound */ + hsize_t dims2[2] = {3, 2}; + char buf12[3][2][STR_SIZE] = {{"ab", "cd"}, {"ef", "gh"}, {"ij", "kl"}}; /* string */ + char * buf12a[3][2]; /* VL string */ + char buf22[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* bitfield, opaque */ + s_t buf32[6] = {{1, 2.0}, {3, 4.0}, {5, 6.0}, {7, 8.0}, {9, 10.0}, {11, 12.0}}; /* compound */ hobj_ref_t buf42[3][2]; /* reference */ e_t buf452[3][2]; /* enum */ hvl_t buf52[3][2]; /* vlen */ int buf62[6][3] = {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}, {13, 14, 15}, {16, 17, 18}}; /* array */ - int buf72[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* integer */ - float buf82[3][2] = {{1.0F, 2.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; /* float */ + int buf72[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* integer */ + float buf82[3][2] = {{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}}; /* float */ /* create 3D attributes with dimension [4][3][2], 24 elements */ hsize_t dims3[3] = {4, 3, 2}; @@ -5931,7 +5931,7 @@ write_attr_strings(hid_t loc_id, const char *dset_name, hid_t fid, for (k = 0; k < 2; k++) if (make_diffs) { buf33[i][j][k].a = 0; - buf33[i][j][k].b = 0.0F; + buf33[i][j][k].b = 0.0; } else { buf33[i][j][k].a = (char)n++; @@ -6141,7 +6141,7 @@ write_attr_strings(hid_t loc_id, const char *dset_name, hid_t fid, for (k = 0; k < 2; k++) { if (make_diffs) { buf73[i][j][k] = 0; - buf83[i][j][k] = 0.0F; + buf83[i][j][k] = 0.0; } else { buf73[i][j][k] = n++; @@ -6199,27 +6199,27 @@ write_attr_in(hid_t loc_id, const char *dset_name, hid_t fid, char buf1[2][STR_SIZE] = {"ab", "de"}; /* string */ char * buf1a[2]; /* VL string */ char buf2[2] = {1, 2}; /* bitfield, opaque */ - s_t buf3[2] = {{1, 2.0F}, {3, 4.0F}}; /* compound */ + s_t buf3[2] = {{1, 2.0}, {3, 4.0}}; /* compound */ hobj_ref_t buf4[2]; /* reference */ e_t buf45[2] = {RED, RED}; /* enum */ hvl_t buf5[2]; /* vlen */ hsize_t dimarray[1] = {3}; /* array dimension */ int buf6[2][3] = {{1, 2, 3}, {4, 5, 6}}; /* array */ int buf7[2] = {1, 2}; /* integer */ - float buf8[2] = {1.0F, 2.0F}; /* float */ + float buf8[2] = {1.0, 2.0}; /* float */ /* create 2D attributes with dimension [3][2], 6 elements */ - hsize_t dims2[2] = {3, 2}; - char buf12[3][2][STR_SIZE] = {{"ab", "cd"}, {"ef", "gh"}, {"ij", "kl"}}; /* string */ - char * buf12a[3][2]; /* VL string */ - char buf22[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* bitfield, opaque */ - s_t buf32[6] = {{1, 2.0F}, {3, 4.0F}, {5, 6.0F}, {7, 8.0F}, {9, 10.0F}, {11, 12.0F}}; /* compound */ + hsize_t dims2[2] = {3, 2}; + char buf12[3][2][STR_SIZE] = {{"ab", "cd"}, {"ef", "gh"}, {"ij", "kl"}}; /* string */ + char * buf12a[3][2]; /* VL string */ + char buf22[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* bitfield, opaque */ + s_t buf32[6] = {{1, 2.0}, {3, 4.0}, {5, 6.0}, {7, 8.0}, {9, 10.0}, {11, 12.0}}; /* compound */ hobj_ref_t buf42[3][2]; /* reference */ e_t buf452[3][2]; /* enum */ hvl_t buf52[3][2]; /* vlen */ int buf62[6][3] = {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}, {13, 14, 15}, {16, 17, 18}}; /* array */ - int buf72[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* integer */ - float buf82[3][2] = {{1.0F, 2.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; /* float */ + int buf72[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* integer */ + float buf82[3][2] = {{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}}; /* float */ /* create 3D attributes with dimension [4][3][2], 24 elements */ hsize_t dims3[3] = {4, 3, 2}; @@ -6920,7 +6920,7 @@ write_attr_in(hid_t loc_id, const char *dset_name, hid_t fid, for (k = 0; k < 2; k++) { if (make_diffs) { buf33[i][j][k].a = 0; - buf33[i][j][k].b = 0.0F; + buf33[i][j][k].b = 0.0; } else { buf33[i][j][k].a = (char)n++; @@ -7130,7 +7130,7 @@ write_attr_in(hid_t loc_id, const char *dset_name, hid_t fid, for (k = 0; k < 2; k++) { if (make_diffs) { buf73[i][j][k] = 0; - buf83[i][j][k] = 0.0F; + buf83[i][j][k] = 0.0; } else { buf73[i][j][k] = n++; @@ -7191,26 +7191,26 @@ write_dset_in(hid_t loc_id, const char *dset_name, hid_t fid, char buf1[2][STR_SIZE] = {"ab", "de"}; /* string */ char * buf1a[2]; /* VL string */ char buf2[2] = {1, 2}; /* bitfield, opaque */ - s_t buf3[2] = {{1, 2.0F}, {3, 4.0F}}; /* compound */ + s_t buf3[2] = {{1, 2.0}, {3, 4.0}}; /* compound */ hobj_ref_t buf4[2]; /* reference */ e_t buf45[2] = {RED, GREEN}; /* enum */ hvl_t buf5[2]; /* vlen */ hsize_t dimarray[1] = {3}; /* array dimension */ int buf6[2][3] = {{1, 2, 3}, {4, 5, 6}}; /* array */ int buf7[2] = {1, 2}; /* integer */ - float buf8[2] = {1.0F, 2.0F}; /* float */ + float buf8[2] = {1.0, 2.0}; /* float */ /* create 2D attributes with dimension [3][2], 6 elements */ - hsize_t dims2[2] = {3, 2}; - char buf12[3][2][STR_SIZE] = {{"ab", "cd"}, {"ef", "gh"}, {"ij", "kl"}}; /* string */ - char * buf12a[3][2]; /* VL string */ - char buf22[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* bitfield, opaque */ - s_t buf32[6] = {{1, 2.0F}, {3, 4.0F}, {5, 6.0F}, {7, 8.0F}, {9, 10.0F}, {11, 12.0F}}; /* compound */ + hsize_t dims2[2] = {3, 2}; + char buf12[3][2][STR_SIZE] = {{"ab", "cd"}, {"ef", "gh"}, {"ij", "kl"}}; /* string */ + char * buf12a[3][2]; /* VL string */ + char buf22[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* bitfield, opaque */ + s_t buf32[6] = {{1, 2.0}, {3, 4.0}, {5, 6.0}, {7, 8.0}, {9, 10.0}, {11, 12.0}}; /* compound */ hobj_ref_t buf42[3][2]; /* reference */ hvl_t buf52[3][2]; /* vlen */ int buf62[6][3] = {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11, 12}, {13, 14, 15}, {16, 17, 18}}; /* array */ - int buf72[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* integer */ - float buf82[3][2] = {{1.0F, 2.0F}, {3.0F, 4.0F}, {5.0F, 6.0F}}; /* float */ + int buf72[3][2] = {{1, 2}, {3, 4}, {5, 6}}; /* integer */ + float buf82[3][2] = {{1.0, 2.0}, {3.0, 4.0}, {5.0, 6.0}}; /* float */ /* create 3D attributes with dimension [4][3][2], 24 elements */ hsize_t dims3[3] = {4, 3, 2}; @@ -7678,7 +7678,7 @@ write_dset_in(hid_t loc_id, const char *dset_name, hid_t fid, for (k = 0; k < 2; k++) { if (make_diffs) { buf33[i][j][k].a = 0; - buf33[i][j][k].b = 0.0F; + buf33[i][j][k].b = 0.0; } else { buf33[i][j][k].a = (char)n++; @@ -7776,7 +7776,7 @@ write_dset_in(hid_t loc_id, const char *dset_name, hid_t fid, for (k = 0; k < 2; k++) { if (make_diffs) { buf73[i][j][k] = 0; - buf83[i][j][k] = 0.0F; + buf83[i][j][k] = 0.0; } else { buf73[i][j][k] = n++; @@ -7905,7 +7905,7 @@ gen_datareg(hid_t fid, int make_diffs /* flag to modify data buffers */) * Function: test_hyperslab * * Purpose: test diff by hyperslabs. create a dataset with 1GB dimensions - * by iterating trough 1KB hyperslabs + * by iterating through 1KB hyperslabs * *------------------------------------------------------------------------- */ diff --git a/tools/test/h5diff/testfiles/h5diff_10.txt b/tools/test/h5diff/testfiles/h5diff_10.txt index 3631db7..853aede 100644 --- a/tools/test/h5diff/testfiles/h5diff_10.txt +++ b/tools/test/h5diff/testfiles/h5diff_10.txt @@ -36,6 +36,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -117,7 +129,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_600.txt b/tools/test/h5diff/testfiles/h5diff_600.txt index 8a4ce6d..2893b78 100644 --- a/tools/test/h5diff/testfiles/h5diff_600.txt +++ b/tools/test/h5diff/testfiles/h5diff_600.txt @@ -36,6 +36,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -117,7 +129,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_603.txt b/tools/test/h5diff/testfiles/h5diff_603.txt index d4c7336..4317282 100644 --- a/tools/test/h5diff/testfiles/h5diff_603.txt +++ b/tools/test/h5diff/testfiles/h5diff_603.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_606.txt b/tools/test/h5diff/testfiles/h5diff_606.txt index 3123587..c32142f 100644 --- a/tools/test/h5diff/testfiles/h5diff_606.txt +++ b/tools/test/h5diff/testfiles/h5diff_606.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_612.txt b/tools/test/h5diff/testfiles/h5diff_612.txt index ce2d887..a080f29 100644 --- a/tools/test/h5diff/testfiles/h5diff_612.txt +++ b/tools/test/h5diff/testfiles/h5diff_612.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_615.txt b/tools/test/h5diff/testfiles/h5diff_615.txt index 61fd036..a9bd76c 100644 --- a/tools/test/h5diff/testfiles/h5diff_615.txt +++ b/tools/test/h5diff/testfiles/h5diff_615.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_621.txt b/tools/test/h5diff/testfiles/h5diff_621.txt index dc8655d..cf1af15 100644 --- a/tools/test/h5diff/testfiles/h5diff_621.txt +++ b/tools/test/h5diff/testfiles/h5diff_621.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_622.txt b/tools/test/h5diff/testfiles/h5diff_622.txt index f20ffbe..3b325de 100644 --- a/tools/test/h5diff/testfiles/h5diff_622.txt +++ b/tools/test/h5diff/testfiles/h5diff_622.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_623.txt b/tools/test/h5diff/testfiles/h5diff_623.txt index f73d202..be34fb4 100644 --- a/tools/test/h5diff/testfiles/h5diff_623.txt +++ b/tools/test/h5diff/testfiles/h5diff_623.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testfiles/h5diff_624.txt b/tools/test/h5diff/testfiles/h5diff_624.txt index 903b738..ebb76af 100644 --- a/tools/test/h5diff/testfiles/h5diff_624.txt +++ b/tools/test/h5diff/testfiles/h5diff_624.txt @@ -37,6 +37,18 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] HDF5 file specified --vol-info-2 VOL-specific info to pass to the VOL connector used for opening the second HDF5 file specified + --vfd-value-1 Value (ID) of the VFL driver to use for opening the + first HDF5 file specified + --vfd-name-1 Name of the VFL driver to use for opening the first + HDF5 file specified + --vfd-info-1 VFD-specific info to pass to the VFL driver used for + opening the first HDF5 file specified + --vfd-value-2 Value (ID) of the VFL driver to use for opening the + second HDF5 file specified + --vfd-name-2 Name of the VFL driver to use for opening the second + HDF5 file specified + --vfd-info-2 VFD-specific info to pass to the VFL driver used for + opening the second HDF5 file specified --follow-symlinks Follow symbolic links (soft links and external links and compare the) links' target objects. @@ -118,7 +130,7 @@ usage: h5diff [OPTIONS] file1 file2 [obj1[ obj2]] This option can be used repeatedly to exclude multiple paths. Modes of output: - Default mode: print the number of differences found and where they occured + Default mode: print the number of differences found and where they occurred -r Report mode: print the above plus the differences -v Verbose mode: print the above plus a list of objects and warnings -q Quiet mode: do not print output diff --git a/tools/test/h5diff/testh5diff.sh.in b/tools/test/h5diff/testh5diff.sh.in index bd27dd0..6871463 100644 --- a/tools/test/h5diff/testh5diff.sh.in +++ b/tools/test/h5diff/testh5diff.sh.in @@ -455,7 +455,7 @@ done # RUNSERIAL is used. Check if it can return exit code from executalbe correctly. if [ -n "$RUNSERIAL_NOEXITCODE" ]; then - echo "***Warning*** Serial Exit Code is not passed back to shell corretly." + echo "***Warning*** Serial Exit Code is not passed back to shell correctly." echo "***Warning*** Exit code checking is skipped." h5haveexitcode=no fi @@ -518,7 +518,7 @@ TOOLTEST() { # don't add exit code check in pmode, as it causes failure. (exit code # is from mpirun not tool) # if any problem occurs relate to an exit code, it will be caught in - # serial mode, so the test is fullfilled. + # serial mode, so the test is fulfilled. if test $h5haveexitcode = 'yes' -a -z "$pmode"; then echo "EXIT CODE: $EXIT_CODE" >> $actual fi @@ -926,7 +926,7 @@ TOOLTEST h5diff_207.txt -c h5diff_basic2.h5 h5diff_basic2.h5 g2/dset8 g2/dset9 TOOLTEST h5diff_208.txt -c h5diff_dset_zero_dim_size1.h5 h5diff_dset_zero_dim_size2.h5 # non-comparable dataset with comparable attribute, and other comparable datasets. -# Also test non-compatible attributes with different type, dimention, rank. +# Also test non-compatible attributes with different type, dimension, rank. # All the comparables should display differences. TOOLTEST h5diff_220.txt -c non_comparables1.h5 non_comparables2.h5 /g1 @@ -1089,7 +1089,7 @@ TOOLTEST h5diff_474.txt -v h5diff_danglelinks1.h5 h5diff_danglelinks2.h5 /ext_li TOOLTEST h5diff_475.txt -v h5diff_danglelinks1.h5 h5diff_danglelinks2.h5 /ext_link1 # ############################################################################## -# # test for group diff recursivly +# # test for group diff recursively # ############################################################################## # root TOOLTEST h5diff_500.txt -v h5diff_grp_recurse1.h5 h5diff_grp_recurse2.h5 / / diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index c0f279d..933aba9 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -863,7 +863,7 @@ # test for named data types ADD_H5_TEST (tcomp-2 0 --enable-error-stack -t /type1 --datatype /type2 --datatype=/group1/type3 tcompound.h5) ADD_H5_TEST_N (tcomp-2 0 --enable-error-stack -N /type1 --any_path /type2 --any_path=/group1/type3 tcompound.h5) - # test for unamed type + # test for unnamed type ADD_H5ERR_MASK_TEST (tcomp-3 0 "object '#6632' doesn't exist" "--enable-error-stack;-t;/#6632;-g;/group2;tcompound.h5") # test complicated compound datatype ADD_H5_TEST (tcomp-4 0 --enable-error-stack tcompound_complex.h5) diff --git a/tools/test/h5dump/CMakeTestsPBITS.cmake b/tools/test/h5dump/CMakeTestsPBITS.cmake index 06be332..0c3297f 100644 --- a/tools/test/h5dump/CMakeTestsPBITS.cmake +++ b/tools/test/h5dump/CMakeTestsPBITS.cmake @@ -158,7 +158,7 @@ # Limits: # Maximum number of packed bits is 8 (for now). # Maximum integer size is 8*sizeof(long long). - # Maximun Offset is Maximum size - 1. + # Maximum Offset is Maximum size - 1. # Maximum Offset+Length is Maximum size. # Tests: # Normal operation on both signed and unsigned int datasets. diff --git a/tools/test/h5dump/CMakeVFDTests.cmake b/tools/test/h5dump/CMakeVFDTests.cmake index 01834b3..7ab8c1f 100644 --- a/tools/test/h5dump/CMakeVFDTests.cmake +++ b/tools/test/h5dump/CMakeVFDTests.cmake @@ -15,20 +15,7 @@ ### T E S T I N G ### ############################################################################## ############################################################################## - -set (VFD_H5DUMP_LIST - sec2 - stdio - core - core_paged - split - multi - family -) - -if (H5_HAVE_DIRECT) - set (VFD_H5DUMP_LIST ${VFD_H5DUMP_LIST} direct) -endif () +H5_CREATE_VFD_DIR() # -------------------------------------------------------------------- # Copy all the HDF5 files from the source directory into the test directory @@ -37,8 +24,7 @@ set (HDF5_VFD_H5DUMP_FILES packedbits ) -foreach (vfdtest ${VFD_H5DUMP_LIST}) - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdname}") +foreach (vfdtest ${VFD_LIST}) foreach (h5_tfile ${HDF5_VFD_H5DUMP_FILES}) HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_tfile}.h5" "${PROJECT_BINARY_DIR}/${vfdtest}/${h5_tfile}.h5" "HDF5_VFD_H5DUMP_files") HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_tfile}.ddl" "${PROJECT_BINARY_DIR}/${vfdtest}/${h5_tfile}.ddl" "HDF5_VFD_H5DUMP_files") @@ -79,7 +65,7 @@ endmacro () ############################################################################## # Run test with different Virtual File Driver -foreach (vfd ${VFD_H5DUMP_LIST}) +foreach (vfd ${VFD_LIST}) # test for signed/unsigned datasets ADD_VFD_H5DUMP_TEST (${vfd} packedbits 0 --enable-error-stack packedbits.h5) endforeach () diff --git a/tools/test/h5dump/binread.c b/tools/test/h5dump/binread.c index d6206ca..341a085 100644 --- a/tools/test/h5dump/binread.c +++ b/tools/test/h5dump/binread.c @@ -59,7 +59,7 @@ usage(void) */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { FILE * stream; size_t numread; diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index e74054c..5258211 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -236,7 +236,7 @@ typedef struct s1_t { #define ARRAY3_DIM2 3 /* VL string datatype name */ -/* TODO remove complier error not used, remove the link when everything is OK */ +/* TODO remove compiler error not used, remove the link when everything is OK */ /* #define VLSTR_TYPE "vl_string_type" */ /* "File 41" macros */ @@ -498,7 +498,7 @@ gent_dataset(void) for (i = 0; i < 30; i++) for (j = 0; j < 20; j++) - dset2[i][j] = 0.0001F * (float)j + (float)i; + dset2[i][j] = 0.0001 * (double)j + (double)i; H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_data); @@ -605,7 +605,7 @@ gent_attribute(void) attr = H5Acreate2(root, "attr3", H5T_IEEE_F64BE, space, H5P_DEFAULT, H5P_DEFAULT); for (i = 0; i < 10; i++) - d[i] = 0.1F * (float)i; + d[i] = 0.1 * (double)i; H5Awrite(attr, H5T_NATIVE_DOUBLE, d); H5Sclose(space); @@ -1019,7 +1019,7 @@ gent_compound_dt(void) for (i = 0; i < (int)sdim; i++) { dset1[i].a = i; dset1[i].b = (float)(i * i); - dset1[i].c = (float)(1.0F / (float)(i + 1)); + dset1[i].c = (double)(1.0 / (double)(i + 1)); dset2[i].a = i; dset2[i].b = (float)((float)i + (float)i * 0.1F); @@ -1128,7 +1128,7 @@ gent_compound_dt(void) H5Dclose(dataset); H5Gclose(group); - /* unamed data type */ + /* unnamed data type */ group = H5Gcreate2(fid, "/group2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); type = H5Tcreate(H5T_COMPOUND, sizeof(dset5_t)); @@ -1206,7 +1206,7 @@ gent_compound_dt2(void) for (i = 0; i < (int)sdim; i++) { dset1[i].a = i; dset1[i].b = (float)(i * i); - dset1[i].c = (float)(1.0F / (float)(i + 1)); + dset1[i].c = (double)(1.0 / (double)(i + 1)); dset2[i].a = i; dset2[i].b = (float)((float)i + (float)i * 0.1F); @@ -1312,7 +1312,7 @@ gent_compound_dt2(void) H5Dclose(dataset); H5Gclose(group); - /* unamed data type */ + /* unnamed data type */ group = H5Gcreate2(fid, "/group2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); type = H5Tcreate(H5T_COMPOUND, sizeof(dset5_t)); @@ -1522,11 +1522,11 @@ gent_loop2(void) fid = H5Fcreate(FILE11, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - /* create group object g1 and implcit path from root object */ + /* create group object g1 and implicit path from root object */ group = H5Gcreate2(fid, "/g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Gclose(group); - /* create group object g2 and implcit path from root object */ + /* create group object g2 and implicit path from root object */ group = H5Gcreate2(fid, "/g2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Gclose(group); @@ -1646,7 +1646,7 @@ gent_many(void) space2 = H5Screate_simple(1, dims, NULL); attr = H5Acreate2(dataset, "attr3", H5T_IEEE_F64BE, space2, H5P_DEFAULT, H5P_DEFAULT); for (i = 0; i < 10; i++) - d[i] = 0.1F * (float)i; + d[i] = 0.1 * (double)i; H5Awrite(attr, H5T_NATIVE_DOUBLE, d); H5Sclose(space2); H5Aclose(attr); @@ -2468,7 +2468,7 @@ gent_attrreg(void) static void gent_nestcomp(void) { - /* Compound memeber of the compound datatype*/ + /* Compound member of the compound datatype*/ typedef struct cmp_t { char a; float b[2]; @@ -2503,7 +2503,7 @@ gent_nestcomp(void) for (i = 0; i < 10; i++) { s1[i].a = i; s1[i].b = (float)(i * i); - s1[i].c = 1.0F / (float)(i + 1); + s1[i].c = 1.0 / (double)(i + 1); s1[i].d.a = (char)(65 + i); s1[i].d.b[0] = -100.0F; s1[i].d.b[1] = 100.0F; @@ -2541,7 +2541,7 @@ gent_nestcomp(void) H5Tinsert(s2_tid, "c_name", HOFFSET(s2_t, c), H5T_NATIVE_DOUBLE); H5Tinsert(s2_tid, "b_name", HOFFSET(s2_t, b), H5T_NATIVE_FLOAT); - /* Insert compound memeber created above */ + /* Insert compound member created above */ H5Tinsert(s2_tid, "d_name", HOFFSET(s2_t, d), cmp_tid); /* @@ -5077,7 +5077,7 @@ gent_compound_complex(void) Array1[m].e = (float)((float)m * 0.96F); for (n = 0; n < F41_ARRAY_DIMf; n++) - Array1[m].f[n] = ((float)m * 1024.9637F); + Array1[m].f[n] = ((double)m * 1024.9637); Array1[m].g = 'm'; } @@ -5218,7 +5218,7 @@ gent_named_dtype_attr(void) sid = H5Screate(H5S_SCALAR); HDassert(sid > 0); - /* Create attribute on commited datatype */ + /* Create attribute on committed datatype */ aid = H5Acreate2(tid, F42_ATTRNAME, H5T_STD_I32LE, sid, H5P_DEFAULT, H5P_DEFAULT); HDassert(aid > 0); @@ -5735,7 +5735,7 @@ gent_filters(void) HDassert(ret >= 0); if (supports_comments & H5VL_OPT_QUERY_SUPPORTED) { - ret = H5Oset_comment_by_name(fid, "mytype", "This is a commited datatype", H5P_DEFAULT); + ret = H5Oset_comment_by_name(fid, "mytype", "This is a committed datatype", H5P_DEFAULT); HDassert(ret >= 0); } @@ -6361,7 +6361,7 @@ gent_ldouble_scalar(void) hid_t tid; hid_t sid; hsize_t dims[1] = {6}; - long double buf[6] = {0.0, 1.0, 2.0, 3.0, 4.0, 5.0}; + long double buf[6] = {0.0L, 1.0L, 2.0L, 3.0L, 4.0L, 5.0L}; if ((fid = H5Fcreate(FILE88, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; @@ -6803,7 +6803,7 @@ gent_attr_creation_order(void) goto out; /*------------------------------------------------------------------------- - * create a dataset with creation order tracked for attributes and atributes in it + * create a dataset with creation order tracked for attributes and attributes in it *------------------------------------------------------------------------- */ @@ -6827,7 +6827,7 @@ gent_attr_creation_order(void) did = H5I_INVALID_HID; /*------------------------------------------------------------------------- - * create a dataset without creation order tracked for attributes and atributes in it + * create a dataset without creation order tracked for attributes and attributes in it *------------------------------------------------------------------------- */ @@ -6851,7 +6851,7 @@ gent_attr_creation_order(void) did = H5I_INVALID_HID; /*------------------------------------------------------------------------- - * create a group with creation order tracked for attributes and atributes in it + * create a group with creation order tracked for attributes and attributes in it *------------------------------------------------------------------------- */ @@ -6874,7 +6874,7 @@ gent_attr_creation_order(void) gid = H5I_INVALID_HID; /*------------------------------------------------------------------------- - * create a group without creation order tracked for attributes and atributes in it + * create a group without creation order tracked for attributes and attributes in it *------------------------------------------------------------------------- */ @@ -6897,7 +6897,7 @@ gent_attr_creation_order(void) gid = H5I_INVALID_HID; /*------------------------------------------------------------------------- - * create a named datatype with creation order tracked for attributes and atributes in it + * create a named datatype with creation order tracked for attributes and attributes in it *------------------------------------------------------------------------- */ @@ -6923,7 +6923,7 @@ gent_attr_creation_order(void) tid = H5I_INVALID_HID; /*------------------------------------------------------------------------- - * create a named datatype without creation order tracked for attributes and atributes in it + * create a named datatype without creation order tracked for attributes and attributes in it *------------------------------------------------------------------------- */ @@ -7022,7 +7022,7 @@ gent_fpformat(void) { hid_t fid, sid, did; hsize_t dims[1] = {6}; - double dbuf[6] = {-0.1234567f, 0.1234567f, 0, 0, 0, 0}; + double dbuf[6] = {-0.1234567, 0.1234567, 0, 0, 0, 0}; float fbuf[6] = {-0.1234567f, 0.1234567f, 0, 0, 0, 0}; fid = H5Fcreate(FILE60, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -7264,15 +7264,33 @@ gent_packedbits(void) hid_t space = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F66_XDIM][F66_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F66_XDIM][F66_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F66_XDIM][F66_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F66_XDIM][F66_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F66_XDIM][F66_YDIM8]; + } * dset8; + struct { + int16_t arr[F66_XDIM][F66_YDIM16]; + } * dset16; + struct { + int32_t arr[F66_XDIM][F66_YDIM32]; + } * dset32; + struct { + int64_t arr[F66_XDIM][F66_YDIM64]; + } * dset64; + struct { + double arr[F66_XDIM][F66_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -7286,15 +7304,15 @@ gent_packedbits(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F66_XDIM, F66_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE66, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -7306,13 +7324,13 @@ gent_packedbits(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); valu8bits = (uint8_t)(valu8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); H5Sclose(space); H5Dclose(dataset); @@ -7324,13 +7342,13 @@ gent_packedbits(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); valu16bits = (uint16_t)(valu16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); H5Sclose(space); H5Dclose(dataset); @@ -7342,13 +7360,13 @@ gent_packedbits(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; valu32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); H5Sclose(space); H5Dclose(dataset); @@ -7360,13 +7378,13 @@ gent_packedbits(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; valu64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); H5Sclose(space); H5Dclose(dataset); @@ -7378,13 +7396,13 @@ gent_packedbits(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); val8bits = (int8_t)(val8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8[0]); + H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); H5Sclose(space); H5Dclose(dataset); @@ -7396,13 +7414,13 @@ gent_packedbits(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); val16bits = (int16_t)(val16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16[0]); + H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); H5Sclose(space); H5Dclose(dataset); @@ -7414,13 +7432,13 @@ gent_packedbits(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; val32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); H5Sclose(space); H5Dclose(dataset); @@ -7432,13 +7450,13 @@ gent_packedbits(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; val64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); H5Sclose(space); H5Dclose(dataset); @@ -7450,9 +7468,9 @@ gent_packedbits(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001F * (float)j + (float)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl[0]); + H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); H5Sclose(space); H5Dclose(dataset); @@ -7489,15 +7507,33 @@ gent_attr_intsize(void) hid_t root = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F66_XDIM][F66_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F66_XDIM][F66_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F66_XDIM][F66_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F66_XDIM][F66_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F66_XDIM][F66_YDIM8]; + } * dset8; + struct { + int16_t arr[F66_XDIM][F66_YDIM16]; + } * dset16; + struct { + int32_t arr[F66_XDIM][F66_YDIM64]; + } * dset32; + struct { + int64_t arr[F66_XDIM][F66_YDIM64]; + } * dset64; + struct { + double arr[F66_XDIM][F66_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -7511,15 +7547,15 @@ gent_attr_intsize(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F66_XDIM, F66_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE69, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); root = H5Gopen2(fid, "/", H5P_DEFAULT); @@ -7532,14 +7568,14 @@ gent_attr_intsize(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } - H5Awrite(attr, H5T_NATIVE_UINT8, dsetu8[0]); + H5Awrite(attr, H5T_NATIVE_UINT8, dsetu8); H5Sclose(space); H5Aclose(attr); @@ -7551,14 +7587,14 @@ gent_attr_intsize(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } - H5Awrite(attr, H5T_NATIVE_UINT16, dsetu16[0]); + H5Awrite(attr, H5T_NATIVE_UINT16, dsetu16); H5Sclose(space); H5Aclose(attr); @@ -7570,14 +7606,14 @@ gent_attr_intsize(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_UINT32, dsetu32[0]); + H5Awrite(attr, H5T_NATIVE_UINT32, dsetu32); H5Sclose(space); H5Aclose(attr); @@ -7589,14 +7625,14 @@ gent_attr_intsize(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_UINT64, dsetu64[0]); + H5Awrite(attr, H5T_NATIVE_UINT64, dsetu64); H5Sclose(space); H5Aclose(attr); @@ -7608,14 +7644,14 @@ gent_attr_intsize(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } - H5Awrite(attr, H5T_NATIVE_INT8, dset8[0]); + H5Awrite(attr, H5T_NATIVE_INT8, dset8); H5Sclose(space); H5Aclose(attr); @@ -7627,14 +7663,14 @@ gent_attr_intsize(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } - H5Awrite(attr, H5T_NATIVE_INT16, dset16[0]); + H5Awrite(attr, H5T_NATIVE_INT16, dset16); H5Sclose(space); H5Aclose(attr); @@ -7646,14 +7682,14 @@ gent_attr_intsize(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_INT32, dset32[0]); + H5Awrite(attr, H5T_NATIVE_INT32, dset32); H5Sclose(space); H5Aclose(attr); @@ -7665,14 +7701,14 @@ gent_attr_intsize(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_INT64, dset64[0]); + H5Awrite(attr, H5T_NATIVE_INT64, dset64); H5Sclose(space); H5Aclose(attr); @@ -7684,9 +7720,9 @@ gent_attr_intsize(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001F * (float)j + (float)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Awrite(attr, H5T_NATIVE_DOUBLE, dsetdbl[0]); + H5Awrite(attr, H5T_NATIVE_DOUBLE, dsetdbl); H5Sclose(space); H5Aclose(attr); @@ -7994,7 +8030,7 @@ gent_compound_intsizes(void) for (n = 0; n < (int)dims[0]; n++) for (o = 0; o < (int)dims[1]; o++) - Array1[m].dsetdbl[n][o] = 0.0001F * (float)o + (float)n; + Array1[m].dsetdbl[n][o] = 0.0001 * (double)o + (double)n; } /* Create the array data type for the 8 bits signed int array */ @@ -8284,7 +8320,7 @@ gent_compound_attr_intsizes(void) for (n = 0; n < (int)dims[0]; n++) for (o = 0; o < (int)dims[1]; o++) - Array1[m].dsetdbl[n][o] = 0.0001F * (float)o + (float)n; + Array1[m].dsetdbl[n][o] = 0.0001 * (double)o + (double)n; } /* Create the array data type for the 8 bits signed int array */ @@ -8586,15 +8622,33 @@ gent_intscalars(void) hid_t tid = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F73_XDIM][F73_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F73_XDIM][F73_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F73_XDIM][F73_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F73_XDIM][F73_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F73_XDIM][F73_YDIM8]; + } * dset8; + struct { + int16_t arr[F73_XDIM][F73_YDIM16]; + } * dset16; + struct { + int32_t arr[F73_XDIM][F73_YDIM32]; + } * dset32; + struct { + int64_t arr[F73_XDIM][F73_YDIM64]; + } * dset64; + struct { + double arr[F73_XDIM][F73_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -8608,15 +8662,15 @@ gent_intscalars(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F73_XDIM, F73_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE73, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -8629,14 +8683,14 @@ gent_intscalars(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); H5Sclose(space); H5Dclose(dataset); @@ -8649,14 +8703,14 @@ gent_intscalars(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); H5Sclose(space); H5Dclose(dataset); @@ -8669,14 +8723,14 @@ gent_intscalars(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); H5Sclose(space); H5Dclose(dataset); @@ -8689,14 +8743,14 @@ gent_intscalars(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); H5Sclose(space); H5Dclose(dataset); @@ -8709,14 +8763,14 @@ gent_intscalars(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); H5Sclose(space); H5Dclose(dataset); @@ -8729,14 +8783,14 @@ gent_intscalars(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); H5Sclose(space); H5Dclose(dataset); @@ -8749,14 +8803,14 @@ gent_intscalars(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); H5Sclose(space); H5Dclose(dataset); @@ -8769,14 +8823,14 @@ gent_intscalars(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); H5Sclose(space); H5Dclose(dataset); @@ -8789,9 +8843,9 @@ gent_intscalars(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001F * (float)j + (float)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); H5Sclose(space); H5Dclose(dataset); @@ -8829,15 +8883,33 @@ gent_attr_intscalars(void) hid_t tid = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F73_XDIM][F73_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F73_XDIM][F73_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F73_XDIM][F73_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F73_XDIM][F73_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F73_XDIM][F73_YDIM8]; + } * dset8; + struct { + int16_t arr[F73_XDIM][F73_YDIM16]; + } * dset16; + struct { + int32_t arr[F73_XDIM][F73_YDIM32]; + } * dset32; + struct { + int64_t arr[F73_XDIM][F73_YDIM64]; + } * dset64; + struct { + double arr[F73_XDIM][F73_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -8851,15 +8923,15 @@ gent_attr_intscalars(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F73_XDIM, F73_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE74, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); root = H5Gopen2(fid, "/", H5P_DEFAULT); @@ -8873,14 +8945,14 @@ gent_attr_intscalars(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } - H5Awrite(attr, tid, dsetu8[0]); + H5Awrite(attr, tid, dsetu8); H5Sclose(space); H5Aclose(attr); @@ -8893,14 +8965,14 @@ gent_attr_intscalars(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } - H5Awrite(attr, tid, dsetu16[0]); + H5Awrite(attr, tid, dsetu16); H5Sclose(space); H5Aclose(attr); @@ -8913,14 +8985,14 @@ gent_attr_intscalars(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } - H5Awrite(attr, tid, dsetu32[0]); + H5Awrite(attr, tid, dsetu32); H5Sclose(space); H5Aclose(attr); @@ -8933,14 +9005,14 @@ gent_attr_intscalars(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } - H5Awrite(attr, tid, dsetu64[0]); + H5Awrite(attr, tid, dsetu64); H5Sclose(space); H5Aclose(attr); @@ -8953,14 +9025,14 @@ gent_attr_intscalars(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } - H5Awrite(attr, tid, dset8[0]); + H5Awrite(attr, tid, dset8); H5Sclose(space); H5Aclose(attr); @@ -8973,14 +9045,14 @@ gent_attr_intscalars(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } - H5Awrite(attr, tid, dset16[0]); + H5Awrite(attr, tid, dset16); H5Sclose(space); H5Aclose(attr); @@ -8993,14 +9065,14 @@ gent_attr_intscalars(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } - H5Awrite(attr, tid, dset32[0]); + H5Awrite(attr, tid, dset32); H5Sclose(space); H5Aclose(attr); @@ -9013,14 +9085,14 @@ gent_attr_intscalars(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } - H5Awrite(attr, tid, dset64[0]); + H5Awrite(attr, tid, dset64); H5Sclose(space); H5Aclose(attr); @@ -9033,9 +9105,9 @@ gent_attr_intscalars(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001F * (float)j + (float)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Awrite(attr, tid, dsetdbl[0]); + H5Awrite(attr, tid, dsetdbl); H5Sclose(space); H5Aclose(attr); @@ -9236,7 +9308,7 @@ gent_compound_int_array(void) dims[0] = F76_DIM8; for (n = 0; n < (int)dims[0]; n++) - Cmpd1[m].dsetdbl[n] = 0.0001F + (float)n; + Cmpd1[m].dsetdbl[n] = 0.0001 + (double)n; } /* Create the array data type for the 8 bits signed int array */ @@ -9477,8 +9549,8 @@ gent_compound_ints(void) val64bits <<= 1; /* Double Dummy set for failure tests */ - Cmpd1[m].dsetdbl = 0.0001F + (float)m; - Cmpd2[m].dsetdbl = 0.0001F + (float)m; + Cmpd1[m].dsetdbl = 0.0001 + (double)m; + Cmpd2[m].dsetdbl = 0.0001 + (double)m; } /* Create the dataspace */ @@ -9618,15 +9690,33 @@ gent_intattrscalars(void) hid_t tid = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F73_XDIM][F73_YDIM8]; + } *dsetu8 = NULL; + struct { + uint16_t arr[F73_XDIM][F73_YDIM16]; + } *dsetu16 = NULL; + struct { + uint32_t arr[F73_XDIM][F73_YDIM32]; + } *dsetu32 = NULL; + struct { + uint64_t arr[F73_XDIM][F73_YDIM64]; + } *dsetu64 = NULL; + struct { + int8_t arr[F73_XDIM][F73_YDIM8]; + } *dset8 = NULL; + struct { + int16_t arr[F73_XDIM][F73_YDIM16]; + } *dset16 = NULL; + struct { + int32_t arr[F73_XDIM][F73_YDIM32]; + } *dset32 = NULL; + struct { + int64_t arr[F73_XDIM][F73_YDIM64]; + } *dset64 = NULL; + struct { + double arr[F73_XDIM][F73_YDIM8]; + } *dsetdbl = NULL; uint8_t valu8bits; uint16_t valu16bits; @@ -9640,15 +9730,15 @@ gent_intattrscalars(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F73_XDIM, F73_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE78, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -9661,9 +9751,9 @@ gent_intattrscalars(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } @@ -9671,7 +9761,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); /* Attribute of 8 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU08, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu8[0]); + H5Awrite(attr, tid, dsetu8); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9685,9 +9775,9 @@ gent_intattrscalars(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } @@ -9695,7 +9785,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); /* Attribute of 16 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU16, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu16[0]); + H5Awrite(attr, tid, dsetu16); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9709,9 +9799,9 @@ gent_intattrscalars(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } @@ -9719,7 +9809,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); /* Attribute of 32 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU32, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu32[0]); + H5Awrite(attr, tid, dsetu32); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9733,9 +9823,9 @@ gent_intattrscalars(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } @@ -9743,7 +9833,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); /* Attribute of 64 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU64, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu64[0]); + H5Awrite(attr, tid, dsetu64); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9757,9 +9847,9 @@ gent_intattrscalars(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } @@ -9767,7 +9857,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); /* Attribute of 8 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS08, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset8[0]); + H5Awrite(attr, tid, dset8); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9781,9 +9871,9 @@ gent_intattrscalars(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } @@ -9791,7 +9881,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); /* Attribute of 16 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS16, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset16[0]); + H5Awrite(attr, tid, dset16); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9805,9 +9895,9 @@ gent_intattrscalars(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } @@ -9815,7 +9905,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); /* Attribute of 32 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS32, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset32[0]); + H5Awrite(attr, tid, dset32); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9829,9 +9919,9 @@ gent_intattrscalars(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } @@ -9839,7 +9929,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); /* Attribute of 64 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS64, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset64[0]); + H5Awrite(attr, tid, dset64); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9853,12 +9943,12 @@ gent_intattrscalars(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001F * (float)j + (float)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); /* Attribute of double */ attr = H5Acreate2(dataset, F73_DUMMYDBL, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetdbl[0]); + H5Awrite(attr, tid, dsetdbl); H5Aclose(attr); H5Sclose(space); @@ -9896,15 +9986,33 @@ gent_intsattrs(void) hid_t aspace = H5I_INVALID_HID; hsize_t dims[2], adims[1]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F66_XDIM][F66_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F66_XDIM][F66_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F66_XDIM][F66_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F66_XDIM][F66_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F66_XDIM][F66_YDIM8]; + } * dset8; + struct { + int16_t arr[F66_XDIM][F66_YDIM16]; + } * dset16; + struct { + int32_t arr[F66_XDIM][F66_YDIM32]; + } * dset32; + struct { + int64_t arr[F66_XDIM][F66_YDIM64]; + } * dset64; + struct { + double arr[F66_XDIM][F66_YDIM8]; + } * dsetdbl; uint8_t * asetu8 = NULL; uint16_t *asetu16 = NULL; @@ -9928,15 +10036,15 @@ gent_intsattrs(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F66_XDIM, F66_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); asetu8 = HDcalloc(F66_XDIM * F66_YDIM8, sizeof(uint8_t)); asetu16 = HDcalloc(F66_XDIM * F66_YDIM16, sizeof(uint16_t)); @@ -9958,16 +10066,16 @@ gent_intsattrs(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; - asetu8[i * dims[1]] = dsetu8[i][0]; + dsetu8->arr[i][0] = valu8bits; + asetu8[i * dims[1]] = dsetu8->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); - asetu8[i * dims[1] + j] = dsetu8[i][j]; + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); + asetu8[i * dims[1] + j] = dsetu8->arr[i][j]; } valu8bits = (uint8_t)(valu8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); /* Attribute of 8 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM8; aspace = H5Screate_simple(1, adims, NULL); @@ -9986,16 +10094,16 @@ gent_intsattrs(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; - asetu16[i * dims[1]] = dsetu16[i][0]; + dsetu16->arr[i][0] = valu16bits; + asetu16[i * dims[1]] = dsetu16->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); - asetu16[i * dims[1] + j] = dsetu16[i][j]; + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); + asetu16[i * dims[1] + j] = dsetu16->arr[i][j]; } valu16bits = (uint16_t)(valu16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); /* Attribute of 16 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM16; aspace = H5Screate_simple(1, adims, NULL); @@ -10014,16 +10122,16 @@ gent_intsattrs(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; - asetu32[i * dims[1]] = dsetu32[i][0]; + dsetu32->arr[i][0] = valu32bits; + asetu32[i * dims[1]] = dsetu32->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; - asetu32[i * dims[1] + j] = dsetu32[i][j]; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; + asetu32[i * dims[1] + j] = dsetu32->arr[i][j]; } valu32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); /* Attribute of 32 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM32; aspace = H5Screate_simple(1, adims, NULL); @@ -10042,16 +10150,16 @@ gent_intsattrs(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; - asetu64[i * dims[1]] = dsetu64[i][0]; + dsetu64->arr[i][0] = valu64bits; + asetu64[i * dims[1]] = dsetu64->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; - asetu64[i * dims[1] + j] = dsetu64[i][j]; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; + asetu64[i * dims[1] + j] = dsetu64->arr[i][j]; } valu64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); /* Attribute of 64 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM64; aspace = H5Screate_simple(1, adims, NULL); @@ -10070,16 +10178,16 @@ gent_intsattrs(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; - aset8[i * dims[1]] = dset8[i][0]; + dset8->arr[i][0] = val8bits; + aset8[i * dims[1]] = dset8->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); - aset8[i * dims[1] + j] = dset8[i][j]; + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); + aset8[i * dims[1] + j] = dset8->arr[i][j]; } val8bits = (int8_t)(val8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8[0]); + H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); /* Attribute of 8 bits signed int */ adims[0] = F66_XDIM * F66_YDIM8; aspace = H5Screate_simple(1, adims, NULL); @@ -10098,16 +10206,16 @@ gent_intsattrs(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; - aset16[i * dims[1]] = dset16[i][0]; + dset16->arr[i][0] = val16bits; + aset16[i * dims[1]] = dset16->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); - aset16[i * dims[1] + j] = dset16[i][j]; + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); + aset16[i * dims[1] + j] = dset16->arr[i][j]; } val16bits = (int16_t)(val16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16[0]); + H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); /* Attribute of 16 bits signed int */ adims[0] = F66_XDIM * F66_YDIM16; aspace = H5Screate_simple(1, adims, NULL); @@ -10126,16 +10234,16 @@ gent_intsattrs(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; - aset32[i * dims[1]] = dset32[i][0]; + dset32->arr[i][0] = val32bits; + aset32[i * dims[1]] = dset32->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; - aset32[i * dims[1] + j] = dset32[i][j]; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; + aset32[i * dims[1] + j] = dset32->arr[i][j]; } val32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); /* Attribute of 32 bits signed int */ adims[0] = F66_XDIM * F66_YDIM32; aspace = H5Screate_simple(1, adims, NULL); @@ -10154,16 +10262,16 @@ gent_intsattrs(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; - aset64[i * dims[1]] = dset64[i][0]; + dset64->arr[i][0] = val64bits; + aset64[i * dims[1]] = dset64->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; - aset64[i * dims[1] + j] = dset64[i][j]; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; + aset64[i * dims[1] + j] = dset64->arr[i][j]; } val64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); /* Attribute of 64 bits signed int */ adims[0] = F66_XDIM * F66_YDIM64; aspace = H5Screate_simple(1, adims, NULL); @@ -10182,11 +10290,11 @@ gent_intsattrs(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) { - dsetdbl[i][j] = 0.0001F * (float)j + (float)i; - asetdbl[i * dims[1] + j] = dsetdbl[i][j]; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; + asetdbl[i * dims[1] + j] = dsetdbl->arr[i][j]; } - H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl[0]); + H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); /* Attribute of double */ adims[0] = F66_XDIM * F66_YDIM8; aspace = H5Screate_simple(1, adims, NULL); @@ -10238,9 +10346,15 @@ gent_floatsattrs(void) hid_t aspace = H5I_INVALID_HID; hsize_t dims[2], adims[1]; - float ** dset32 = NULL; - double ** dset64 = NULL; - long double **dset128 = NULL; + struct { + float arr[F89_XDIM][F89_YDIM32]; + } * dset32; + struct { + double arr[F89_XDIM][F89_YDIM64]; + } * dset64; + struct { + long double arr[F89_XDIM][F89_YDIM128]; + } * dset128; float * aset32 = NULL; double * aset64 = NULL; @@ -10253,9 +10367,9 @@ gent_floatsattrs(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dset32, float, F89_XDIM, F89_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, double, F89_XDIM, F89_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset128, long double, F89_XDIM, F89_YDIM128); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dset128 = malloc(sizeof(*dset128)); aset32 = HDcalloc(F89_XDIM * F89_YDIM32, sizeof(float)); aset64 = HDcalloc(F89_XDIM * F89_YDIM64, sizeof(double)); @@ -10277,16 +10391,16 @@ gent_floatsattrs(void) val32bits = (float)F89_YDIM32; for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; - aset32[i * dims[1]] = dset32[i][0]; + dset32->arr[i][0] = val32bits; + aset32[i * dims[1]] = dset32->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = (float)(j * dims[0] + i) / (float)F89_YDIM32; - aset32[i * dims[1] + j] = dset32[i][j]; + dset32->arr[i][j] = (float)(j * dims[0] + i) / (float)F89_YDIM32; + aset32[i * dims[1] + j] = dset32->arr[i][j]; } val32bits -= (float)1; } - H5Dwrite(dataset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); /* Attribute of 32 bits float */ adims[0] = F89_XDIM * F89_YDIM32; aspace = H5Screate_simple(1, adims, NULL); @@ -10305,16 +10419,16 @@ gent_floatsattrs(void) val64bits = (double)F89_YDIM64; for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; - aset64[i * dims[1]] = dset64[i][0]; + dset64->arr[i][0] = val64bits; + aset64[i * dims[1]] = dset64->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = (double)(j * dims[0] + i) / (double)F89_YDIM64; - aset64[i * dims[1] + j] = dset64[i][j]; + dset64->arr[i][j] = (double)(j * dims[0] + i) / (double)F89_YDIM64; + aset64[i * dims[1] + j] = dset64->arr[i][j]; } val64bits -= (double)1; } - H5Dwrite(dataset, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); /* Attribute of 64 bits double */ adims[0] = F89_XDIM * F89_YDIM64; aspace = H5Screate_simple(1, adims, NULL); @@ -10333,16 +10447,16 @@ gent_floatsattrs(void) val128bits = (long double)F89_YDIM128; for (i = 0; i < dims[0]; i++) { - dset128[i][0] = val128bits; - aset128[i * dims[1]] = dset128[i][0]; + dset128->arr[i][0] = val128bits; + aset128[i * dims[1]] = dset128->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset128[i][j] = (long double)(j * dims[0] + i) / (long double)F89_YDIM128; - aset128[i * dims[1] + j] = dset128[i][j]; + dset128->arr[i][j] = (long double)(j * dims[0] + i) / (long double)F89_YDIM128; + aset128[i * dims[1] + j] = dset128->arr[i][j]; } val128bits -= (long double)1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset128[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset128); /* Attribute of 128 bits long double */ adims[0] = F89_XDIM * F89_YDIM128; aspace = H5Screate_simple(1, adims, NULL); @@ -10534,13 +10648,17 @@ gent_bitnopaquefields(void) static void gent_intsfourdims(void) { - hid_t fid, dataset, space; - hsize_t dims[F81_RANK]; - uint32_t dset1[F81_ZDIM][F81_YDIM][F81_XDIM][F81_WDIM]; + hid_t fid, dataset, space; + hsize_t dims[F81_RANK]; + struct { + uint32_t arr[F81_ZDIM][F81_YDIM][F81_XDIM][F81_WDIM]; + } * dset1; unsigned int i, j, k, l; fid = H5Fcreate(FILE81, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + dset1 = malloc(sizeof(*dset1)); + /* Dataset of 32 bits unsigned int */ dims[0] = F81_ZDIM; dims[1] = F81_YDIM; @@ -10553,7 +10671,7 @@ gent_intsfourdims(void) for (j = 0; j < F81_YDIM; j++) for (k = 0; k < F81_XDIM; k++) for (l = 0; l < F81_WDIM; l++) - dset1[i][j][k][l] = + dset1->arr[i][j][k][l] = i * F81_YDIM * F81_XDIM * F81_WDIM + j * F81_XDIM * F81_WDIM + k * F81_WDIM + l; H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset1); @@ -10561,6 +10679,7 @@ gent_intsfourdims(void) H5Dclose(dataset); H5Fclose(fid); + free(dset1); } /*------------------------------------------------------------------------- diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index 9464302..f9f3734 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -1171,7 +1171,7 @@ TOOLTEST tcomp-1.ddl --enable-error-stack tcompound.h5 # test for named data types TOOLTEST tcomp-2.ddl --enable-error-stack -t /type1 --datatype /type2 --datatype=/group1/type3 tcompound.h5 TOOLTEST tcomp-2.ddl --enable-error-stack -N /type1 --any_path /type2 --any_path=/group1/type3 tcompound.h5 -# test for unamed type +# test for unnamed type TOOLTEST4 tcomp-3.ddl --enable-error-stack -t /#6632 -g /group2 tcompound.h5 # test complicated compound datatype TOOLTEST tcomp-4.ddl --enable-error-stack tcompound_complex.h5 diff --git a/tools/test/h5dump/testh5dumppbits.sh.in b/tools/test/h5dump/testh5dumppbits.sh.in index 6689749..eaf91fd 100644 --- a/tools/test/h5dump/testh5dumppbits.sh.in +++ b/tools/test/h5dump/testh5dumppbits.sh.in @@ -516,7 +516,7 @@ TOOLTEST tnofilename-with-packed-bits.ddl --enable-error-stack # Limits: # Maximum number of packed bits is 8 (for now). # Maximum integer size is 64 (for now). -# Maximun Offset is 63 (Maximum size - 1). +# Maximum Offset is 63 (Maximum size - 1). # Maximum Offset+Length is 64 (Maximum size). # Tests: # Normal operation on both signed and unsigned int datasets. diff --git a/tools/test/h5format_convert/h5fc_gentest.c b/tools/test/h5format_convert/h5fc_gentest.c index 09cca45..ccb64b5 100644 --- a/tools/test/h5format_convert/h5fc_gentest.c +++ b/tools/test/h5format_convert/h5fc_gentest.c @@ -381,7 +381,7 @@ error: /* * Function: gen_err_level() * - * Generate a file to test the situtation described in HDFFV-9434: + * Generate a file to test the situation described in HDFFV-9434: * Exceed the limit of v1-btree level * * Create a file with H5Pset_istore_k(fcpl, 1). diff --git a/tools/test/h5format_convert/testh5fc.sh.in b/tools/test/h5format_convert/testh5fc.sh.in index 762f43d..df083d1 100644 --- a/tools/test/h5format_convert/testh5fc.sh.in +++ b/tools/test/h5format_convert/testh5fc.sh.in @@ -313,7 +313,7 @@ TOOLTEST_MASK_OUT() { $RM $actual $actual_err $actual_sav $actual_err_sav fi } -# Same as TOOLTEST_OUT except only compares error outout +# Same as TOOLTEST_OUT except only compares error output TOOLTEST_ERR() { # Prepare expected and actual output expect="$TESTDIR/$1" diff --git a/tools/test/h5jam/getub.c b/tools/test/h5jam/getub.c index fd21d6c..4e67e98 100644 --- a/tools/test/h5jam/getub.c +++ b/tools/test/h5jam/getub.c @@ -15,7 +15,7 @@ #include "h5tools.h" #include "h5tools_utils.h" -void parse_command_line(int argc, const char *argv[]); +void parse_command_line(int argc, const char *const *argv); /* Name of tool */ #define PROGRAM_NAME "getub" @@ -52,7 +52,7 @@ usage(const char *prog) *------------------------------------------------------------------------- */ void -parse_command_line(int argc, const char *argv[]) +parse_command_line(int argc, const char *const *argv) { int opt; @@ -77,7 +77,7 @@ parse_command_line(int argc, const char *argv[]) } /* end parse_command_line() */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { int fd = H5I_INVALID_HID; unsigned size; @@ -91,7 +91,7 @@ main(int argc, const char *argv[]) /* Initialize h5tools lib */ h5tools_init(); - parse_command_line(argc, argv); + parse_command_line(argc, (const char *const *)argv); if (NULL == nbytes) { /* missing arg */ diff --git a/tools/test/h5jam/tellub.c b/tools/test/h5jam/tellub.c index e6769ec..f264d9b 100644 --- a/tools/test/h5jam/tellub.c +++ b/tools/test/h5jam/tellub.c @@ -56,7 +56,7 @@ usage(const char *prog) */ static void -parse_command_line(int argc, const char *argv[]) +parse_command_line(int argc, const char *const *argv) { int opt; @@ -98,7 +98,7 @@ leave(int ret) *------------------------------------------------------------------------- */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { char * ifname; hid_t ifile = H5I_INVALID_HID; @@ -113,7 +113,7 @@ main(int argc, const char *argv[]) /* Initialize h5tools lib */ h5tools_init(); - parse_command_line(argc, argv); + parse_command_line(argc, (const char *const *)argv); /* enable error reporting if command line option */ h5tools_error_report(); diff --git a/tools/test/h5jam/testh5jam.sh.in b/tools/test/h5jam/testh5jam.sh.in index caa3416..191fef0 100644 --- a/tools/test/h5jam/testh5jam.sh.in +++ b/tools/test/h5jam/testh5jam.sh.in @@ -345,7 +345,7 @@ CHECK_NOUB() { # JAMTEST user_block file.h5 [--clobber] [ofile.h5] # # Test the 'jam' tool: -# 1. figure out the input and output, and the comparision +# 1. figure out the input and output, and the comparison # that will be done. # 2. call 'jam' with the appropriate arguments # 3. check the user block is correct in the output (Check_UB) diff --git a/tools/test/h5ls/h5ls_plugin.sh.in b/tools/test/h5ls/h5ls_plugin.sh.in index 29e982d..0077b07 100644 --- a/tools/test/h5ls/h5ls_plugin.sh.in +++ b/tools/test/h5ls/h5ls_plugin.sh.in @@ -166,7 +166,7 @@ VERIFY() { # the `nerrors' global variable and (if $verbose is set) display up to $NLINS # lines of the actual output from the tool test. The actual output is not # removed if $HDF5_NOCLEANUP has a non-zero value. -# Arguemnts: +# Arguments: # $1 -- actual output filename to use # $2 and on -- argument for the h5ls tool TOOLTEST() { diff --git a/tools/test/h5ls/testh5ls.sh.in b/tools/test/h5ls/testh5ls.sh.in index ee0dad7..36583da 100644 --- a/tools/test/h5ls/testh5ls.sh.in +++ b/tools/test/h5ls/testh5ls.sh.in @@ -164,7 +164,7 @@ $SRC_H5LS_TESTFILES/tdset_idx.ls # RUNSERIAL is used. Check if it can return exit code from executalbe correctly. if [ -n "$RUNSERIAL_NOEXITCODE" ]; then - echo "***Warning*** Serial Exit Code is not passed back to shell corretly." + echo "***Warning*** Serial Exit Code is not passed back to shell correctly." echo "***Warning*** Exit code checking is skipped." h5haveexitcode=no fi @@ -230,7 +230,7 @@ TESTING() { # the `nerrors' global variable and (if $verbose is set) display up to $NLINS # lines of the actual output from the tool test. The actual output is not # removed if $HDF5_NOCLEANUP has a non-zero value. -# Arguemnts: +# Arguments: # $1 -- actual output filename to use # $2 and on -- argument for the h5ls tool TOOLTEST() { diff --git a/tools/test/h5ls/testh5lsvds.sh.in b/tools/test/h5ls/testh5lsvds.sh.in index 4814f85..ab2d137 100644 --- a/tools/test/h5ls/testh5lsvds.sh.in +++ b/tools/test/h5ls/testh5lsvds.sh.in @@ -103,7 +103,7 @@ $SRC_H5LS_TESTFILES/vds/tvds-5.ls # RUNSERIAL is used. Check if it can return exit code from executalbe correctly. if [ -n "$RUNSERIAL_NOEXITCODE" ]; then - echo "***Warning*** Serial Exit Code is not passed back to shell corretly." + echo "***Warning*** Serial Exit Code is not passed back to shell correctly." echo "***Warning*** Exit code checking is skipped." h5haveexitcode=no fi @@ -169,7 +169,7 @@ TESTING() { # the `nerrors' global variable and (if $verbose is set) display up to $NLINS # lines of the actual output from the tool test. The actual output is not # removed if $HDF5_NOCLEANUP has a non-zero value. -# Arguemnts: +# Arguments: # $1 -- actual output filename to use # $2 and on -- argument for the h5ls tool TOOLTEST() { diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index 397c3ac..09648ff 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -1541,7 +1541,7 @@ ADD_H5_TEST (bug1814 "TEST" ${FILE_REF}) # test attribute with various references (bug1797 / HDFFV-5932) -# the references in attribute of compund or vlen datatype +# the references in attribute of compound or vlen datatype ADD_H5_TEST (HDFFV-5932 "TEST" ${FILE_ATTR_REF}) # Add test for memory leak in attribute. This test is verified by CTEST. diff --git a/tools/test/h5repack/CMakeVFDTests.cmake b/tools/test/h5repack/CMakeVFDTests.cmake index e26941b..f8ac10b 100644 --- a/tools/test/h5repack/CMakeVFDTests.cmake +++ b/tools/test/h5repack/CMakeVFDTests.cmake @@ -15,20 +15,7 @@ ### T E S T I N G ### ############################################################################## ############################################################################## - -set (VFD_LIST - sec2 - stdio - core - core_paged - split - multi - family -) - -if (H5_HAVE_DIRECT) - set (VFD_LIST ${VFD_LIST} direct) -endif () +H5_CREATE_VFD_DIR() ############################################################################## ############################################################################## @@ -38,7 +25,6 @@ endif () macro (ADD_VFD_TEST vfdname resultcode) if (NOT HDF5_ENABLE_USING_MEMCHECKER) - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/${vfdname}") add_test ( NAME H5REPACK-${vfdname}-h5repacktest-clear-objects COMMAND ${CMAKE_COMMAND} -E remove diff --git a/tools/test/h5repack/dynlib_rpk.c b/tools/test/h5repack/dynlib_rpk.c index b228344..bef032a 100644 --- a/tools/test/h5repack/dynlib_rpk.c +++ b/tools/test/h5repack/dynlib_rpk.c @@ -83,7 +83,7 @@ H5Z_filter_dynlib1(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_ add_on = (int)cd_values[0]; if (flags & H5Z_FLAG_REVERSE) { /*read*/ - /* Substract the "add on" value to all the data values */ + /* Subtract the "add on" value to all the data values */ while (buf_left > 0) { *int_ptr++ -= add_on; buf_left -= sizeof(int); diff --git a/tools/test/h5repack/dynlib_vrpk.c b/tools/test/h5repack/dynlib_vrpk.c index 5eac4f2..a45de65 100644 --- a/tools/test/h5repack/dynlib_vrpk.c +++ b/tools/test/h5repack/dynlib_vrpk.c @@ -91,7 +91,7 @@ H5Z_filter_dynlib4(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_ add_on = (int)cd_values[0]; if (flags & H5Z_FLAG_REVERSE) { /*read*/ - /* Substract the "add on" value to all the data values */ + /* Subtract the "add on" value to all the data values */ while (buf_left > 0) { *int_ptr++ -= add_on; buf_left -= sizeof(int); diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index 1e54670..a241ea8 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -525,7 +525,7 @@ VERIFY_LAYOUT_ALL() #--------------------------------- # check the layout from a dataset - # check if the other layouts still exsit + # check if the other layouts still exist VERIFY "layouts" ( cd $TESTDIR @@ -1713,10 +1713,10 @@ TOOLTEST1 family tfamily%05d.h5 TOOLTEST bug1814 h5repack_refs.h5 # test attribute with various references (bug1797 / HDFFV-5932) -# the references in attribute of compund or vlen datatype +# the references in attribute of compound or vlen datatype TOOLTEST HDFFV-5932 h5repack_attr_refs.h5 -# Add test for memory leak in attirbute. This test is verified by CTEST. +# Add test for memory leak in attribute. This test is verified by CTEST. # 1. leak from vlen string # 2. leak from compound type without reference member # (HDFFV-7840, ) diff --git a/tools/test/h5repack/h5repackgentest.c b/tools/test/h5repack/h5repackgentest.c index 576b1c7..c025891 100644 --- a/tools/test/h5repack/h5repackgentest.c +++ b/tools/test/h5repack/h5repackgentest.c @@ -27,9 +27,9 @@ * There is no restriction on the name, number, or structure of datasets and * groups in HDF5 file. * - * The inluded datatypes should be more than adequate to verify the correctness + * The included datatypes should be more than adequate to verify the correctness * of the behavior -- if one type can be consolidated from external storage, - * then thay all can. + * then they all can. */ #include "hdf5.h" @@ -307,7 +307,7 @@ generate_f32le(hbool_t external) /* Generate values */ for (i = 0, k = 0, n = 0; (hsize_t)i < dims[0]; i++) { for (j = 0; (hsize_t)j < dims[1]; j++, k++, n++) { - wdata[k] = n * 801.1f * ((k % 5 == 1) ? (-1) : (1)); + wdata[k] = n * 801.1F * ((k % 5 == 1) ? (-1) : (1)); } } diff --git a/tools/test/h5repack/h5repacktst.c b/tools/test/h5repack/h5repacktst.c index b62fa53..e7cc21a 100644 --- a/tools/test/h5repack/h5repacktst.c +++ b/tools/test/h5repack/h5repacktst.c @@ -624,7 +624,7 @@ main(void) SKIPPED(); #endif - TESTING(" addding shuffle filter"); + TESTING(" adding shuffle filter"); /*------------------------------------------------------------------------- * test an individual object option @@ -653,7 +653,7 @@ main(void) *------------------------------------------------------------------------- */ - TESTING(" addding shuffle filter to all"); + TESTING(" adding shuffle filter to all"); if (h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; @@ -1592,17 +1592,17 @@ main(void) #endif /*------------------------------------------------------------------------- - * test file with aligment + * test file with alignment *------------------------------------------------------------------------- */ - TESTING(" file with aligment"); + TESTING(" file with alignment"); #ifdef H5_HAVE_FILTER_DEFLATE if (h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; - /* add the options for aligment */ + /* add the options for alignment */ pack_options.alignment = 1; pack_options.threshold = 1; @@ -1613,7 +1613,7 @@ main(void) if (h5repack_verify(FNAME8, FNAME8OUT, &pack_options) <= 0) GOERROR; - /* verify aligment */ + /* verify alignment */ { hsize_t threshold; hsize_t alignment; @@ -2432,14 +2432,15 @@ make_szip(hid_t loc_id) unsigned szip_pixels_per_block = 8; hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; int szip_can_encode = 0; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2463,7 +2464,7 @@ make_szip(hid_t loc_id) /* set szip data */ if (H5Pset_szip(dcpl, szip_options_mask, szip_pixels_per_block) < 0) goto error; - if (make_dset(loc_id, "dset_szip", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_szip", sid, dcpl, buf) < 0) goto error; } else @@ -2506,15 +2507,16 @@ make_deflate(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; hobj_ref_t bufref[1]; /* reference */ hsize_t dims1r[1] = {1}; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2534,7 +2536,7 @@ make_deflate(hid_t loc_id) /* set deflate data */ if (H5Pset_deflate(dcpl, 9) < 0) goto error; - if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf) < 0) goto error; /* create a reference to the dataset, test second seeep of file for references */ @@ -2585,13 +2587,14 @@ make_shuffle(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2611,7 +2614,7 @@ make_shuffle(hid_t loc_id) /* set the shuffle filter */ if (H5Pset_shuffle(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -2654,13 +2657,14 @@ make_fletcher32(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2683,7 +2687,7 @@ make_fletcher32(hid_t loc_id) /* set the checksum filter */ if (H5Pset_fletcher32(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -2726,15 +2730,17 @@ make_nbit(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hid_t dtid = H5I_INVALID_HID; hid_t dsid = H5I_INVALID_HID; + hid_t dxpl = H5P_DEFAULT; hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2746,6 +2752,16 @@ make_nbit(hid_t loc_id) if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) goto error; +#ifdef H5_HAVE_PARALLEL + /* Set up collective writes for parallel driver */ + if (h5_using_parallel_driver(NULL)) { + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + if (H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE) < 0) + goto error; + } +#endif + dtid = H5Tcopy(H5T_NATIVE_INT); if (H5Tset_precision(dtid, (H5Tget_precision(dtid) - 1)) < 0) goto error; @@ -2757,7 +2773,7 @@ make_nbit(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_nbit", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Dclose(dsid); @@ -2765,7 +2781,7 @@ make_nbit(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_int31", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Dclose(dsid); @@ -2773,6 +2789,8 @@ make_nbit(hid_t loc_id) * close *------------------------------------------------------------------------- */ + if (dxpl != H5P_DEFAULT && H5Pclose(dxpl) < 0) + goto error; if (H5Sclose(sid) < 0) goto error; if (H5Pclose(dcpl) < 0) @@ -2788,6 +2806,7 @@ error: H5E_BEGIN_TRY { H5Tclose(dtid); + H5Pclose(dxpl); H5Pclose(dcpl); H5Sclose(sid); H5Dclose(dsid); @@ -2813,15 +2832,17 @@ make_scaleoffset(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hid_t dtid = H5I_INVALID_HID; hid_t dsid = H5I_INVALID_HID; + hid_t dxpl = H5P_DEFAULT; hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2833,6 +2854,16 @@ make_scaleoffset(hid_t loc_id) if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) goto error; +#ifdef H5_HAVE_PARALLEL + /* Set up collective writes for parallel driver */ + if (h5_using_parallel_driver(NULL)) { + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + if (H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE) < 0) + goto error; + } +#endif + dtid = H5Tcopy(H5T_NATIVE_INT); /* remove the filters from the dcpl */ @@ -2842,20 +2873,22 @@ make_scaleoffset(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_scaleoffset", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Dclose(dsid); if ((dsid = H5Dcreate2(loc_id, "dset_none", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Tclose(dtid); H5Dclose(dsid); /*------------------------------------------------------------------------- - * close space and dcpl + * close space, dxpl and dcpl *------------------------------------------------------------------------- */ + if (dxpl != H5P_DEFAULT && H5Pclose(dxpl) < 0) + goto error; if (H5Sclose(sid) < 0) goto error; if (H5Pclose(dcpl) < 0) @@ -2868,6 +2901,7 @@ make_scaleoffset(hid_t loc_id) error: H5E_BEGIN_TRY { + H5Pclose(dxpl); H5Dclose(dsid); H5Tclose(dtid); H5Pclose(dcpl); @@ -2894,22 +2928,24 @@ make_all_filters(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hid_t dtid = H5I_INVALID_HID; hid_t dsid = H5I_INVALID_HID; + hid_t dxpl = H5P_DEFAULT; #if defined(H5_HAVE_FILTER_SZIP) unsigned szip_options_mask = H5_SZIP_ALLOW_K13_OPTION_MASK | H5_SZIP_NN_OPTION_MASK; unsigned szip_pixels_per_block = 8; #endif /* H5_HAVE_FILTER_SZIP */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; #if defined(H5_HAVE_FILTER_SZIP) int szip_can_encode = 0; #endif /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2921,6 +2957,16 @@ make_all_filters(hid_t loc_id) if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) goto error; +#ifdef H5_HAVE_PARALLEL + /* Set up collective writes for parallel driver */ + if (h5_using_parallel_driver(NULL)) { + if ((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto error; + if (H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE) < 0) + goto error; + } +#endif + /* set the shuffle filter */ if (H5Pset_shuffle(dcpl) < 0) goto error; @@ -2949,7 +2995,7 @@ make_all_filters(hid_t loc_id) goto error; #endif - if (make_dset(loc_id, "dset_all", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_all", sid, dcpl, buf) < 0) goto error; /* remove the filters from the dcpl */ @@ -2958,7 +3004,7 @@ make_all_filters(hid_t loc_id) /* set the checksum filter */ if (H5Pset_fletcher32(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf) < 0) goto error; /* Make sure encoding is enabled */ @@ -2970,7 +3016,7 @@ make_all_filters(hid_t loc_id) /* set szip data */ if (H5Pset_szip(dcpl, szip_options_mask, szip_pixels_per_block) < 0) goto error; - if (make_dset(loc_id, "dset_szip", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_szip", sid, dcpl, buf) < 0) goto error; } else { @@ -2984,7 +3030,7 @@ make_all_filters(hid_t loc_id) /* set the shuffle filter */ if (H5Pset_shuffle(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf) < 0) goto error; #if defined(H5_HAVE_FILTER_DEFLATE) @@ -2994,7 +3040,7 @@ make_all_filters(hid_t loc_id) /* set deflate data */ if (H5Pset_deflate(dcpl, 1) < 0) goto error; - if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf) < 0) goto error; #endif @@ -3010,7 +3056,7 @@ make_all_filters(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_nbit", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; /* close */ @@ -3021,6 +3067,8 @@ make_all_filters(hid_t loc_id) if (H5Sclose(sid) < 0) goto error; + if (dxpl != H5P_DEFAULT && H5Pclose(dxpl) < 0) + goto error; if (H5Pclose(dcpl) < 0) goto error; @@ -3033,6 +3081,7 @@ error: { H5Tclose(dtid); H5Dclose(dsid); + H5Pclose(dxpl); H5Pclose(dcpl); H5Sclose(sid); } @@ -3152,15 +3201,16 @@ make_layout(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; int i; char name[16]; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /*------------------------------------------------------------------------- * make several dataset with no filters @@ -3168,7 +3218,7 @@ make_layout(hid_t loc_id) */ for (i = 0; i < 4; i++) { HDsprintf(name, "dset%d", i + 1); - if (write_dset(loc_id, RANK, dims, name, H5T_NATIVE_INT, buf[0]) < 0) + if (write_dset(loc_id, RANK, dims, name, H5T_NATIVE_INT, buf) < 0) goto error; } @@ -3189,7 +3239,7 @@ make_layout(hid_t loc_id) */ if (H5Pset_layout(dcpl, H5D_COMPACT) < 0) goto error; - if (make_dset(loc_id, "dset_compact", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_compact", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -3198,7 +3248,7 @@ make_layout(hid_t loc_id) */ if (H5Pset_layout(dcpl, H5D_CONTIGUOUS) < 0) goto error; - if (make_dset(loc_id, "dset_contiguous", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_contiguous", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -3207,7 +3257,7 @@ make_layout(hid_t loc_id) */ if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) goto error; - if (make_dset(loc_id, "dset_chunk", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_chunk", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -3264,13 +3314,13 @@ make_layout2(hid_t loc_id) hsize_t s_dims[RANK] = {S_DIM1, S_DIM2}; /* Dataspace (< 1 k) */ hsize_t chunk_dims[RANK] = {S_DIM1 / 2, S_DIM2 / 2}; /* Dimension sizes for chunks */ - int **s_buf = NULL; /* Temporary buffer */ - /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(s_buf, int, S_DIM1, S_DIM2); + struct { + int arr[S_DIM1][S_DIM2]; + } *s_buf = malloc(sizeof(*s_buf)); if (NULL == s_buf) goto error; - H5TEST_FILL_2D_ARRAY(s_buf, int, S_DIM1, S_DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(s_buf, int); /* Create dataspaces */ if ((s_sid = H5Screate_simple(RANK, s_dims, NULL)) < 0) @@ -3281,7 +3331,7 @@ make_layout2(hid_t loc_id) goto error; if (H5Pset_layout(contig_dcpl, H5D_CONTIGUOUS) < 0) goto error; - if (make_dset(loc_id, CONTIG_S, s_sid, contig_dcpl, s_buf[0]) < 0) + if (make_dset(loc_id, CONTIG_S, s_sid, contig_dcpl, s_buf) < 0) goto error; /* Create chunked datasets */ @@ -3289,7 +3339,7 @@ make_layout2(hid_t loc_id) goto error; if (H5Pset_chunk(chunked_dcpl, RANK, chunk_dims) < 0) goto error; - if (make_dset(loc_id, CHUNKED_S_FIX, s_sid, chunked_dcpl, s_buf[0]) < 0) + if (make_dset(loc_id, CHUNKED_S_FIX, s_sid, chunked_dcpl, s_buf) < 0) goto error; ret_value = 0; @@ -3343,19 +3393,21 @@ make_layout3(hid_t loc_id) hsize_t chunk_dims1[RANK] = {DIM1_L3 * 2, 5}; hsize_t chunk_dims2[RANK] = {SDIM1_L3 + 2, SDIM2_L3 / 2}; hsize_t chunk_dims3[RANK] = {SDIM1_L3 - 2, SDIM2_L3 / 2}; - int ** buf1 = NULL; - int ** buf2 = NULL; - /* Create and fill arrays */ - H5TEST_ALLOCATE_2D_ARRAY(buf1, int, DIM1_L3, DIM2_L3); - if (NULL == buf1) - goto error; - H5TEST_FILL_2D_ARRAY(buf1, int, DIM1_L3, DIM2_L3); + /* Create arrays */ + struct { + int arr[DIM1_L3][DIM2_L3]; + } *buf1 = malloc(sizeof(*buf1)); + struct { + int arr[SDIM1_L3][SDIM2_L3]; + } *buf2 = malloc(sizeof(*buf2)); - H5TEST_ALLOCATE_2D_ARRAY(buf2, int, SDIM1_L3, SDIM2_L3); - if (NULL == buf2) + if (NULL == buf1 || NULL == buf2) goto error; - H5TEST_FILL_2D_ARRAY(buf2, int, SDIM1_L3, SDIM2_L3); + + /* Fill arrays */ + H5TEST_FILL_2D_HEAP_ARRAY(buf1, int); + H5TEST_FILL_2D_HEAP_ARRAY(buf2, int); /*------------------------------------------------------------------------- * make chunked dataset with @@ -3373,7 +3425,7 @@ make_layout3(hid_t loc_id) if (H5Pset_chunk(dcpl1, RANK, chunk_dims1) < 0) goto error; - if (make_dset(loc_id, "chunk_unlimit1", sid1, dcpl1, buf1[0]) < 0) + if (make_dset(loc_id, "chunk_unlimit1", sid1, dcpl1, buf1) < 0) goto error; /*------------------------------------------------------------------------- @@ -3394,7 +3446,7 @@ make_layout3(hid_t loc_id) if (H5Pset_chunk(dcpl2, RANK, chunk_dims2) < 0) goto error; - if (make_dset(loc_id, "chunk_unlimit2", sid2, dcpl2, buf2[0]) < 0) + if (make_dset(loc_id, "chunk_unlimit2", sid2, dcpl2, buf2) < 0) goto error; /*------------------------------------------------------------------------- @@ -3411,7 +3463,7 @@ make_layout3(hid_t loc_id) if (H5Pset_chunk(dcpl3, RANK, chunk_dims3) < 0) goto error; - if (make_dset(loc_id, "chunk_unlimit3", sid2, dcpl3, buf2[0]) < 0) + if (make_dset(loc_id, "chunk_unlimit3", sid2, dcpl3, buf2) < 0) goto error; /*------------------------------------------------------------------------- @@ -5802,11 +5854,25 @@ out: static int make_dset(hid_t loc_id, const char *name, hid_t sid, hid_t dcpl, void *buf) { - hid_t did = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t dxpl_id = H5P_DEFAULT; if ((did = H5Dcreate2(loc_id, name, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) return -1; - if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + +#ifdef H5_HAVE_PARALLEL + /* Set up collective writes for parallel driver */ + if (h5_using_parallel_driver(NULL)) { + if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto out; + if (H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) < 0) + goto out; + } +#endif + + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) + goto out; + if (dxpl_id != H5P_DEFAULT && H5Pclose(dxpl_id) < 0) goto out; if (H5Dclose(did) < 0) return -1; @@ -5835,17 +5901,30 @@ out: static int write_dset(hid_t loc_id, int rank, hsize_t *dims, const char *dset_name, hid_t tid, void *buf) { - hid_t did = H5I_INVALID_HID; - hid_t sid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + hid_t sid = H5I_INVALID_HID; + hid_t dxpl_id = H5P_DEFAULT; if ((sid = H5Screate_simple(rank, dims, NULL)) < 0) return -1; if ((did = H5Dcreate2(loc_id, dset_name, tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto out; if (buf) { - if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) +#ifdef H5_HAVE_PARALLEL + /* Set up collective writes for parallel driver */ + if (h5_using_parallel_driver(NULL)) { + if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) + goto out; + if (H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) < 0) + goto out; + } +#endif + + if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) goto out; } + if (dxpl_id != H5P_DEFAULT && H5Pclose(dxpl_id) < 0) + goto out; if (H5Dclose(did) < 0) goto out; if (H5Sclose(sid) < 0) @@ -5856,6 +5935,7 @@ write_dset(hid_t loc_id, int rank, hsize_t *dims, const char *dset_name, hid_t t out: H5E_BEGIN_TRY { + H5Pclose(dxpl_id); H5Dclose(did); H5Sclose(sid); } @@ -6244,7 +6324,7 @@ gen_refered_objs(hid_t loc_id) goto out; } - /* create normal dataset which is refered */ + /* create normal dataset which is referred */ did2 = H5Dcreate2(loc_id, NAME_OBJ_DS2, H5T_STD_I8LE, sid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (did2 < 0) { HDfprintf(stderr, "Error: %s %d> H5Dcreate2 failed.\n", __func__, __LINE__); @@ -6284,7 +6364,7 @@ out: * Generate object references to objects (dataset,group and named datatype) * * Note: - * copied from h5copygentest.c and upate to create named datatype + * copied from h5copygentest.c and update to create named datatype * * Programmer: Jonathan Kim (March 18, 2010) *------------------------------------------------------------------------*/ diff --git a/tools/test/h5repack/testfiles/README b/tools/test/h5repack/testfiles/README index 4096dee..4cb66f4 100644 --- a/tools/test/h5repack/testfiles/README +++ b/tools/test/h5repack/testfiles/README @@ -1,5 +1,5 @@ h5repack_nested_8bit_enum_deflated.h5: h5repack_nested_8bit_enum.h5: - enuberated 8bit type nested in compount type. Original file provided + enuberated 8bit type nested in compound type. Original file provided by a user (HDFFV-8667) as a test file. Used h5copy to extract only the Compound type dataset. The non-deflated version is produced by h5repack. diff --git a/tools/test/h5repack/testfiles/h5repack-help.txt b/tools/test/h5repack/testfiles/h5repack-help.txt index cb10d22..c1caf52 100644 --- a/tools/test/h5repack/testfiles/h5repack-help.txt +++ b/tools/test/h5repack/testfiles/h5repack-help.txt @@ -3,7 +3,8 @@ usage: h5repack [OPTIONS] file1 file2 file2 Output HDF5 File OPTIONS -h, --help Print a usage message and exit - -v, --verbose Verbose mode, print object information + -v N, --verbose=N Verbose mode, print object information. + N - is an integer greater than 1, 2 displays read/write timing -V, --version Print version number and exit -n, --native Use a native HDF5 type when repacking --enable-error-stack Prints messages from the HDF5 error stack as they @@ -20,6 +21,18 @@ usage: h5repack [OPTIONS] file1 file2 HDF5 file specified --dst-vol-info VOL-specific info to pass to the VOL connector used for opening the output HDF5 file specified + --src-vfd-value Value (ID) of the VFL driver to use for opening the + input HDF5 file specified + --src-vfd-name Name of the VFL driver to use for opening the input + HDF5 file specified + --src-vfd-info VFD-specific info to pass to the VFL driver used for + opening the input HDF5 file specified + --dst-vfd-value Value (ID) of the VFL driver to use for opening the + output HDF5 file specified + --dst-vfd-name Name of the VFL driver to use for opening the output + HDF5 file specified + --dst-vfd-info VFD-specific info to pass to the VFL driver used for + opening the output HDF5 file specified -L, --latest Use latest version of file format This option will take precedence over the options --low and --high diff --git a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl index 15ae813..0992a0c 100644 --- a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl +++ b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl @@ -11,7 +11,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -33,7 +33,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -55,7 +55,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -77,7 +77,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -99,7 +99,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -121,7 +121,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -143,7 +143,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { diff --git a/tools/test/h5stat/testh5stat.sh.in b/tools/test/h5stat/testh5stat.sh.in index 647f06a..1b4e504 100644 --- a/tools/test/h5stat/testh5stat.sh.in +++ b/tools/test/h5stat/testh5stat.sh.in @@ -13,7 +13,7 @@ # # Tests for the h5stat tool # -# Modifcations: +# Modifications: # Vailin Choi; July 2013 # Add tests for -l, -m, -a options # diff --git a/tools/test/misc/h5clear_gentest.c b/tools/test/misc/h5clear_gentest.c index 53fbc42..97feabd 100644 --- a/tools/test/misc/h5clear_gentest.c +++ b/tools/test/misc/h5clear_gentest.c @@ -16,7 +16,7 @@ /* The HDF5 test files */ const char *FILENAME[] = { "h5clear_sec2_v3.h5", /* 0 -- sec2 file with superblock version 3 */ - "h5clear_log_v3.h5", /* 1 -- log file with superblock veresion 3 */ + "h5clear_log_v3.h5", /* 1 -- log file with superblock version 3 */ "h5clear_sec2_v0.h5", /* 2 -- sec2 file with superblock version 0 */ "h5clear_sec2_v2.h5" /* 3 -- sec2 file with superblock version 2 */ }; @@ -56,24 +56,26 @@ const char *FILENAME_ENHANCE[] = { static int gen_cache_image_file(const char *fname) { - hid_t fid = H5I_INVALID_HID; /* File ID */ - hid_t did = -1, sid = H5I_INVALID_HID; /* Dataset ID, dataspace ID */ - hid_t fapl = H5I_INVALID_HID; /* File access property list */ - hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ - hsize_t dims[2]; /* Dimension sizes */ - hsize_t chunks[2]; /* Chunked dimension sizes */ - int i, j; /* Local index variables */ - int ** buf = NULL; /* Buffer for data to write */ - H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t did = -1, sid = H5I_INVALID_HID; /* Dataset ID, dataspace ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ + hsize_t dims[2]; /* Dimension sizes */ + hsize_t chunks[2]; /* Chunked dimension sizes */ + int i, j; /* Local index variables */ + struct { + int arr[50][100]; + } * buf; /* Buffer for data to write */ + H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */ {H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, TRUE, FALSE, H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, 50, 100); + buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; for (i = 0; i < 50; i++) for (j = 0; j < 100; j++) - buf[i][j] = i * j; + buf->arr[i][j] = i * j; /* Create a copy of file access property list */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -108,7 +110,7 @@ gen_cache_image_file(const char *fname) goto error; /* Write to the dataset */ - if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) goto error; /* Closing */ @@ -426,7 +428,7 @@ main(void) if ((my_fapl = H5Pcopy(fapl2)) < 0) goto error; /* Create the file */ - HDsprintf(fname, "%s%s", new_format ? "latest_" : "", FILENAME[0]); + HDsnprintf(fname, sizeof(fname), "%s%s", new_format ? "latest_" : "", FILENAME[0]); if ((fid = H5Fcreate(fname, H5F_ACC_TRUNC | (new_format ? 0 : H5F_ACC_SWMR_WRITE), H5P_DEFAULT, my_fapl)) < 0) goto error; @@ -451,7 +453,7 @@ main(void) goto error; /* Create the file */ - HDsprintf(fname, "%s%s", new_format ? "latest_" : "", FILENAME[1]); + HDsnprintf(fname, sizeof(fname), "%s%s", new_format ? "latest_" : "", FILENAME[1]); if ((fid = H5Fcreate(fname, H5F_ACC_TRUNC | (new_format ? 0 : H5F_ACC_SWMR_WRITE), H5P_DEFAULT, my_fapl)) < 0) goto error; diff --git a/tools/test/misc/h5perf_gentest.c b/tools/test/misc/h5perf_gentest.c index 332e49c..248fd92 100644 --- a/tools/test/misc/h5perf_gentest.c +++ b/tools/test/misc/h5perf_gentest.c @@ -255,9 +255,9 @@ create_perf_test_file(const char *fname, int ngrps, int ndsets, int nattrs, hsiz buf_vlen_s = (char **)calloc(dim0, sizeof(char *)); buf_str = malloc(dim0 * sizeof(*buf_str)); - /* allocate array of doulbe pointers */ + /* allocate array of double pointers */ buf_double2d = (double **)calloc(dims2d[0], sizeof(double *)); - /* allocate a contigous chunk of memory for the data */ + /* allocate a contiguous chunk of memory for the data */ buf_double2d[0] = (double *)calloc(dims2d[0] * dims2d[1], sizeof(double)); /* assign memory city to pointer array */ for (i = 1; i < dims2d[0]; i++) @@ -396,7 +396,7 @@ create_perf_test_file(const char *fname, int ngrps, int ndsets, int nattrs, hsiz H5Dclose(did); /* 6 add compound data */ - HDsprintf(name, "%05d compund data", j); + HDsprintf(name, "%05d compound data", j); did = H5Dcreate(gid1, name, cmp_tid, sid_1d, H5P_DEFAULT, dcpl, H5P_DEFAULT); H5Dwrite(did, cmp_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_comp); if (!j) diff --git a/tools/test/misc/testfiles/h5mkgrp_help.txt b/tools/test/misc/testfiles/h5mkgrp_help.txt index 9525230..5d81b34 100644 --- a/tools/test/misc/testfiles/h5mkgrp_help.txt +++ b/tools/test/misc/testfiles/h5mkgrp_help.txt @@ -11,4 +11,10 @@ usage: h5mkgrp [OPTIONS] FILE GROUP... HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c index b515961..832447e 100644 --- a/tools/test/perform/chunk_cache.c +++ b/tools/test/perform/chunk_cache.c @@ -39,7 +39,7 @@ #define RDCC_NSLOTS 5 #define RDCC_NBYTES (1024 * 1024 * 10) -#define RDCC_W0 0.75F +#define RDCC_W0 0.75 #define FILTER_COUNTER 306 static size_t nbytes_global; @@ -81,7 +81,7 @@ counter(unsigned H5_ATTR_UNUSED flags, size_t H5_ATTR_UNUSED cd_nelmts, static void cleanup(void) { - if (!getenv("HDF5_NOCLEANUP")) { + if (!getenv(HDF5_NOCLEANUP)) { remove(FILENAME); } } @@ -98,7 +98,9 @@ create_dset1(hid_t file) hid_t dcpl = H5I_INVALID_HID; hsize_t dims[RANK] = {DSET1_DIM1, DSET1_DIM2}; hsize_t chunk_dims[RANK] = {CHUNK1_DIM1, CHUNK1_DIM2}; - int ** data = NULL; /* data for writing */ + struct { + int arr[DSET1_DIM1][DSET1_DIM2]; + } *data = malloc(sizeof(*data)); /* Create the data space. */ if ((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -122,9 +124,8 @@ create_dset1(hid_t file) 0) goto error; - /* Create & fill array */ - H5TEST_ALLOCATE_2D_ARRAY(data, int, DSET1_DIM1, DSET1_DIM2); - H5TEST_FILL_2D_ARRAY(data, int, DSET1_DIM1, DSET1_DIM2); + /* Fill array */ + H5TEST_FILL_2D_HEAP_ARRAY(data, int); /* Write data to dataset */ if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) @@ -162,7 +163,9 @@ create_dset2(hid_t file) hid_t dcpl = H5I_INVALID_HID; hsize_t dims[RANK] = {DSET2_DIM1, DSET2_DIM2}; hsize_t chunk_dims[RANK] = {CHUNK2_DIM1, CHUNK2_DIM2}; - int ** data = NULL; /* data for writing */ + struct { + int arr[DSET2_DIM1][DSET2_DIM2]; + } *data = malloc(sizeof(*data)); /* Create the data space. */ if ((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -185,9 +188,8 @@ create_dset2(hid_t file) 0) goto error; - /* Create & fill array */ - H5TEST_ALLOCATE_2D_ARRAY(data, int, DSET2_DIM1, DSET2_DIM2); - H5TEST_FILL_2D_ARRAY(data, int, DSET2_DIM1, DSET2_DIM2); + /* Fill array */ + H5TEST_FILL_2D_HEAP_ARRAY(data, int); /* Write data to dataset */ if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) diff --git a/tools/test/perform/direct_write_perf.c b/tools/test/perform/direct_write_perf.c index 371c4f9..1d7756b 100644 --- a/tools/test/perform/direct_write_perf.c +++ b/tools/test/perform/direct_write_perf.c @@ -625,7 +625,7 @@ test_unix_write(void) TEST_ERROR; } else if (op_size == 0) { - printf(" unable to write sufficent data to file because %s \n", strerror(errno)); + printf(" unable to write sufficient data to file because %s \n", strerror(errno)); TEST_ERROR; } } @@ -655,7 +655,7 @@ main(void) hid_t fapl = H5P_DEFAULT; int i; - sprintf(filename, "%s.h5", FILENAME[0]); + snprintf(filename, sizeof(filename), "%s.h5", FILENAME[0]); create_file(fapl); test_direct_write_uncompressed_data(fapl); diff --git a/tools/test/perform/overhead.c b/tools/test/perform/overhead.c index 60ec8d8..c974928 100644 --- a/tools/test/perform/overhead.c +++ b/tools/test/perform/overhead.c @@ -119,7 +119,7 @@ usage(const char *prog) static void cleanup(void) { - if (!getenv("HDF5_NOCLEANUP")) { + if (!getenv(HDF5_NOCLEANUP)) { remove(FILE_NAME_1); } } @@ -187,7 +187,7 @@ test(fill_t fill_style, const double splits[], hbool_t verbose, hbool_t use_rdcc if (!use_rdcc) { if (H5Pget_cache(fapl, &mdc_nelmts, NULL, NULL, NULL) < 0) goto error; - if (H5Pset_cache(fapl, mdc_nelmts, 0, 0, 0.0F) < 0) + if (H5Pset_cache(fapl, mdc_nelmts, 0, 0, 0.0) < 0) goto error; } if ((file = H5Fcreate(FILE_NAME_1, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) diff --git a/tools/test/perform/perf_meta.c b/tools/test/perform/perf_meta.c index d8ed9ec..ad76711 100644 --- a/tools/test/perform/perf_meta.c +++ b/tools/test/perform/perf_meta.c @@ -304,7 +304,7 @@ create_dsets(hid_t file) * Create a dataset using the default dataset creation properties. */ for (i = 0; i < NUM_DSETS; i++) { - HDsprintf(dset_name, "dataset %d", i); + HDsnprintf(dset_name, sizeof(dset_name), "dataset %d", i); if ((dataset = H5Dcreate2(file, dset_name, H5T_NATIVE_DOUBLE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; @@ -366,14 +366,14 @@ create_attrs_1(void) * Create all(user specifies the number) attributes for each dataset */ for (i = 0; i < NUM_DSETS; i++) { - HDsprintf(dset_name, "dataset %d", i); + HDsnprintf(dset_name, sizeof(dset_name), "dataset %d", i); open_t.start = retrieve_time(); if ((dataset = H5Dopen2(file, dset_name, H5P_DEFAULT)) < 0) goto error; perf(&open_t, open_t.start, retrieve_time()); for (j = 0; j < NUM_ATTRS; j++) { - HDsprintf(attr_name, "all attrs for each dset %d", j); + HDsnprintf(attr_name, sizeof(attr_name), "all attrs for each dset %d", j); attr_t.start = retrieve_time(); if ((attr = H5Acreate2(dataset, attr_name, H5T_NATIVE_DOUBLE, small_space, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -468,7 +468,7 @@ create_attrs_2(void) * Create all(user specifies the number) attributes for each new dataset */ for (i = 0; i < NUM_DSETS; i++) { - HDsprintf(dset_name, "dataset %d", i); + HDsnprintf(dset_name, sizeof(dset_name), "dataset %d", i); create_t.start = retrieve_time(); if ((dataset = H5Dcreate2(file, dset_name, H5T_NATIVE_DOUBLE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -476,7 +476,7 @@ create_attrs_2(void) perf(&create_t, create_t.start, retrieve_time()); for (j = 0; j < NUM_ATTRS; j++) { - HDsprintf(attr_name, "all attrs for each dset %d", j); + HDsnprintf(attr_name, sizeof(attr_name), "all attrs for each dset %d", j); attr_t.start = retrieve_time(); if ((attr = H5Acreate2(dataset, attr_name, H5T_NATIVE_DOUBLE, small_space, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -579,14 +579,14 @@ create_attrs_3(void) for (i = 0; i < loop_num; i++) { for (j = 0; j < NUM_DSETS; j++) { - HDsprintf(dset_name, "dataset %d", j); + HDsnprintf(dset_name, sizeof(dset_name), "dataset %d", j); open_t.start = retrieve_time(); if ((dataset = H5Dopen2(file, dset_name, H5P_DEFAULT)) < 0) goto error; perf(&open_t, open_t.start, retrieve_time()); for (k = 0; k < BATCH_ATTRS; k++) { - HDsprintf(attr_name, "some attrs for each dset %d %d", i, k); + HDsnprintf(attr_name, sizeof(attr_name), "some attrs for each dset %d %d", i, k); attr_t.start = retrieve_time(); if ((attr = H5Acreate2(dataset, attr_name, H5T_NATIVE_DOUBLE, small_space, H5P_DEFAULT, H5P_DEFAULT)) < 0) diff --git a/tools/test/perform/pio_standalone.h b/tools/test/perform/pio_standalone.h index f1fb946..e64a765 100644 --- a/tools/test/perform/pio_standalone.h +++ b/tools/test/perform/pio_standalone.h @@ -461,7 +461,8 @@ struct h5_long_options { */ }; -extern int H5_get_option(int argc, const char **argv, const char *opt, const struct h5_long_options *l_opt); +extern int H5_get_option(int argc, const char *const *argv, const char *opt, + const struct h5_long_options *l_opt); extern int nCols; /*max number of columns for outputting */ diff --git a/tools/test/perform/sio_standalone.h b/tools/test/perform/sio_standalone.h index 99cca75..57180e5 100644 --- a/tools/test/perform/sio_standalone.h +++ b/tools/test/perform/sio_standalone.h @@ -476,7 +476,8 @@ struct h5_long_options { */ }; -extern int H5_get_option(int argc, const char **argv, const char *opt, const struct h5_long_options *l_opt); +extern int H5_get_option(int argc, const char *const *argv, const char *opt, + const struct h5_long_options *l_opt); extern int nCols; /*max number of columns for outputting */ diff --git a/tools/test/perform/zip_perf.c b/tools/test/perform/zip_perf.c index d8cd3a1..74b1685 100644 --- a/tools/test/perform/zip_perf.c +++ b/tools/test/perform/zip_perf.c @@ -103,7 +103,7 @@ error(const char *fmt, ...) static void cleanup(void) { - if (!HDgetenv("HDF5_NOCLEANUP")) + if (!HDgetenv(HDF5_NOCLEANUP)) HDunlink(filename); HDfree(filename); } @@ -267,7 +267,7 @@ usage(void) HDfprintf(stdout, " data you want [default: 0]"); HDfprintf(stdout, " -s S, --file-size=S Maximum size of uncompressed file [default: 64M]\n"); HDfprintf(stdout, " -B S, --max-buffer_size=S Maximum size of buffer [default: 1M]\n"); - HDfprintf(stdout, " -b S, --min-buffer_size=S Minumum size of buffer [default: 128K]\n"); + HDfprintf(stdout, " -b S, --min-buffer_size=S Minimum size of buffer [default: 128K]\n"); HDfprintf(stdout, " -p D, --prefix=D The directory prefix to place the file\n"); HDfprintf(stdout, " -r, --random-test Use random data to write to the file\n"); HDfprintf(stdout, " [default: no]\n"); @@ -396,7 +396,7 @@ do_write_test(unsigned long file_size, unsigned long min_buf_size, unsigned long error("out of memory"); } - compression_time = 0.0F; + compression_time = 0.0; if (random_test) fill_with_random_data(src, src_len); @@ -489,7 +489,7 @@ do_write_test(unsigned long file_size, unsigned long min_buf_size, unsigned long * Modifications: */ int -main(int argc, const char *argv[]) +main(int argc, char *argv[]) { unsigned long min_buf_size = 128 * ONE_KB, max_buf_size = ONE_MB; unsigned long file_size = 64 * ONE_MB; @@ -500,7 +500,7 @@ main(int argc, const char *argv[]) /* Initialize h5tools lib */ h5tools_init(); - while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) > 0) { + while ((opt = H5_get_option(argc, (const char *const *)argv, s_opts, l_opts)) > 0) { switch ((char)opt) { case '0': case '1': @@ -551,7 +551,7 @@ main(int argc, const char *argv[]) } if (min_buf_size > max_buf_size) - error("minmum buffer size (%d) exceeds maximum buffer size (%d)", min_buf_size, max_buf_size); + error("minimum buffer size (%d) exceeds maximum buffer size (%d)", min_buf_size, max_buf_size); HDfprintf(stdout, "Filesize: %ld\n", file_size); diff --git a/tools/testfiles/h5dump-help.txt b/tools/testfiles/h5dump-help.txt index 5b11223..53c666b 100644 --- a/tools/testfiles/h5dump-help.txt +++ b/tools/testfiles/h5dump-help.txt @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/help-1.ls b/tools/testfiles/help-1.ls index 2167e00..11ecb75 100644 --- a/tools/testfiles/help-1.ls +++ b/tools/testfiles/help-1.ls @@ -54,6 +54,12 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/help-2.ls b/tools/testfiles/help-2.ls index 2167e00..11ecb75 100644 --- a/tools/testfiles/help-2.ls +++ b/tools/testfiles/help-2.ls @@ -54,6 +54,12 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/help-3.ls b/tools/testfiles/help-3.ls index 2167e00..11ecb75 100644 --- a/tools/testfiles/help-3.ls +++ b/tools/testfiles/help-3.ls @@ -54,6 +54,12 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl index 5b11223..53c666b 100644 --- a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl +++ b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsIncomplete.ddl b/tools/testfiles/pbits/tpbitsIncomplete.ddl index 5b11223..53c666b 100644 --- a/tools/testfiles/pbits/tpbitsIncomplete.ddl +++ b/tools/testfiles/pbits/tpbitsIncomplete.ddl @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl index 5b11223..53c666b 100644 --- a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl +++ b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsLengthPositive.ddl b/tools/testfiles/pbits/tpbitsLengthPositive.ddl index 5b11223..53c666b 100644 --- a/tools/testfiles/pbits/tpbitsLengthPositive.ddl +++ b/tools/testfiles/pbits/tpbitsLengthPositive.ddl @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl index 5b11223..53c666b 100644 --- a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl +++ b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl index 5b11223..53c666b 100644 --- a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl +++ b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl index 5b11223..53c666b 100644 --- a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl +++ b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl @@ -28,6 +28,12 @@ usage: h5dump [OPTIONS] files HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/textlinksrc-nodangle-1.ls b/tools/testfiles/textlinksrc-nodangle-1.ls index 2167e00..11ecb75 100644 --- a/tools/testfiles/textlinksrc-nodangle-1.ls +++ b/tools/testfiles/textlinksrc-nodangle-1.ls @@ -54,6 +54,12 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/tgroup-1.ls b/tools/testfiles/tgroup-1.ls index 2167e00..11ecb75 100644 --- a/tools/testfiles/tgroup-1.ls +++ b/tools/testfiles/tgroup-1.ls @@ -54,6 +54,12 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] HDF5 file specified --vol-info VOL-specific info to pass to the VOL connector used for opening the HDF5 file specified + --vfd-value Value (ID) of the VFL driver to use for opening the + HDF5 file specified + --vfd-name Name of the VFL driver to use for opening the + HDF5 file specified + --vfd-info VFD-specific info to pass to the VFL driver used for + opening the HDF5 file specified file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index d6a5afd..9d904bb 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -1,5 +1,20 @@ cmake_minimum_required (VERSION 3.10) project (HDF5_UTILS C) -add_subdirectory (mirror_vfd) -add_subdirectory (vfd_swmr) +if (BUILD_TESTING) + add_subdirectory (test) +endif () + +option (HDF5_BUILD_UTILS "Build HDF5 Utils" ON) +if (HDF5_BUILD_UTILS) + add_subdirectory (mirror_vfd) +endif () + +#-- Add the h5dwalk and test executables +if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL) + add_subdirectory(tools) +endif() + +if (HDF5_HAVE_AUX_PROCESS) + add_subdirectory (vfd_swmr) +endif() diff --git a/utils/Makefile.am b/utils/Makefile.am index fa877ef..425b234 100644 --- a/utils/Makefile.am +++ b/utils/Makefile.am @@ -20,7 +20,31 @@ include $(top_srcdir)/config/commence.am CONFIG=ordered +if BUILD_TESTS_CONDITIONAL + TESTUTIL_DIR =test +else + TESTUTIL_DIR= +endif + +if MIRROR_VFD_CONDITIONAL + MIRROR_VFD_DIR = mirror_vfd +else + MIRROR_VFD_DIR= +endif + +if BUILD_TOOLS_CONDITIONAL + TOOLS_DIR =tools +else + TOOLS_DIR= +endif + +if AUX_PROCESS_CONDITIONAL + VFD_SWMR_DIR=vfd_swmr +else + VFD_SWMR_DIR= +endif + # All subdirectories -SUBDIRS=mirror_vfd vfd_swmr +SUBDIRS=$(MIRROR_VFD_DIR) $(TESTUTIL_DIR) $(TOOLS_DIR) $(VFD_SWMR_DIR) include $(top_srcdir)/config/conclude.am diff --git a/utils/mirror_vfd/mirror_remote.c b/utils/mirror_vfd/mirror_remote.c index f61580d..8bb9544 100644 --- a/utils/mirror_vfd/mirror_remote.c +++ b/utils/mirror_vfd/mirror_remote.c @@ -147,7 +147,7 @@ mirror_log_bytes(struct mirror_log_info *info, unsigned int level, size_t n_byte * ---------------------------------------------------------------------------- */ loginfo_t * -mirror_log_init(char *path, char *prefix, unsigned int verbosity) +mirror_log_init(char *path, const char *prefix, unsigned int verbosity) { loginfo_t *info = NULL; diff --git a/utils/mirror_vfd/mirror_remote.h b/utils/mirror_vfd/mirror_remote.h index 6f37b0b..9415f7f 100644 --- a/utils/mirror_vfd/mirror_remote.h +++ b/utils/mirror_vfd/mirror_remote.h @@ -42,7 +42,7 @@ typedef struct mirror_log_info { void mirror_log(loginfo_t *info, unsigned int level, const char *format, ...); void mirror_log_bytes(loginfo_t *info, unsigned int level, size_t n_bytes, const unsigned char *buf); -loginfo_t *mirror_log_init(char *path, char *prefix, unsigned int verbosity); +loginfo_t *mirror_log_init(char *path, const char *prefix, unsigned int verbosity); int mirror_log_term(loginfo_t *loginfo); herr_t run_writer(int socketfd, H5FD_mirror_xmit_open_t *xmit_open); diff --git a/utils/mirror_vfd/mirror_server.c b/utils/mirror_vfd/mirror_server.c index 9de07c1..5381d95 100644 --- a/utils/mirror_vfd/mirror_server.c +++ b/utils/mirror_vfd/mirror_server.c @@ -80,7 +80,7 @@ * line. * * `log_prepend_type` (int) - * Flag that the logging messages should have the assocaited verbosity + * Flag that the logging messages should have the associated verbosity * level present in the line (e.g., "WARN", "ERROR", or "INFO"). * * `log_path` (char *) @@ -94,14 +94,14 @@ * --------------------------------------------------------------------------- */ struct op_args { - uint32_t magic; - int help; - int main_port; - int verbosity; - int log_prepend_serv; - int log_prepend_type; - char log_path[PATH_MAX + 1]; - char writer_log_path[PATH_MAX + 1]; + uint32_t magic; + int help; + int main_port; + unsigned int verbosity; + int log_prepend_serv; + int log_prepend_type; + char log_path[PATH_MAX + 1]; + char writer_log_path[PATH_MAX + 1]; }; /* --------------------------------------------------------------------------- @@ -224,7 +224,7 @@ parse_args(int argc, char **argv, struct op_args *args_out) } /* end if port */ else if (!HDstrncmp(argv[i], "--verbosity=", 12)) { mirror_log(NULL, V_INFO, "parsing 'verbosity' (%s)", argv[i] + 12); - args_out->verbosity = HDatoi(argv[i] + 12); + args_out->verbosity = (unsigned int)HDatoi(argv[i] + 12); } /* end if verbosity */ else if (!HDstrncmp(argv[i], "--logpath=", 10)) { mirror_log(NULL, V_INFO, "parsing 'logpath' (%s)", argv[i] + 10); @@ -456,7 +456,7 @@ error: * --------------------------------------------------------------------------- */ static void -wait_for_child(int sig) +wait_for_child(int H5_ATTR_UNUSED sig) { while (HDwaitpid(-1, NULL, WNOHANG) > 0) ; @@ -476,7 +476,7 @@ handle_requests(struct server_run *run) { int connfd = -1; /**/ char mybuf[H5FD_MIRROR_XMIT_OPEN_SIZE]; /**/ - int ret; /* general-purpose error-checking */ + ssize_t ret; /* general-purpose error-checking */ int pid; /* process ID of fork */ struct sigaction sa; int ret_value = 0; @@ -521,14 +521,13 @@ handle_requests(struct server_run *run) /* Read handshake from port connection. */ - ret = (int)HDread(connfd, &mybuf, H5FD_MIRROR_XMIT_OPEN_SIZE); - if (-1 == ret) { + if ((ret = HDread(connfd, &mybuf, H5FD_MIRROR_XMIT_OPEN_SIZE)) < 0) { mirror_log(run->loginfo, V_ERR, "read:%d", ret); goto error; } mirror_log(run->loginfo, V_INFO, "received %d bytes", ret); mirror_log(run->loginfo, V_ALL, "```"); - mirror_log_bytes(run->loginfo, V_ALL, ret, (const unsigned char *)mybuf); + mirror_log_bytes(run->loginfo, V_ALL, (size_t)ret, (const unsigned char *)mybuf); mirror_log(run->loginfo, V_ALL, "```"); /* Respond to handshake message. diff --git a/utils/mirror_vfd/mirror_server_stop.c b/utils/mirror_vfd/mirror_server_stop.c index bf306d9..024b33a 100644 --- a/utils/mirror_vfd/mirror_server_stop.c +++ b/utils/mirror_vfd/mirror_server_stop.c @@ -15,7 +15,7 @@ * Exists for cross-platform, optionally remote shutdown. */ -#include "H5private.h" /* System compatability call-wrapper macros */ +#include "H5private.h" /* System compatibility call-wrapper macros */ #ifdef H5_HAVE_MIRROR_VFD @@ -66,7 +66,7 @@ usage(void) "\n" "Options:\n" " -h | --help Print this usage message and exit.\n" - " --ip=ADDR IP Address of remote server (defaut %s)\n" + " --ip=ADDR IP Address of remote server (default %s)\n" " --port=PORT Handshake port of remote server (default %d)\n", MSHS_DEFAULT_IP, MSHS_DEFAULT_PORTNO); } /* end usage() */ diff --git a/utils/mirror_vfd/mirror_writer.c b/utils/mirror_vfd/mirror_writer.c index d3f12de..5726db5 100644 --- a/utils/mirror_vfd/mirror_writer.c +++ b/utils/mirror_vfd/mirror_writer.c @@ -788,8 +788,7 @@ do_write(struct mirror_session *session, const unsigned char *xmit_buf) */ sum_bytes_written = 0; do { - nbytes_in_packet = HDread(session->sockfd, buf, H5FD_MIRROR_DATA_BUFFER_MAX); - if (-1 == nbytes_in_packet) { + if ((nbytes_in_packet = HDread(session->sockfd, buf, H5FD_MIRROR_DATA_BUFFER_MAX)) < 0) { mirror_log(session->loginfo, V_ERR, "can't read into databuffer"); reply_error(session, "can't read data buffer"); return -1; @@ -798,7 +797,7 @@ do_write(struct mirror_session *session, const unsigned char *xmit_buf) mirror_log(session->loginfo, V_INFO, "received %zd bytes", nbytes_in_packet); if (HEXDUMP_WRITEDATA) { mirror_log(session->loginfo, V_ALL, "DATA:\n```"); - mirror_log_bytes(session->loginfo, V_ALL, nbytes_in_packet, (const unsigned char *)buf); + mirror_log_bytes(session->loginfo, V_ALL, (size_t)nbytes_in_packet, (const unsigned char *)buf); mirror_log(session->loginfo, V_ALL, "```"); } @@ -859,8 +858,7 @@ receive_communique(struct mirror_session *session, struct sock_comm *comm) mirror_log(session->loginfo, V_INFO, "ready to receive"); /* TODO */ - read_ret = HDread(session->sockfd, comm->raw, H5FD_MIRROR_XMIT_BUFFER_MAX); - if (-1 == read_ret) { + if ((read_ret = HDread(session->sockfd, comm->raw, H5FD_MIRROR_XMIT_BUFFER_MAX)) < 0) { mirror_log(session->loginfo, V_ERR, "read:%zd", read_ret); goto error; } diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt new file mode 100644 index 0000000..921fbd0 --- /dev/null +++ b/utils/test/CMakeLists.txt @@ -0,0 +1,35 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TEST C) + +################################################################################# +# Test program sources +################################################################################# + +macro (ADD_H5_EXE file) + add_executable (${file} ${HDF5_TEST_SOURCE_DIR}/${file}.c) + target_include_directories (${file} PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TEST_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(${file} PRIVATE "${HDF5_CMAKE_C_FLAGS}") + if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (${file} STATIC) + target_link_libraries (${file} PRIVATE ${HDF5_TEST_LIB_TARGET}) + else () + TARGET_C_PROPERTIES (${file} SHARED) + target_link_libraries (${file} PRIVATE ${HDF5_TEST_LIBSH_TARGET}) + endif () + set_target_properties (${file} PROPERTIES FOLDER test) +endmacro () + +############################################################################## +### S W I M M E R T E S T U T I L S ### +############################################################################## +set (H5_UTIL_TESTS) + +if (HDF5_TEST_SWMR) + set (H5_UTIL_TESTS ${H5_UTIL_TESTS} swmr_check_compat_vfd) +endif () + +if (H5_UTIL_TESTS) + foreach (h5_test ${H5_UTIL_TESTS}) + ADD_H5_EXE(${h5_test}) + endforeach () +endif () diff --git a/utils/test/Makefile.am b/utils/test/Makefile.am new file mode 100644 index 0000000..164562f --- /dev/null +++ b/utils/test/Makefile.am @@ -0,0 +1,34 @@ +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src and tools/lib directories +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -I$(top_srcdir)/utils/test + +# These are our main targets, the tools + +noinst_PROGRAMS=swmr_check_compat_vfd + +# Programs all depend on the hdf5 library, the tools library, and the HL +# library. +LDADD=$(LIBH5TEST) $(LIBHDF5) + +CHECK_CLEANFILES+=*.h5 + +include $(top_srcdir)/config/conclude.am diff --git a/utils/test/swmr_check_compat_vfd.c b/utils/test/swmr_check_compat_vfd.c new file mode 100644 index 0000000..720c747 --- /dev/null +++ b/utils/test/swmr_check_compat_vfd.c @@ -0,0 +1,53 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Purpose: This is a small program that checks if the HDF5_DRIVER + * environment variable is set to a value that supports SWMR. + * + * It is intended for use in shell scripts. + */ + +#include "h5test.h" + +/* This file needs to access the file driver testing code */ +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ +#define H5FD_TESTING +#include "H5FDpkg.h" /* File drivers */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Inspects the HDF5_DRIVER environment variable, which + * determines the VFD that the test harness will use with + * the majority of the tests. + * + * Return: VFD supports SWMR: EXIT_SUCCESS + * + * VFD does not support SWMR + * or failure: EXIT_FAILURE + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + char *driver = NULL; + + driver = HDgetenv(HDF5_DRIVER); + + if (H5FD__supports_swmr_test(driver)) + return EXIT_SUCCESS; + else + return EXIT_FAILURE; + +} /* end main() */ diff --git a/utils/tools/CMakeLists.txt b/utils/tools/CMakeLists.txt new file mode 100644 index 0000000..49562d7 --- /dev/null +++ b/utils/tools/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_UTILS_TOOLS C) + + +if (HDF5_BUILD_PARALLEL_TOOLS) + add_subdirectory (h5dwalk) +endif() + +#-- Add the tests +if (BUILD_TESTING) + add_subdirectory (test) +endif() diff --git a/utils/tools/Makefile.am b/utils/tools/Makefile.am new file mode 100644 index 0000000..0c89aff --- /dev/null +++ b/utils/tools/Makefile.am @@ -0,0 +1,38 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Tools HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +if PARALLEL_TOOLS_CONDITIONAL + H5DWALK=h5dwalk +else + H5DWALK= +endif + +if BUILD_TESTS_CONDITIONAL + TESTSERIAL_DIR =test +else + TESTSERIAL_DIR= +endif + +CONFIG=ordered + +# All subdirectories +SUBDIRS=$(H5DWALK) $(TESTSERIAL_DIR) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/h5dwalk/CMakeLists.txt b/utils/tools/h5dwalk/CMakeLists.txt new file mode 100644 index 0000000..244cc26 --- /dev/null +++ b/utils/tools/h5dwalk/CMakeLists.txt @@ -0,0 +1,66 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_UTILS_TOOLS_H5DWALK C) + +# -------------------------------------------------------------------- +# Add the h5dwalk and test executables +# -------------------------------------------------------------------- +if (NOT ONLY_SHARED_LIBS) + add_executable (h5dwalk ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) +# add_custom_target(generate_demo ALL +# DEPENDS "${HDF5_TOOLS_DIR}/test/demo_destfiles.test" +# ) + target_include_directories (h5dwalk PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(h5dwalk PRIVATE "${HDF5_CMAKE_C_FLAGS}") + TARGET_C_PROPERTIES (h5dwalk STATIC) + target_link_libraries (h5dwalk PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${MFU_LIBRARY} "$<$:${MPI_C_LIBRARIES}>") + set_target_properties (h5dwalk PROPERTIES FOLDER tools) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk") + + set (H5_DEP_EXECUTABLES h5dwalk) +endif () + +if (BUILD_SHARED_LIBS) + add_executable (h5dwalk-shared ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) + target_include_directories (h5dwalk-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(h5dwalk-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") + TARGET_C_PROPERTIES (h5dwalk-shared SHARED) + target_link_libraries (h5dwalk-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${MFU_LIBRARY} "$<$:${MPI_C_LIBRARIES}>") + set_target_properties (h5dwalk-shared PROPERTIES FOLDER tools) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk-shared") + + set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} h5dwalk-shared) +endif () + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + if (NOT ONLY_SHARED_LIBS) + clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk) + else () + clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk-shared) + endif () +endif () + +############################################################################## +############################################################################## +### I N S T A L L A T I O N ### +############################################################################## +############################################################################## + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- +if (HDF5_EXPORTED_TARGETS) + foreach (exec ${H5_DEP_EXECUTABLES}) + INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} toolsapplications) + endforeach () + + install ( + TARGETS + ${H5_DEP_EXECUTABLES} + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications + ) +endif () diff --git a/utils/tools/h5dwalk/Makefile.am b/utils/tools/h5dwalk/Makefile.am new file mode 100644 index 0000000..34cdb32 --- /dev/null +++ b/utils/tools/h5dwalk/Makefile.am @@ -0,0 +1,37 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src directory +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib $(H5DWALK_CPPFLAGS) + +# These are our main targets, the tools +# h5dwalk_SOURCES=h5dwalk.c $(TOOLSOURCES) +bin_PROGRAMS=h5dwalk +#bin_SCRIPTS=install-examples + +# Add h5stat specific linker flags here +h5dwalk_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) $(H5DWALK_LDFLAGS) + +# Tell automake to clean h5redeploy script +CLEANFILES= + +# All programs rely on hdf5 library and h5tools library +h5dwalk_LDADD=$(LIBH5TOOLS) $(LIBHDF5) $(H5DWALK_LIBS) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/h5dwalk/h5dwalk.1 b/utils/tools/h5dwalk/h5dwalk.1 new file mode 100644 index 0000000..eb0e5e8 --- /dev/null +++ b/utils/tools/h5dwalk/h5dwalk.1 @@ -0,0 +1,42 @@ +.TH "h5dwalk" 1 +.SH NAME +h5dwalk \- Provides a means of extending HDF5 tools by using parallelism on groups of files. +.SH SYNOPSIS +h5dwalk [OPTIONS] -T h5tool [H5TOOL_options...] +.SH DESCRIPTION +h5dwalk utilizes the mpiFileUtils library to invoke a selected HDF5 tool on a collection of files. The mpiFileUtils library provides the facilities to walk directory trees and provide a selection of files contained therein. This selection can be filtered in various ways. At present, h5dwalk filters the original file selection to include only HDF5 formatted files. The resulting collection or collections can be utilized as the file inputs to the selected h5tool. +.SH OPTIONS +.TP +.B \-h +or +.B \-\-help +Print a usage message and exit. +.TP +.B \-i +or +.B \-\-input filename +Read command input from a file. Not yet implemented. +.TP +.B \-o +or +.B \-\-output filename +Captures the hdf5 tool output into a named file. +.TP +.B \-l +or +.B \-\-log [file] +Captures hdf5 tool output into a individual log files. If an optional file (directory) is specified, then output from all tool instances will be written in the given file directory. Without the optional filename, each tool instance output will be captured in a new log file whose name is associated with the hdf5 tool that was run and is written in the current working directory. +.TP +.B \-E +or +.B \-\-error [file] +Show all HDF5 error reporting. Behavior is similar to --log, i.e. errors can either be logged in a single named file or in individual tool specific files. Not yet implemented. +.TP +.B \-T +or +.B \-\-tool hdf5_tool +Specifies the hdf5 tool that should be invoked for each file in a collection of files. The collection consists of individual HDF5 files found by walking a specified directory tree which is used in place of the normal tool filename argument. The '-T' option should appear on the command line just prior to the HDF5 tool argument options. +.TP +.SH "SEE ALSO" +\&\fIh5dump\fR\|(1), \fIh5diff\fR\|(1), \fIh5repart\fR\|(1), \fIh5diff\fR\|(1), +\&\fIh5import\fR\|(1), \fIgif2h5\fR\|(1), \fIh52gif\fR\|(1), \fIh5perf\fR\|(1) diff --git a/utils/tools/h5dwalk/h5dwalk.c b/utils/tools/h5dwalk/h5dwalk.c new file mode 100644 index 0000000..5a22d75 --- /dev/null +++ b/utils/tools/h5dwalk/h5dwalk.c @@ -0,0 +1,1714 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "H5private.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#include "hdf5.h" + +#include "libcircle.h" +#include "dtcmp.h" +#include "mfu.h" +#include "mfu_flist.h" +#include "mfu_errors.h" +#include "mfu_flist_internal.h" + +/* Name of tool */ +#define PROGRAMNAME "h5dwalk" + +#ifdef DAOS_SUPPORT +#include "mfu_daos.h" +#endif + +static char *user_cmd = NULL; +static char mpierrstr[MPI_MAX_ERROR_STRING]; +static int mpierrlen; +static int sg_mpi_rank = 0; +static int current_input_index = 0; +static int processing_inputfile = 0; + +static void dh5tool_flist_write_text(const char *name, mfu_flist bflist); +static void run_command(int argc, char **argv, char *cmdline, const char *fname); +static void add_executable(int argc, char **argv, char *cmdstring, int *f_index, int f_count); +static int process_input_file(char *inputname, int myrank, int size); +static void usage(void); + +H5_ATTR_NORETURN void h5dwalk_exit(int status); + +/* keep stats during walk */ +uint64_t total_dirs = 0; +uint64_t total_files = 0; +uint64_t total_links = 0; +uint64_t total_unknown = 0; +uint64_t total_bytes = 0; +/* global flags which indicate whether we need + * to capture tool outputs into a file... + * Related to this is whether the stderr should + * be logged separately. + */ +#define BUFT_SIZE 131072 +/* FIXME: 'buft_max' should probably be configurable.. */ +size_t buft_max = 64; +size_t buft_count = 0; +buf_t **buf_cache = NULL; + +int log_output_in_single_file = 0; +char *output_log_file = NULL; + +int log_stdout_in_file = 0; +char *txtlog = NULL; + +int log_errors_in_file = 0; +char *errlog = NULL; + +int use_config_file = 0; +int config_index[4] = { + 0, +}; + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +#define MAX_DISTRIBUTE_SEPARATORS 128 +struct distribute_option { + int separator_number; + uint64_t separators[MAX_DISTRIBUTE_SEPARATORS]; +}; + +static const char * s_opts = "hl*E*i:o:T:"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, + {"log_text", optional_arg, 'l'}, + {"error", optional_arg, 'E'}, + {"input", require_arg, 'i'}, + {"output", require_arg, 'o'}, + {"tool", require_arg, 'T'}, + {NULL, 0, '\0'}}; +static void +save_command(const char *argv0) +{ + assert(argv0); + user_cmd = HDstrdup(argv0); +} + +static void +create_default_separators(struct distribute_option *option, mfu_flist *flist, uint64_t *size, + size_t *separators, uint64_t *global_max_file_size) +{ + /* get local max file size for Allreduce */ + uint64_t local_max_file_size = 0; + for (uint64_t i = 0; i < *size; i++) { + uint64_t file_size = mfu_flist_file_get_size(*flist, i); + if (file_size > local_max_file_size) { + local_max_file_size = file_size; + } + } + + /* get the max file size across all ranks */ + MPI_Allreduce(&local_max_file_size, global_max_file_size, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); + + /* print and convert max file size to appropriate units */ + double max_size_tmp; + const char *max_size_units; + mfu_format_bytes(*global_max_file_size, &max_size_tmp, &max_size_units); + HDprintf("Max File Size: %.3lf %s\n", max_size_tmp, max_size_units); + + /* round next_pow_2 to next multiple of 10 */ + uint64_t max_magnitude_bin = (uint64_t)((ceil(log2((double)(*global_max_file_size)) / 10)) * 10); + + /* get bin ranges based on max file size */ + option->separators[0] = 1; + + /* plus one is for zero count bin */ + *separators = (size_t)(max_magnitude_bin / 10); + uint64_t power = 10; + for (int i = 1; power <= max_magnitude_bin; i++) { + double raised_2 = pow(2, (double)(power)); + option->separators[i] = (uint64_t)raised_2; + power += 10; + } +} + +static int +h5dwalk_map_fn(mfu_flist flist __attribute__((unused)), uint64_t idx, int ranks, + void *args __attribute__((unused))) +{ + int rank = (int)((int)idx % ranks); + return rank; +} + +static int +print_flist_distribution(int file_histogram, struct distribute_option *option, mfu_flist *pflist, int rank) +{ + /* file list to use */ + mfu_flist flist = *pflist; + + /* get local size for each rank, and max file sizes */ + uint64_t size = mfu_flist_size(flist); + uint64_t global_max_file_size; + + size_t separators = 0; + if (file_histogram) { + /* create default separators */ + create_default_separators(option, &flist, &size, &separators, &global_max_file_size); + } + else { + separators = (size_t)option->separator_number; + } + + /* allocate a count for each bin, initialize the bin counts to 0 + * it is separator + 1 because the last bin is the last separator + * to the DISTRIBUTE_MAX */ + uint64_t *dist = (uint64_t *)MFU_MALLOC((separators + 1) * sizeof(uint64_t)); + + /* initialize the bin counts to 0 */ + for (size_t i = 0; i <= separators; i++) { + dist[i] = 0; + } + + /* for each file, identify appropriate bin and increment its count */ + for (size_t i = 0; i < size; i++) { + /* get the size of the file */ + uint64_t file_size = mfu_flist_file_get_size(flist, i); + + /* loop through the bins and find the one the file belongs to, + * set last bin to -1, if a bin is not found while looping through the + * list of file size separators, then it belongs in the last bin + * so (last file size - MAX bin) */ + int64_t max_bin_flag = -1; + for (size_t j = 0; j < separators; j++) { + if (file_size <= option->separators[j]) { + /* found the bin set bin index & increment its count */ + dist[j]++; + + /* a file for this bin was found so can't belong to + * last bin (so set the flag) & exit the loop */ + max_bin_flag = 1; + break; + } + } + + /* if max_bin_flag is still -1 then the file belongs to the last bin */ + if (max_bin_flag < 0) { + dist[separators]++; + } + } + + /* get the total sum across all of the bins */ + uint64_t *disttotal = (uint64_t *)MFU_MALLOC((separators + 1) * sizeof(uint64_t)); + MPI_Allreduce(dist, disttotal, (int)(separators + 1), MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); + + /* Print the file distribution */ + if (rank == 0) { + /* number of files in a bin */ + uint64_t number; + double size_tmp; + const char *size_units; + HDprintf("%-27s %s\n", "Range", "Number"); + for (size_t i = 0; i <= separators; i++) { + HDprintf("%s", "[ "); + if (i == 0) { + HDprintf("%7.3lf %3s", 0.000, "B"); + } + else { + mfu_format_bytes((uint64_t)option->separators[i - 1], &size_tmp, &size_units); + HDprintf("%7.3lf %3s", size_tmp, size_units); + } + + printf("%s", " - "); + + if (file_histogram) { + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + number = disttotal[i]; + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + HDprintf("%7.3lf %3s ) %" PRIu64 "\n", size_tmp, size_units, number); + } + else { + if (i == separators) { + number = disttotal[i]; + HDprintf("%10s ) %" PRIu64 "\n", "MAX", number); + } + else { + number = disttotal[i]; + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + HDprintf("%7.3lf %3s ) %" PRIu64 "\n", size_tmp, size_units, number); + } + } + } + } + + /* free the memory used to hold bin counts */ + mfu_free(&disttotal); + mfu_free(&dist); + + return 0; +} + +/* * Search the right position to insert the separator * If the separator exists already, return failure * + * Otherwise, locate the right position, and move the array forward to save the separator. + */ +static int +distribute_separator_add(struct distribute_option *option, uint64_t separator) +{ + int low = 0; + int high; + int middle; + int pos; + int count; + + count = option->separator_number; + option->separator_number++; + if (option->separator_number > MAX_DISTRIBUTE_SEPARATORS) { + HDprintf("Too many separators"); + return -1; + } + + if (count == 0) { + option->separators[0] = separator; + return 0; + } + + high = count - 1; + while (low < high) { + middle = (high - low) / 2 + low; + if (option->separators[middle] == separator) + return -1; + /* In the left half */ + else if (option->separators[middle] < separator) + low = middle + 1; + /* In the right half */ + else + high = middle; + } + assert(low == high); + if (option->separators[low] == separator) + return -1; + + if (option->separators[low] < separator) + pos = low + 1; + else + pos = low; + + if (pos < count) + HDmemmove(&option->separators[low + 1], &option->separators[low], + sizeof(*option->separators) * (uint64_t)(count - pos)); + + option->separators[pos] = separator; + return 0; +} + +static int +distribution_parse(struct distribute_option *option, const char *string) +{ + char * ptr; + char * next; + unsigned long long separator; + char * str; + int status = 0; + + if (strncmp(string, "size", strlen("size")) != 0) { + return -1; + } + + option->separator_number = 0; + if (strlen(string) == strlen("size")) { + return 0; + } + + if (string[strlen("size")] != ':') { + return -1; + } + + str = HDstrdup(string); + /* Parse separators */ + ptr = str + strlen("size:"); + next = ptr; + while (ptr && ptr < str + strlen(string)) { + next = strchr(ptr, ','); + if (next != NULL) { + *next = '\0'; + next++; + } + + if (mfu_abtoull(ptr, &separator) != MFU_SUCCESS) { + HDprintf("Invalid separator \"%s\"\n", ptr); + status = -1; + goto out; + } + + if (distribute_separator_add(option, separator)) { + HDprintf("Duplicated separator \"%llu\"\n", separator); + status = -1; + goto out; + } + + ptr = next; + } + +out: + mfu_free(&str); + return status; +} + +static void +usage(void) +{ + if (sg_mpi_rank) + return; + + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "Usage: h5dwalk [options] ...\n"); +#ifdef DAOS_SUPPORT + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "DAOS paths can be specified as:\n"); + PRINTVALSTREAM(rawoutstream, " daos:///[/] | \n"); +#endif + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "Options:\n"); + PRINTVALSTREAM(rawoutstream, " -i, --input - read list from file\n"); + PRINTVALSTREAM(rawoutstream, " -o, --output - write output summary to the named file.\n"); + PRINTVALSTREAM(rawoutstream, + " -E, --error - write processed errors to file in text format\n"); + PRINTVALSTREAM( + rawoutstream, + " -l, --log_text - write individual tool outputs to a file. Logs can be written to an " + "optional named directory.\n"); + PRINTVALSTREAM(rawoutstream, " -T, --tool - name of the HDF5 tool to invoke\n"); + PRINTVALSTREAM(rawoutstream, " -h, --help - print usage\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "For more information see https://mpifileutils.readthedocs.io. \n"); + PRINTVALSTREAM(rawoutstream, "\n"); +} + +/* given an index, return pointer to that file element, + * NULL if index is not in range */ +static elem_t * +list_get_elem(flist_t *flist, uint64_t idx) +{ + /* return pointer to element if index is within range */ + uint64_t max = flist->list_count; + if (idx < max) { + elem_t *elem = flist->list_index[idx]; + return elem; + } + return NULL; +} + +#ifdef VERBOSE +/* print information about a file given the index and rank (used in print_files) */ +static void +print_file(mfu_flist flist, uint64_t idx) +{ + /* store types as strings for print_file */ + char type_str_unknown[] = "UNK"; + char type_str_dir[] = "DIR"; + char type_str_file[] = "REG"; + char type_str_link[] = "LNK"; + + /* get filename */ + const char *file = mfu_flist_file_get_name(flist, idx); + + if (mfu_flist_have_detail(flist)) { + /* get mode */ + mode_t mode = (mode_t)mfu_flist_file_get_mode(flist, idx); + uint64_t acc = mfu_flist_file_get_atime(flist, idx); + uint64_t mod = mfu_flist_file_get_mtime(flist, idx); + uint64_t cre = mfu_flist_file_get_ctime(flist, idx); + uint64_t size = mfu_flist_file_get_size(flist, idx); + const char *username = mfu_flist_file_get_username(flist, idx); + const char *groupname = mfu_flist_file_get_groupname(flist, idx); + + char access_s[30]; + char modify_s[30]; + char create_s[30]; + time_t access_t = (time_t)acc; + time_t modify_t = (time_t)mod; + time_t create_t = (time_t)cre; + size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t)); + size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t)); + size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t)); + if (access_rc == 0 || modify_rc == 0 || create_rc == 0) { + /* error */ + access_s[0] = '\0'; + modify_s[0] = '\0'; + create_s[0] = '\0'; + } + + char mode_format[11]; + mfu_format_mode(mode, mode_format); + + double size_tmp; + const char *size_units; + mfu_format_bytes(size, &size_tmp, &size_units); + + HDprintf("%s %s %s %7.3f %3s %s %s\n", mode_format, username, groupname, size_tmp, size_units, + modify_s, file); + } + else { + /* get type */ + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + char * type_str = type_str_unknown; + if (type == MFU_TYPE_DIR) { + type_str = type_str_dir; + } + else if (type == MFU_TYPE_FILE) { + type_str = type_str_file; + } + else if (type == MFU_TYPE_LINK) { + type_str = type_str_link; + } + + HDprintf("Type=%s File=%s\n", type_str, file); + } +} + +/* TODO: move this somewhere or modify existing print_file */ +/* print information about a file given the index and rank (used in print_files) */ +static size_t +print_file_text(mfu_flist flist, uint64_t idx, char *buffer, size_t bufsize) +{ + size_t numbytes = 0; + + /* store types as strings for print_file */ + char type_str_unknown[] = "UNK"; + char type_str_dir[] = "DIR"; + char type_str_file[] = "REG"; + char type_str_link[] = "LNK"; + + /* get filename */ + const char *file = mfu_flist_file_get_name(flist, idx); + + if (mfu_flist_have_detail(flist)) { + /* get mode */ + mode_t mode = (mode_t)mfu_flist_file_get_mode(flist, idx); + + uint64_t acc = mfu_flist_file_get_atime(flist, idx); + uint64_t mod = mfu_flist_file_get_mtime(flist, idx); + uint64_t cre = mfu_flist_file_get_ctime(flist, idx); + uint64_t size = mfu_flist_file_get_size(flist, idx); + const char *username = mfu_flist_file_get_username(flist, idx); + const char *groupname = mfu_flist_file_get_groupname(flist, idx); + + char access_s[30]; + char modify_s[30]; + char create_s[30]; + time_t access_t = (time_t)acc; + time_t modify_t = (time_t)mod; + time_t create_t = (time_t)cre; + size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t)); + size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t)); + size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t)); + if (access_rc == 0 || modify_rc == 0 || create_rc == 0) { + /* error */ + access_s[0] = '\0'; + modify_s[0] = '\0'; + create_s[0] = '\0'; + } + + char mode_format[11]; + mfu_format_mode(mode, mode_format); + + double size_tmp; + const char *size_units; + mfu_format_bytes(size, &size_tmp, &size_units); + + numbytes = (size_t)snHDprintf(buffer, bufsize, "%s %s %s %7.3f %3s %s %s\n", mode_format, username, + groupname, size_tmp, size_units, modify_s, file); + } + else { + /* get type */ + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + char * type_str = type_str_unknown; + if (type == MFU_TYPE_DIR) { + type_str = type_str_dir; + } + else if (type == MFU_TYPE_FILE) { + type_str = type_str_file; + } + else if (type == MFU_TYPE_LINK) { + type_str = type_str_link; + } + + numbytes = (size_t)snHDprintf(buffer, bufsize, "Type=%s File=%s\n", type_str, file); + } + + return numbytes; +} +#endif + +static size_t +get_local_bufsize(uint64_t *bufsize) +{ + size_t total = 0; + if (buft_count > 0) { + buf_t *lastbuf = buf_cache[buft_count - 1]; + size_t remaining = lastbuf->count; + total = (lastbuf->bufsize * buft_count) - remaining; + *bufsize = (uint64_t)(lastbuf->bufsize); + } + return total; +} + +static void +dh5tool_flist_write_text(const char *name, mfu_flist bflist) +{ + /* convert handle to flist_t */ + flist_t *flist = (flist_t *)bflist; + + /* get our rank and size of the communicator */ + int rank, ranks; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &ranks); + + /* start timer */ + double start_write = MPI_Wtime(); + + /* total list items */ + uint64_t all_count = mfu_flist_global_size(flist); + + /* report the filename we're writing to */ + if (mfu_rank == 0) { + MFU_LOG(MFU_LOG_INFO, "Writing to output file: %s", name); + } + + uint64_t idx = 0; + char * ptr = NULL; + + /* if we block things up into 128MB chunks, how many iterations + * to write everything? */ + // uint64_t maxwrite = 128 * 1024 * 1024; + uint64_t maxwrite = 0; + size_t local_total = get_local_bufsize(&maxwrite); + uint64_t iters = 0; + if (local_total > 0) + iters = (uint64_t)local_total / maxwrite; + + if (iters * maxwrite < (uint64_t)local_total) { + iters++; + } + + /* get max iterations across all procs */ + uint64_t all_iters; + MPI_Allreduce(&iters, &all_iters, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); + + /* use mpi io hints to stripe across OSTs */ + MPI_Info info; + MPI_Info_create(&info); + + /* change number of ranks to string to pass to MPI_Info */ + char str_buf[12]; + HDprintf(str_buf, "%d", ranks); + + /* no. of I/O devices for lustre striping is number of ranks */ + MPI_Info_set(info, "striping_factor", str_buf); + + /* open file */ + MPI_Status status; + MPI_File fh; + const char *datarep = "native"; + int amode = MPI_MODE_WRONLY | MPI_MODE_CREATE; + + int mpirc = MPI_File_open(MPI_COMM_WORLD, (const char *)name, amode, info, &fh); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to open file for writing: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* truncate file to 0 bytes */ + mpirc = MPI_File_set_size(fh, 0); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to truncate file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* set file view to be sequence of datatypes past header */ + mpirc = MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, datarep, MPI_INFO_NULL); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to set view on file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* compute byte offset to write our element */ + uint64_t offset = 0; + uint64_t bytes = (uint64_t)local_total; + MPI_Exscan(&bytes, &offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); + MPI_Offset write_offset = (MPI_Offset)offset; + + uint64_t written = 0; + while (all_iters > 0) { + /* compute number of bytes left to write */ + uint64_t remaining = (uint64_t)local_total - written; + + /* maybe Incr pointer to our next buffer */ + if (remaining == 0) { + idx++; + if (buf_cache[idx]->buf == NULL) { + } + } + + /* compute count we'll write in this iteration */ + int write_count = (int)maxwrite; + if (remaining < maxwrite) { + write_count = (int)remaining; + } + /* Get the buffer to output to the selected file */ + ptr = buf_cache[idx]->buf; + + /* collective write of file data */ + mpirc = MPI_File_write_at_all(fh, write_offset, ptr, write_count, MPI_BYTE, &status); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to write to file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* update our offset into the file */ + write_offset += (MPI_Offset)write_count; + + /* update number of bytes written so far */ + written += (uint64_t)write_count; + + /* update pointer into our buffer */ + ptr += write_count; + + /* decrement our collective write loop counter */ + all_iters--; + } + + /* free buffer */ + // mfu_free(&buf); + + /* close file */ + mpirc = MPI_File_close(&fh); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to close file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* free mpi info */ + MPI_Info_free(&info); + + /* end timer */ + double end_write = MPI_Wtime(); + + /* report write count, time, and rate */ + if (mfu_rank == 0) { + double secs = end_write - start_write; + double rate = 0.0; + if (secs > 0.0) { + rate = ((double)all_count) / secs; + } + MFU_LOG(MFU_LOG_INFO, "Wrote %lu files in %.3lf seconds (%.3lf files/sec)", all_count, secs, rate); + } + + return; +} + +static void +filter_hdf_files(mfu_flist *pflist, char *regex_exp, int exclude, int name) +{ + mfu_flist flist = *pflist; + mfu_flist eligible = mfu_flist_subset(flist); + uint64_t idx = 0; + uint64_t files = mfu_flist_size(flist); + while (idx < files) { + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK || type == MFU_TYPE_UNKNOWN) { + const char *file = mfu_flist_file_get_name(flist, idx); + int accessible = H5Fis_accessible(file, H5P_DEFAULT); + if (accessible) + mfu_flist_file_copy(flist, idx, eligible); + } + idx++; + } + + mfu_flist_summarize(eligible); + + /* assume we'll use the full list */ + // mfu_flist srclist = flist; + mfu_flist srclist = eligible; + + /* filter the list if needed */ + mfu_flist filtered_flist = MFU_FLIST_NULL; + if (regex_exp != NULL) { + /* filter the list based on regex */ + filtered_flist = mfu_flist_filter_regex(eligible, regex_exp, exclude, name); + + /* update our source list to use the filtered list instead of the original */ + srclist = filtered_flist; + } + + mfu_flist_free(&flist); + *pflist = srclist; + return; +} + +static int +fill_file_list(mfu_flist new_flist, const char *config_filename, int myrank, int size) +{ + int index = 0; + char linebuf[PATH_MAX] = { + '\0', + }; + FILE *config = HDfopen(config_filename, "r"); + if (config == NULL) + return -1; + while (HDfgets(linebuf, sizeof(linebuf), config) != NULL) { + struct stat statbuf; + char * eol = HDstrchr(linebuf, '\n'); + if (eol) + *eol = '\0'; + if (HDstat(linebuf, &statbuf) == 0) { + if (myrank == (index % size)) { + mfu_flist_insert_stat((flist_t *)new_flist, linebuf, O_RDONLY, &statbuf); + } + index++; + } + linebuf[0] = 0; + } + HDfclose(config); + return index; +} + +static int +count_dirpaths(int argc, int startcnt, const char *argv[], int **index_out) +{ + int k; + int path_cnt = 0; + int idx_count = (argc - startcnt); + int * index = NULL; + struct stat pathcheck; + + if (idx_count > 0) { + index = (int *)malloc((size_t)(argc - startcnt) * sizeof(int)); + assert(index); + } + else + return 0; + + for (k = startcnt; k < argc; k++) { + char *slash = NULL; + int c = *argv[k]; + if ((c == '.') || (c == '/')) { + index[path_cnt++] = k; + } + else if ((c == '@')) { + const char *configFile = argv[k] + 1; + if (stat(configFile, &pathcheck) == 0) { + if (S_ISREG(pathcheck.st_mode)) { + config_index[use_config_file++] = k; + } + } + } + else if ((slash = strchr(argv[k], '/')) != NULL) { + if (stat(argv[k], &pathcheck) == 0) { + if (S_ISDIR(pathcheck.st_mode)) + index[path_cnt++] = k; + } + } + } + if ((path_cnt == 0) && (index != NULL)) { + free(index); + return 0; + } + *index_out = index; + return path_cnt; +} + +static char ** +copy_args(int argc, const char *argv[], int *mfu_argc, int *copy_len) +{ + int i, bytes_copied = 0; + int check_mfu_args = 1; + char **argv_copy = (char **)MFU_MALLOC((size_t)(argc + 2) * sizeof(char **)); + assert(argv_copy); + assert(mfu_argc); + assert(copy_len); + save_command(argv[0]); + + for (i = 0; i < argc; i++) { + argv_copy[i] = HDstrdup(argv[i]); + bytes_copied += (int)(strlen(argv[i]) + 1); + argv_copy[i] = HDstrdup(argv[i]); + if (check_mfu_args && (HDstrncmp(argv[i], "-T", 2) == 0)) { + check_mfu_args = 0; + *mfu_argc = i + 1; + } + } + argv_copy[i] = 0; + *copy_len = bytes_copied; + return argv_copy; +} + +typedef struct hash_entry { + int hash; + char * name; + struct hash_entry *next; /* table Collision */ + int nextCount; +} hash_entry_t; + +#ifndef NAME_ENTRIES +#define NAME_ENTRIES 4096 +#endif + +static hash_entry_t filename_cache[NAME_ENTRIES]; + +static int +get_copy_count(char *fname, char *appname) +{ + int filehash = 0, apphash = 0; + size_t k, applen = strlen(appname); + size_t filelen = strlen(fname); + int hash_index; + + for (k = 0; k < filelen; k++) { + filehash += fname[k]; + } + for (k = 0; k < applen; k++) { + apphash += appname[k]; + } + hash_index = filehash % NAME_ENTRIES; + if (filename_cache[hash_index].name == NULL) { + filename_cache[hash_index].hash = apphash; + filename_cache[hash_index].name = HDstrdup(fname); + filename_cache[hash_index].next = NULL; + filename_cache[hash_index].nextCount = 1; + return 0; + } + else if ((apphash == filename_cache[hash_index].hash) && + (strcmp(filename_cache[hash_index].name, fname) == 0)) { + int retval = filename_cache[hash_index].nextCount++; + return retval; + } + else { /* Collision */ + hash_entry_t *nextEntry = &filename_cache[hash_index]; + hash_entry_t *lastEntry = nextEntry; + while (nextEntry) { + if ((apphash == nextEntry->hash) && (strcmp(nextEntry->name, fname) == 0)) { + /* Match (increment nextCount and return) */ + int retval = nextEntry->nextCount++; + return retval; + } + else { + /* No Match (continue search) */ + lastEntry = nextEntry; + nextEntry = lastEntry->next; + } + } + nextEntry = (hash_entry_t *)malloc(sizeof(hash_entry_t)); + if (nextEntry) { + lastEntry->next = nextEntry; + nextEntry->name = HDstrdup(fname); + nextEntry->hash = apphash; + nextEntry->next = NULL; + nextEntry->nextCount = 1; + } + } + return 0; +} + +static void +run_command(int argc __attribute__((unused)), char **argv, char *cmdline, const char *fname) +{ + char filepath[1024]; + char *toolname = argv[0]; + char *buf = NULL; + int use_stdout = 0; + +#ifdef H5_HAVE_WINDOWS + HDprintf("ERROR: %s %s: Unable to support fork/exec on WINDOWS\n", PROGRAMNAME, __func__); + h5dwalk_exit(EXIT_FAILURE); +#else + + /* create a copy of the 1st file passed to the application */ + HDstrcpy(filepath, fname); + + if (log_output_in_single_file || use_stdout) { + pid_t pid; + int pipefd[2]; + buf_t * thisbuft = NULL; + buf_t **bufs = buf_cache; + + if (bufs == NULL) { + bufs = (buf_t **)MFU_CALLOC(buft_max, sizeof(buf_t *)); + assert((bufs != NULL)); + buf_cache = bufs; +#ifdef VERBOSE + if (buft_count == 0) { + HDprintf("[%d] Initial buf_cache allocation: buft_count=%d\n", sg_mpi_rank, buft_count); + } +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert((thisbuft != NULL)); + } + else { + thisbuft = bufs[buft_count - 1]; + assert((thisbuft != NULL)); + /* Check for remaining space in the current buffer */ + /* If none, then create a new buffer */ + if (thisbuft->count == 0) { + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + } + } + if ((thisbuft->buf == NULL)) { + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + assert((thisbuft->buf != NULL)); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->count = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + } + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + pid = fork(); + if (pid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + if (pid == 0) { + close(pipefd[0]); + dup2(pipefd[1], fileno(stdout)); + dup2(pipefd[1], fileno(stderr)); + execvp(argv[0], argv); + } + else { + int w_status; + size_t nbytes; + size_t read_bytes = 0; + uint64_t remaining, offset; + close(pipefd[1]); + buf = thisbuft->buf; + remaining = thisbuft->count; + offset = thisbuft->chars; + nbytes = strlen(cmdline); + /* Record the command line for the log! */ + if (nbytes < remaining) { + HDstrcpy(&buf[offset], cmdline); + thisbuft->chars += nbytes; + thisbuft->count -= nbytes; + remaining -= nbytes; + } + else { /* We're running out of space in the current buffer */ + char *nextpart; + strncpy(&buf[offset], cmdline, remaining); + nextpart = &cmdline[remaining + 1]; + thisbuft->count = 0; + thisbuft->chars += remaining; + + /* Create a new read buffer */ +#ifdef VERBOSE + HDprintf("[%d] Allocate-1 a new read buffer:: buft_count=%d\n", sg_mpi_rank, buft_count); +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert(thisbuft != NULL); + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + /* Copy the remaining cmdline text into the new buffer */ + HDstrcpy(buf, nextpart); + /* And update our buffer info */ + // thisbuft->chars = strlen(nextpart) +1; + thisbuft->chars = strlen(nextpart); + thisbuft->count = BUFT_SIZE - thisbuft->chars; + } + offset = thisbuft->chars; + + do { + waitpid(pid, &w_status, WNOHANG); + if ((nbytes = (size_t)read(pipefd[0], &buf[offset], remaining)) > 0) { + offset += nbytes; + read_bytes += nbytes; + remaining -= nbytes; + if (remaining == 0) { + /* Update the current buffer prior to allocating the new one */ + thisbuft->count = 0; + thisbuft->chars += read_bytes; +#ifdef VERBOSE + HDprintf("[%d] Allocate-2 a new read buffer:: buft_count=%d\n", sg_mpi_rank, + buft_count); +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert(thisbuft != NULL); + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + thisbuft->chars = BUFT_SIZE; + offset = 0; + remaining = BUFT_SIZE; + } + } + } while (!WIFEXITED(w_status)); + close(pipefd[0]); + wait(NULL); + + thisbuft->count = remaining; + thisbuft->chars = thisbuft->bufsize - remaining; + } + } + else if (log_stdout_in_file) { + int log_instance = -1; + pid_t pid; + size_t log_len; + char logpath[2048]; + char logErrors[2048]; + char current_dir[2048]; + char * logbase = HDstrdup(basename(filepath)); + char * thisapp = HDstrdup(basename(toolname)); + + if (processing_inputfile == 0) + log_instance = get_copy_count(logbase, thisapp); + + if (txtlog == NULL) { + if ((log_instance > 0) || processing_inputfile) { + if (processing_inputfile) + log_instance = current_input_index; + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log_%d", + HDgetcwd(current_dir, sizeof(current_dir)), logbase, thisapp, log_instance); + } + else { + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log", + HDgetcwd(current_dir, sizeof(current_dir)), logbase, thisapp); + } + } + else { + log_len = strlen(txtlog); + if ((log_instance > 0) || processing_inputfile) { + if (processing_inputfile) + log_instance = current_input_index; + if (txtlog[log_len - 1] == '/') + HDsnprintf(logpath, sizeof(logpath), "%s%s_%s.log_%d", txtlog, logbase, thisapp, + log_instance); + else + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log_%d", txtlog, logbase, thisapp, + log_instance); + } + else { + if (txtlog[log_len - 1] == '/') + HDsnprintf(logpath, sizeof(logpath), "%s%s_%s.log", txtlog, logbase, thisapp); + else + HDsnprintf(logpath, sizeof(logpath), "%s/%s_%s.log", txtlog, logbase, thisapp); + } + } + + if (log_errors_in_file) { + /* We co-locate the error logs in the same directories as the regular log files. + * The easiest way to do this is to simply replace the .log with .err in a + * copy of the logpath variable. + */ + log_len = strlen(logpath); + HDstrcpy(logErrors, logpath); + HDstrcpy(&logErrors[log_len - 3], "err"); + } + if (mfu_debug_level == MFU_LOG_VERBOSE) { + HDprintf("\tCreating logfile: %s\n", logpath); + fflush(stdout); + } + pid = fork(); + if (pid == 0) { + int efd; + int fd = open(logpath, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + dup2(fd, fileno(stdout)); + if (log_errors_in_file) { + efd = open(logErrors, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + dup2(efd, fileno(stderr)); + close(efd); + } + else + dup2(fd, fileno(stderr)); + close(fd); + execvp(argv[0], argv); + } + int status; + pid = wait(&status); + if (logbase) + free(logbase); + if (thisapp) + free(thisapp); + } /* else if(log_stdout_in_file) */ +#endif /* #ifdef H5_HAVE_WINDOWS */ +} + +int MFU_PRED_EXEC(mfu_flist flist, uint64_t idx, void *arg); +int MFU_PRED_PRINT(mfu_flist flist, uint64_t idx, void *arg); + +int +MFU_PRED_EXEC(mfu_flist flist, uint64_t idx, void *arg) +{ + /* get file name for this item */ + int file_substituted = 0; + const char *fname = mfu_flist_file_get_name(flist, idx); + + char *toolname = NULL; + char filepath[1024]; + + size_t b_offset; + + /* get pointer to encoded argc count and argv array */ + int * count_ptr = arg; + char *buf = (char *)arg + sizeof(int); + + /* get number of argv parameters */ + int k = 0, count = *count_ptr; + toolname = buf; + + /* Get a copy of fname */ + HDstrcpy(filepath, fname); + + /* allocate a char* for each item in the argv array, + * plus one more for a trailing NULL + * 'count' in this case is the number of args, so + * so we add (+1) for the toolname and another (+1) + * for the trailing NULL to terminate the list + */ + + char cmdline[2048]; + char **argv = (char **)MFU_CALLOC((size_t)(count + 2), sizeof(char *)); + + argv[k++] = HDstrdup(toolname); + + HDmemset(cmdline, 0, sizeof(cmdline)); + buf += HDstrlen(toolname) + 1; + /* Reconstruct the command line that the user provided for the h5tool */ + for (k = 1; k < count; k++) { + if (buf[0] == '&') { + const char *fname_arg = NULL; + mfu_flist flist_arg; + void * check_ptr[2] = {NULL, NULL}; + + HDmemcpy(check_ptr, &buf[1], sizeof(void *)); + flist_arg = (mfu_flist)check_ptr[0]; + + /* +2 (see below) accounts for the '&' and the trailing zero pad */ + buf += sizeof(mfu_flist *) + 2; + fname_arg = mfu_flist_file_get_name(flist_arg, idx); + if (fname_arg == NULL) { + HDprintf("[%d] Warning: Unable to resolve file_substitution %d (idx=%ld)\n", sg_mpi_rank, + file_substituted, idx); + argv[k] = HDstrdup(fname); + } + else { + argv[k] = HDstrdup(fname_arg); + file_substituted++; + } + } + else { + argv[k] = HDstrdup(buf); + buf += HDstrlen(argv[k]) + 1; + } + } + + HDsnprintf(cmdline, sizeof(cmdline), "\n---------\nCommand:"); + b_offset = strlen(cmdline); + for (k = 0; k < count; k++) { + HDsprintf(&cmdline[b_offset], " %s", argv[k]); + b_offset = strlen(cmdline); + } + HDsprintf(&cmdline[b_offset], "\n"); + run_command(count, argv, cmdline, fname); + + mfu_free(argv); + + return 0; +} + +int +MFU_PRED_PRINT(mfu_flist flist, uint64_t idx, void *arg __attribute__((unused))) +{ + const char *name = mfu_flist_file_get_name(flist, idx); + HDprintf("%s\n", name); + return 1; +} + +static void +pred_commit(mfu_pred *p) +{ + mfu_pred *cur = p; + while (cur) { + if (cur->f == MFU_PRED_PRINT || cur->f == MFU_PRED_EXEC) { + break; + } + cur = cur->next; + } +} + +static void +add_executable(int argc, char **argv, char *cmdstring, int *f_index, int f_count __attribute__((unused))) +{ + char cmdline[2048]; + HDsnprintf(cmdline, sizeof(cmdline), "\n---------\nCommand: %s\n", cmdstring); + argv[argc] = NULL; + run_command(argc, argv, cmdline, argv[f_index[0]]); + return; +} + +static int +process_input_file(char *inputname, int myrank, int size) +{ + int index = 0; + char linebuf[PATH_MAX] = { + '\0', + }; + FILE * config = HDfopen(inputname, "r"); + mfu_flist flist1 = NULL; + + if (config == NULL) + return -1; + + flist1 = mfu_flist_new(); + + /* Flag the fact that we're processing an inputfile (script) + * so that we can generate a meaningful logfile name... + */ + processing_inputfile = 1; + + while (HDfgets(linebuf, sizeof(linebuf), config) != NULL) { + const char *delim = " \n"; + char * cmdline = NULL; + char * cmd = NULL; + char * arg = NULL; + char * argv[256]; + int fileindex[256]; + int filecount = 0; + int token = 0; + struct stat statbuf; + + char *eol = strchr(linebuf, '\n'); + if (eol) { + *eol = '\0'; + } + cmdline = HDstrdup(linebuf); + cmd = HDstrtok(linebuf, delim); + if (cmd) { + arg = cmd; + while (arg != NULL) { + char c = arg[0]; + if (token > 0) { + if ((c == '.') || (c == '/')) { + /* 'arg' looks to be a filepath */ + if (stat(arg, &statbuf) == 0) { + mfu_flist_insert_stat(flist1, arg, O_RDONLY, &statbuf); + } + fileindex[filecount++] = token; + } + } + argv[token++] = arg; + arg = strtok(NULL, delim); + } + + if (myrank == (index % size)) { + current_input_index = index; + add_executable(token, argv, cmdline, fileindex, filecount); + } + index++; + } + linebuf[0] = 0; + HDfree(cmdline); + } + + if (output_log_file) { + dh5tool_flist_write_text(output_log_file, flist1); + } + HDfclose(config); + + mfu_flist_free(&flist1); + return 0; +} + +int +main(int argc, char *argv[]) +{ + int i; + int rc = 0; + + char *env_var = NULL; + + /* initialize MPI */ + MPI_Init(&argc, (char ***)&argv); + mfu_init(); + + /* Initialize h5tools lib */ + h5tools_init(); + + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + + /* get our rank and the size of comm_world */ + int rank, ranks; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &ranks); + + /* Assign the static global mpi_rank (for debugging) */ + sg_mpi_rank = rank; + +#if 0 + env_var = HDgetenv("HDF5_H5DWALK_PRINT_CMDLINE"); + if (env_var) { + int enable = HDatoi(env_var); + if (enable) { + + } + } +#endif + /* pointer to mfu_walk_opts */ + mfu_walk_opts_t *walk_opts = mfu_walk_opts_new(); + +#ifdef DAOS_SUPPORT + /* DAOS vars */ + daos_args_t *daos_args = daos_args_new(); +#endif + + int args_byte_length = -1; + int mfu_argc = argc; + char * args_buf = NULL; + char **h5tool_argv = copy_args(argc, argv, &mfu_argc, &args_byte_length); + + char *inputname = NULL; + char *outputname = NULL; + char *sortfields = NULL; + char *distribution = NULL; + + int text = 0; + int h5tool_argc = 0; + + mfu_debug_level = MFU_LOG_WARN; + h5tool_argv[argc] = 0; + + /* The struct option declaration can found in bits/getopt_ext.h + * I've reproduced it here: + * struct option { char * name; int has_arg; int *flag; int val}; + */ + int opt; + int tool_selected = 0; + int tool_args_start = -1; + int last_mfu_arg = 0; + + mfu_pred *pred_head = NULL; + + while (!tool_selected) { + opt = H5_get_option(argc, (const char *const *)argv, s_opts, l_opts); + switch ((char)opt) { + default: + usage(); + h5dwalk_exit(EXIT_FAILURE); + break; + case 'i': + inputname = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + if (inputname) + tool_selected = 1; + break; + case 'o': + outputname = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + if (outputname) { + log_output_in_single_file = 1; + output_log_file = HDstrdup(H5_optarg); + text = 1; /* Format TXT, not HDF5 */ + } + break; + case 'E': + log_errors_in_file = 1; + errlog = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + break; + case 'l': + log_stdout_in_file = 1; + if (H5_optarg) + txtlog = HDstrdup(H5_optarg); + break; + case 'T': + /* We need to stop parsing user options at this point. + * all remaining arguments should be utilized as the + * arguments to the selected HDF5 tools. + * We also want to avoid any misinterpretations if + * HDF5 tool options conflict with the MFU options. + */ + tool_selected = 1; + tool_args_start = H5_optind; + h5tool_argc = argc - mfu_argc; + last_mfu_arg = H5_optind; + /* Don't allow any further parsing of arguments */ + break; + case 'h': + usage(); + h5dwalk_exit(EXIT_SUCCESS); + break; + case '?': + usage(); + h5dwalk_exit(EXIT_SUCCESS); + break; + } + } + + if (inputname != NULL) { + if (tool_selected && (rank == 0)) { + if ((log_output_in_single_file == 0) && (log_stdout_in_file == 0)) + puts("WARNING: When utilizing --input, the only other supported " + "runtime argument is --output or -l"); + } + rc = process_input_file(inputname, rank, ranks); + mfu_finalize(); + h5dwalk_exit(rc); + } + + /**************************************************************/ + /* We might consider doing a tool specific argument checking */ + /* to prevent runtime errors. We would also like to allow */ + /* the same command line interface for parallel invocations */ + /* so that users don't get confused. Effectively, we should */ + /* strip out all MFU related arguments and retain copies of */ + /* everything else to pass into a serial instance of the tool */ + /* */ + /* As we move forward, we might allow the HDF5 tool to be */ + /* queried for an acceptable set set of runtime arguments. */ + /* This could be just a simple string to allow getopt_long */ + /* to be invoked on the remaining command line arguments. */ + /**************************************************************/ + + int *path_indices = NULL; + int numpaths = count_dirpaths(argc, tool_args_start, argv, &path_indices); + + const char **argpaths = NULL; + + /* store src and dest path strings */ + const char *path1 = NULL; + const char *path2 = NULL; + size_t pathlen_total = 0; + + if (numpaths && path_indices) { + argpaths = &argv[path_indices[0]]; + } + /* pointer to mfu_file src and dest objects */ + /* The dst object will only be used for tools which + * accept 2 (or more?) file arguments */ + mfu_file_t *mfu_src_file = NULL; + mfu_file_t *mfu_dst_file = NULL; + + /* first item is source and second is dest */ + mfu_param_path *srcpath = NULL; + mfu_param_path *destpath = NULL; + mfu_param_path *paths = NULL; + + mfu_flist flist1 = NULL; + mfu_flist flist2 = NULL; + + /* allocate structure to define walk options */ + if (use_config_file > 0) { + int count1 = 0, count2 = 0; + for (i = 0; i < use_config_file; i++) { + int index = config_index[i]; + const char *config_file = argv[index]; + if (i == 0) { + flist1 = mfu_flist_new(); + count1 = fill_file_list(flist1, config_file + 1, rank, ranks); + } + else if (i == 1) { + flist2 = mfu_flist_new(); + count2 = fill_file_list(flist2, config_file + 1, rank, ranks); + } + } + if (count1 != count2) { + HDprintf("config files have different file counts: (1) %d and (2) %d\n", count1, count2); + } + } + else if (numpaths > 0) { + + /* allocate space for each path */ + paths = (mfu_param_path *)MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path)); + mfu_src_file = mfu_file_new(); + + /* process each path */ + mfu_param_path_set_all((uint64_t)numpaths, (const char **)argpaths, paths, mfu_src_file, true); + + /* don't allow user to specify input file with walk */ + if (inputname != NULL) { + if (paths) { + mfu_free(&paths); + } + usage(); + h5dwalk_exit(EXIT_FAILURE); + } + } + else { + /* if we're not walking, we must be reading, + * and for that we need a file */ + if (inputname == NULL) { + if (rank == 0) { + MFU_LOG(MFU_LOG_ERR, "Either a or --input is required."); + } + usage(); + h5dwalk_exit(EXIT_FAILURE); + } + } + + if (numpaths > 0) { + flist1 = mfu_flist_new(); + srcpath = &paths[0]; + path1 = srcpath->path; + pathlen_total += strlen(path1); + mfu_flist_walk_param_paths(1, srcpath, walk_opts, flist1, mfu_src_file); + } + if (numpaths > 1) { + flist2 = mfu_flist_new(); + mfu_dst_file = mfu_file_new(); + destpath = &paths[1]; + path2 = destpath->path; + pathlen_total += HDstrlen(path2); + mfu_flist_walk_param_paths(1, destpath, walk_opts, flist2, mfu_dst_file); + } + + if (tool_selected && (args_byte_length > 0)) { + pred_head = mfu_pred_new(); + args_buf = (char *)HDmalloc((size_t)(args_byte_length + pathlen_total)); + } + + /* filter files to only include hdf5 files */ + if (flist1) { + filter_hdf_files(&flist1, NULL, 0, 0); + } + if (flist2) { + filter_hdf_files(&flist2, NULL, 0, 0); + } + + /* if (numpaths > 1) + * In a case where we requeire the list indices of files from multiple + * directories to match, we must utilize a mapping function. + * The question to answer is how does the mapping function work? + * The most probable is a sort function, e.g. + * 1) an alphabet sort? + * 2) sort by file size? + * 3) something else? + */ + if (args_buf != NULL) { + int k = 0; + char *ptr = args_buf + sizeof(int); + *(int *)args_buf = h5tool_argc; + for (i = tool_args_start - 1; i < argc; i++) { + int copy_flist = -1; + if (i == config_index[k]) { + copy_flist = k; + } + else if (path_indices && (i == path_indices[k])) { + copy_flist = k; + } + + /* Maybe copy one of the flist pointers */ + if (copy_flist >= 0) { + /* The '&' indicates that what follows is a pointer */ + *ptr++ = '&'; + /* Select which argument list should be used */ + if (k == 0) { + HDmemcpy(ptr, &flist1, sizeof(void *)); + } + if (k == 1) { + HDmemcpy(ptr, &flist2, sizeof(void *)); + } + ptr += sizeof(mfu_flist *); + k++; + } + else { + HDstrcpy(ptr, argv[i]); + ptr += HDstrlen(argv[i]); + } + *ptr++ = 0; + } + *ptr++ = 0; + + mfu_pred_add(pred_head, MFU_PRED_EXEC, (void *)args_buf); + pred_commit(pred_head); + } + + /* apply predicates to each item in list */ + mfu_flist flist3 = mfu_flist_filter_pred(flist1, pred_head); + + /* print summary statistics of flist */ + mfu_flist_print_summary(flist1); + + /* write data to cache file */ + if (outputname != NULL) { + if (!text) { + if (rank == 0) { + puts("output capture needs to be a text formatted file"); + } + } + else { + dh5tool_flist_write_text(outputname, flist1); + } + } + +#ifdef DAOS_SUPPORT + daos_cleanup(daos_args, mfu_file, NULL); +#endif + + /* free users, groups, and files objects */ + mfu_flist_free(&flist1); + if (flist2) + mfu_flist_free(&flist2); + if (flist3) + mfu_flist_free(&flist3); + + /* free memory allocated for options */ + mfu_free(&distribution); + mfu_free(&sortfields); + mfu_free(&outputname); + mfu_free(&inputname); + + /* free the path parameters */ + mfu_param_path_free_all((uint64_t)numpaths, paths); + + /* free memory allocated to hold params */ + mfu_free(&paths); + + /* free the walk options */ + mfu_walk_opts_delete(&walk_opts); + + /* delete file object */ + mfu_file_delete(&mfu_src_file); + + h5tools_close(); + /* shut down MPI */ + mfu_finalize(); + MPI_Finalize(); + + return rc; +} + +/*------------------------------------------------------------------------- + * Function: h5dwalk_exit + * + * Purpose: close the tools library and exit + * + * Return: none + * + * Programmer: Albert Cheng + * Date: Feb 6, 2005 + * + * Comments: + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +H5_ATTR_NORETURN void +h5dwalk_exit(int status) +{ + int require_finalize = 0; + h5tools_close(); + mfu_finalize(); + + /* Check to see whether we need to call MPI_Finalize */ + MPI_Initialized(&require_finalize); + if (require_finalize) + MPI_Finalize(); + + HDexit(status); +} diff --git a/utils/tools/test/CMakeLists.txt b/utils/tools/test/CMakeLists.txt new file mode 100644 index 0000000..0f5335d --- /dev/null +++ b/utils/tools/test/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_TEST C) + +#-- Add the h5diff tests +if (HDF5_BUILD_PARALLEL_TOOLS) + add_subdirectory (h5dwalk) +endif() + diff --git a/utils/tools/test/Makefile.am b/utils/tools/test/Makefile.am new file mode 100644 index 0000000..88104f2 --- /dev/null +++ b/utils/tools/test/Makefile.am @@ -0,0 +1,32 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Tools HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +if PARALLEL_TOOLS_CONDITIONAL + H5DWALK=h5dwalk +else + H5DWALK= +endif + +CONFIG=ordered + +# All subdirectories +SUBDIRS=$(H5DWALK) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/test/h5dwalk/CMakeLists.txt b/utils/tools/test/h5dwalk/CMakeLists.txt new file mode 100644 index 0000000..5f6c992 --- /dev/null +++ b/utils/tools/test/h5dwalk/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_TEST_H5DWALK) + +if (HDF5_BUILD_PARALLEL_TOOLS) + add_custom_command( + OUTPUT ${HDF5_TOOLS_DIR}/test/demo_destfiles.test + COMMAND bash -c ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh + ARGS ${HDF5_TOOLS_DIR}/test ${CMAKE_BINARY_DIR}/bin + DEPENDS ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh + ) +endif () + +if (HDF5_TEST_TOOLS AND HDF5_TEST_SERIAL) + include (CMakeTests.cmake) +endif () diff --git a/utils/tools/test/h5dwalk/CMakeTests.cmake b/utils/tools/test/h5dwalk/CMakeTests.cmake new file mode 100644 index 0000000..b9e52c5 --- /dev/null +++ b/utils/tools/test/h5dwalk/CMakeTests.cmake @@ -0,0 +1,56 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_TEST resultfile resultcode) + # If using memchecker add tests without using scripts + if (HDF5_ENABLE_USING_MEMCHECKER) + message("Entered ADD_H5_TEST - 0") + add_test (NAME H5DWALK-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ ${ARGN}) + set_tests_properties (H5DWALK-${resultfile} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if ("${resultcode}" STREQUAL "1") + set_tests_properties (H5DWALK-${resultfile} PROPERTIES WILL_FAIL "true") + endif () + else () + # Remove any output file left over from previous test run + add_test ( + NAME H5DWALK-${resultfile} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.h5dwalk" + -D "TEST_LIBRARY_DIRECTORY=${LL_PATH}" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + endif () + endmacro () + + ADD_H5_TEST(help-1 0 -h) diff --git a/utils/tools/test/h5dwalk/Makefile.am b/utils/tools/test/h5dwalk/Makefile.am new file mode 100644 index 0000000..c32dd0f --- /dev/null +++ b/utils/tools/test/h5dwalk/Makefile.am @@ -0,0 +1,43 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src directory +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib + +install-examples: + @echo "Creating demo files" && \ + . copy_demo_files.sh $(top_srcdir)/src + +bin_SCRIPTS:install-examples + +#test script and program +TEST_SCRIPT=testh5dwalk.sh copy_demo_files.sh +check_SCRIPTS=$(TEST_SCRIPT) +SCRIPT_DEPEND=../../h5dwalk/h5dwalk$(EXEEXT) + +# Tell automake to clean h5redeploy script +CLEANFILES= + +# These were generated by configure. Remove them only when distclean. +DISTCLEANFILES=testh5dwalk.sh copy_demo_files.sh + +# All programs rely on hdf5 library and h5tools library +LDADD=$(LIBH5TOOLS) $(LIBHDF5) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/test/h5dwalk/copy_demo_files.sh.in b/utils/tools/test/h5dwalk/copy_demo_files.sh.in new file mode 100644 index 0000000..f20bf43 --- /dev/null +++ b/utils/tools/test/h5dwalk/copy_demo_files.sh.in @@ -0,0 +1,86 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +srcdir=@srcdir@ +TOP_BUILDDIR=.. + +# Determine if backward compatibility options enabled +DEPRECATED_SYMBOLS="yes" + +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +CP='cp' + +THIS_DIR=`pwd` +SRC_TOOLS_DIR=$srcdir/../../../../tools + +nerrors=0 +verbose=yes +exit_code=$EXIT_SUCCESS + + +# Add Testing files into the local testfiles directory:: +TESTDIR=./testfiles +test -d $TESTDIR || mkdir $TESTDIR + +echo "HDF5 \"$THIS_DIR/testfiles/h5diff_basic1.h5\" {" > "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt +echo "FILE_CONTENTS { + group / + group /g1 + dataset /g1/d1 + dataset /g1/d2 + dataset /g1/dset1 + dataset /g1/dset10 + dataset /g1/dset11 + dataset /g1/dset12 + dataset /g1/dset3 + dataset /g1/dset5 + dataset /g1/dset6 + dataset /g1/dset7 + dataset /g1/dset8 + dataset /g1/dset9 + dataset /g1/fp1 + dataset /g1/fp15 + dataset /g1/fp16 + dataset /g1/fp17 + dataset /g1/fp18 + dataset /g1/fp18_COPY + dataset /g1/fp19 + dataset /g1/fp19_COPY + dataset /g1/fp2 + dataset /g1/fp20 + dataset /g1/fp20_COPY + dataset /g1/ld + } +}" >> "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt + +# Create the help-1.txt output file for '-h' validation +echo " +Usage: h5dwalk [options] ... + +Options: + -i, --input - read list from file + -o, --output - write output summary to the named file. + -E, --error - write processed errors to file in text format + -l, --log_text - write individual tool outputs to a file. Logs can be written to an optional named directory. + -T, --tool - name of the HDF5 tool to invoke + -h, --help - print usage + +For more information see https://mpifileutils.readthedocs.io. +" > "$THIS_DIR"/testfiles/help-1.txt + +# Make a copy of the help-1.txt output file for --help validation +$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-1.txt +$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-2.txt +# Make a copy of a simple HDF5 datafile which will be used as input for h5dump -n (see the expected output above) +$CP "$SRC_TOOLS_DIR"/test/h5diff/testfiles/h5diff_basic1.h5 "$THIS_DIR"/testfiles diff --git a/utils/tools/test/h5dwalk/help.h5dwalk b/utils/tools/test/h5dwalk/help.h5dwalk new file mode 100644 index 0000000..986cbba --- /dev/null +++ b/utils/tools/test/h5dwalk/help.h5dwalk @@ -0,0 +1,13 @@ + +Usage: h5dwalk [options] ... + +Options: + -i, --input - read list from file + -o, --output - write output summary to the named file. + -E, --error - write processed errors to file in text format + -l, --log_text - write individual tool outputs to a file. Logs can be written to an optional named directory. + -T, --tool - name of the HDF5 tool to invoke + -h, --help - print usage + +For more information see https://mpifileutils.readthedocs.io. + diff --git a/utils/tools/test/h5dwalk/testh5dwalk.sh.in b/utils/tools/test/h5dwalk/testh5dwalk.sh.in new file mode 100644 index 0000000..4f6dbde --- /dev/null +++ b/utils/tools/test/h5dwalk/testh5dwalk.sh.in @@ -0,0 +1,249 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +# Tests for the h5dwalk tool + +# +# + +srcdir=@srcdir@ + +# Determine which filters are available +USE_FILTER_SZIP="@USE_FILTER_SZIP@" +USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" + + +TESTNAME=h5dwalk + +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +THIS_DIR="`pwd`" +ROOTDIR="`cd ../../../..; pwd`" +cd "$THIS_DIR" +TOP_DIR="$ROOTDIR" + + +H5DWALK=../../h5dwalk/h5dwalk +H5DWALK_BIN="$TOP_DIR/utils/tools/h5dwalk/h5dwalk" + + +H5DUMP="$TOP_DIR/src/h5dump/h5dump" +H5DUMP_BIN="$TOP_DIR/tools/src/h5dump/h5dump" + +RM='rm -rf' +CMP='cmp -s' +DIFF='diff -c' +CP='cp' +DIRNAME='dirname' +LS='ls' +AWK='awk' +WC='wc' + +nerrors=0 +verbose=yes + +export LD_LIBRARY_PATH=@LL_PATH@ + +# source dirs +SRC_TOOLS="$TOP_DIR/tools/test" +SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" + +# testfiles source dirs for tools +SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles" +SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles" +SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles" +SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles" +SRC_H5DWALK_TESTFILES="$SRC_TOOLS/h5dwalk/testfiles" +SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles" + +TESTDIR=./testfiles +test -d $TESTDIR || mkdir $TESTDIR + +echo "SRC_H5DIFF_TESTFILES = $SRC_H5DIFF_TESTFILES" +echo "Creating demo files" +. ./copy_demo_files.sh + + + +CLEAN_TESTFILES_AND_TESTDIR() +{ + echo "cleaning logfiles" + $RM $TESTDIR/*log* +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# +TOOLTEST() { + expect="$TESTDIR/$1" + expect_err="$TESTDIR/`basename $1`.err" + actual="$TESTDIR/`basename $1`.out" + actual_err="$TESTDIR/`basename $1`.out.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + shift + + # Run test. + + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + ) 1> $actual 2> $actual_err + + # save actual and actual_err in case they are needed later. + cp $actual $actual_sav + cp $actual_err $actual_err_sav + + if [ ! -f $expect ]; then + # Compare error files if the expect file doesn't exist. + if $CMP $expect_err $actual_err; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.err) differs from actual result (*.out.err)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect_err $actual_err |sed 's/^/ /' + fi + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav + fi +} + +TOOL_LOGTEST() { + expect="$TESTDIR/`basename $1`.txt" + expect_err="$TESTDIR/`basename $1`.err" + actual="$TESTDIR/`basename $1`.log" + actual_err="$TESTDIR/`basename $1`.out.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + shift + + echo "running logtest" + + # Run test. + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + + ) 1> $actual 2> $actual_err + expect_len="`wc -l < $expect`" + + if [ ! -f $actual ]; then + echo "*FAILED*" + echo " The expected .log file is missing" + echo " Perhaps the test failed to run?" + else + actual_len="`wc -l < $actual`" + if [ $actual_len -eq $expect_len ]; then + echo " PASSED" + else + echo "*FAILED*" + echo " The generated .log file length does not match the expected length. $actual_len != $expected_len" + fi + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav + fi +} + +TOOL_CHK_LOGLEN() { + expect=$1 + shift + + echo "running tool_chk_loglen" + + # Run test. + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + ) + + expect_len="`wc -l < $expect`" + if [ "$expect_len" -gt 0 ]; then + echo " PASSED" + else + echo "*FAILED*" + echo " The generated .log file is empty!." + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $expect + fi +} + + +# Print a "SKIP" message +SKIP() { + TESTING $H5DWALK $@6 + echo " -SKIP-" +} + + + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +TOOLTEST help-1.txt -h +TOOLTEST help-2.txt --help +TOOL_LOGTEST h5diff_basic1.h5_h5dump -l -T $H5DUMP_BIN -n ./h5diff_basic1.h5 +TOOL_CHK_LOGLEN showme-h5dump.log -o `pwd`/showme-h5dump.log -T $H5DUMP_BIN -n `pwd` + + +# +# +# Clean up temporary files/directories +CLEAN_TESTFILES_AND_TESTDIR + +if test $nerrors -eq 0 ; then + echo "All $TESTNAME tests passed." + exit $EXIT_SUCCESS +else + echo "$TESTNAME tests failed with $nerrors errors." + exit $EXIT_FAILURE +fi + diff --git a/utils/vfd_swmr/README.md b/utils/vfd_swmr/README.md index dd19c72..bad1483 100644 --- a/utils/vfd_swmr/README.md +++ b/utils/vfd_swmr/README.md @@ -8,7 +8,7 @@ Where: is the path to the metadata file. Must be on a POSIX file syste is the path of the updater files including the directory. This will typically be in an NFS mounted file system. Options: - -a --skip_aux: Exit if VDS across multiple file is being enabled (to be implented in the future). + -a --skip_aux: Exit if VDS across multiple file is being enabled (to be implemented in the future). -c --vfd_config: Quoted string containing the configuration string for the VFD stack to be used. Default: sec2 -l --log_file: Path to the log file. Default: no log file. -m --md_chksum_path: Path to the file containing the checksum values for testing purpose. -- cgit v0.12