diff options
| author | Jordan Henderson <jhenderson@hdfgroup.org> | 2017-03-20 16:51:32 (GMT) |
|---|---|---|
| committer | Jordan Henderson <jhenderson@hdfgroup.org> | 2017-03-20 16:51:32 (GMT) |
| commit | 3a01afc0b15fc63a0fe6800ae3f8f55fc6bfed0a (patch) | |
| tree | d4e88296864e644aae6b37bc07411cb2d3850c0b | |
| parent | c3663d13d5becf8b3eaed18679f2349bd6ab971a (diff) | |
| parent | b359e8f1a2349d177e70d34560d0a089ce37c4c8 (diff) | |
| download | hdf5-3a01afc0b15fc63a0fe6800ae3f8f55fc6bfed0a.zip hdf5-3a01afc0b15fc63a0fe6800ae3f8f55fc6bfed0a.tar.gz hdf5-3a01afc0b15fc63a0fe6800ae3f8f55fc6bfed0a.tar.bz2 | |
Merge pull request #3 in ~JHENDERSON/hdf5 from develop to feature/parallel_filters
* commit 'b359e8f1a2349d177e70d34560d0a089ce37c4c8':
Description: Removed "#ifndef H5_NO_DEPRECATED_SYMBOLS" in file space tests, because the wrappers only use the latest functions now. Platforms tested: Linux/32 2.6 (jam) Linux/64 (platypus) Darwin (osx1010test)
Purpose: Add new C++ wrappers Description: Because H5Pset_file_space and H5Pget_file_space are deprecated, changed to make wrappers for the new functions instead: H5Ps/get_file_space_strategy H5Ps/get_file_space_page_size
Description: Fixed typos. Platforms tested: Linux/64 (jelly)
Description: Deprecating versions of PropList::setProperty that have arguments that miss "const" Platforms tested: Linux/64 (jelly) Linux/64 (platypus) Darwin (osx1010test)
Description: Miscellaneous clean-up: format and comments Platforms tested: Linux/64 (jelly) Darwin (osx1010test)
Add toolset option to cmake configure
Update cmake pubconf to match autotools and add strtoll checks
Switch from HDatoll to HDstrtoll, for Windows compatibility.
output_filter.sh: Comment added to address HDFFV-8270. The sample ontput in the file's comments are not up-to-date with the scripts in the file that remove output unique to certain systems when running test scripts. This output doesn't match expected output files for the tests, causing them to fail. Ther output_filter.sh file removes such output. Currently we don't have access to these systems to update the comments.
Correct double sourcedir name on copied datafiles.
HDFFV-10138 Merge app framework to examples
Final merge of page buffering branch to develop
Purpose: Add new C++ wrappers Description: Added wrappers for H5Iis_valid, H5Ps/get_nlinks, H5Tget_create_plist, H5Oopen, H5Oclose and H5Pset_virtual
Bring changes to I/O parameters from page_buffering branch.
Merge in reentrency changes to "make space in cache" from page_buffering branch.
Minor cleanups and bring over "prefetched dirty" fixes for entries loaded from a cache image.
Remove some usage of "prefetched_dirty" flag (which hasn't been merged from the page_buffering branch yet. Also, bring over improvements to flush candidate entries for parallel code.
Align with incoming page buffering changes: minor cleanups, centralize removing entries from collective metadata read list
269 files changed, 24446 insertions, 7180 deletions
diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index 3f0f7ca..37f3f70 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -174,13 +174,25 @@ if (HDF5_PACK_EXAMPLES) USE_SOURCE_PERMISSIONS COMPONENT hdfdocuments ) - install ( - FILES - ${HDF5_SOURCE_DIR}/release_docs/USING_CMake_Examples.txt - DESTINATION ${HDF5_INSTALL_DATA_DIR} - COMPONENT hdfdocuments - ) endif () + install ( + FILES + ${HDF5_SOURCE_DIR}/release_docs/USING_CMake_Examples.txt + DESTINATION ${HDF5_INSTALL_DATA_DIR} + COMPONENT hdfdocuments + ) + install ( + FILES + ${HDF_RESOURCES_DIR}/CTestScript.cmake + DESTINATION ${HDF5_INSTALL_DATA_DIR} + COMPONENT hdfdocuments + ) + install ( + FILES + ${HDF_RESOURCES_DIR}/HDF5_Examples_options.cmake + DESTINATION ${HDF5_INSTALL_DATA_DIR} + COMPONENT hdfdocuments + ) endif () #----------------------------------------------------------------------------- @@ -579,6 +579,7 @@ ./src/H5Fmpi.c ./src/H5Fquery.c ./src/H5Fsfile.c +./src/H5Fspace.c ./src/H5Fsuper.c ./src/H5Fsuper_cache.c ./src/H5Fpkg.h @@ -793,6 +794,10 @@ ./src/H5Ppublic.h ./src/H5Pstrcpl.c ./src/H5Ptest.c +./src/H5PB.c +./src/H5PBmodule.h +./src/H5PBpkg.h +./src/H5PBprivate.h ./src/H5PL.c ./src/H5PLmodule.h ./src/H5PLprivate.h @@ -895,6 +900,7 @@ ./test/SWMR_UseCase_UG.txt ./test/accum.c ./test/accum_swmr_reader.c +./test/aggr.h5 ./test/app_ref.c ./test/atomic_reader.c ./test/atomic_writer.c @@ -957,6 +963,8 @@ ./test/flush1.c ./test/flush2.c ./test/flushrefresh.c +./test/fsm_aggr_nopersist.h5 +./test/fsm_aggr_persist.h5 ./test/genall5.c ./test/genall5.h ./test/gen_bad_ohdr.c @@ -966,7 +974,6 @@ ./test/gen_deflate.c ./test/gen_file_image.c ./test/gen_filespace.c -./test/gen_idx.c ./test/gen_mergemsg.c ./test/gen_new_array.c ./test/gen_new_fill.c @@ -986,6 +993,12 @@ ./test/getname.c ./test/gheap.c ./test/group_old.h5 +./test/h5fc_ext1_f.h5 +./test/h5fc_ext1_i.h5 +./test/h5fc_ext2_if.h5 +./test/h5fc_ext2_sf.h5 +./test/h5fc_ext3_isf.h5 +./test/h5fc_ext_none.h5 ./test/h5test.c ./test/h5test.h ./test/hyperslab.c @@ -1002,9 +1015,13 @@ ./test/multi_file_v16-r.h5 ./test/multi_file_v16-s.h5 ./test/noencoder.h5 +./test/none.h5 ./test/ntypes.c ./test/ohdr.c ./test/objcopy.c +./test/page_buffer.c +./test/paged_nopersist.h5 +./test/paged_persist.h5 ./test/plugin.c ./test/reserved.c ./test/pool.c @@ -1203,6 +1220,7 @@ ./testpar/COPYING ./testpar/Makefile.am ./testpar/t_cache.c +./testpar/t_cache_image.c ./testpar/t_chunk_alloc.c ./testpar/t_coll_chunk.c ./testpar/t_dset.c @@ -1448,6 +1466,14 @@ ./tools/test/misc/testh5mkgrp.sh.in ./tools/test/misc/testh5repart.sh.in ./tools/test/misc/talign.c +./tools/test/misc/testfiles/h5clear_missing_file.ddl +./tools/test/misc/testfiles/h5clear_no_mdc_image.ddl +./tools/test/misc/testfiles/h5clear_open_fail.ddl +./tools/test/misc/testfiles/h5clear_usage.ddl +./tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 +./tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 +./tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 +./tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 ./tools/test/misc/testfiles/h5mkgrp_help.txt ./tools/test/misc/testfiles/h5mkgrp_version.txt.in ./tools/test/misc/h5perf_gentest.c @@ -1637,7 +1663,6 @@ ./tools/testfiles/tdset.h5 ./tools/testfiles/tdset2.h5 ./tools/testfiles/tdset_idx.ls -./tools/testfiles/tdset_idx.ddl ./tools/testfiles/tdset_idx.h5 ./tools/testfiles/tempty.ddl ./tools/testfiles/tempty.h5 @@ -2270,7 +2295,6 @@ ./tools/test/h5diff/testfiles/h5diff_457.txt ./tools/test/h5diff/testfiles/h5diff_458.txt ./tools/test/h5diff/testfiles/h5diff_459.txt -./tools/test/h5diff/testfiles/h5diff_idx.txt ./tools/test/h5diff/testfiles/h5diff_465.txt ./tools/test/h5diff/testfiles/h5diff_466.txt ./tools/test/h5diff/testfiles/h5diff_467.txt @@ -2319,8 +2343,6 @@ ./tools/test/h5diff/testfiles/h5diff_basic2.h5 ./tools/test/h5diff/testfiles/h5diff_dset1.h5 ./tools/test/h5diff/testfiles/h5diff_dset2.h5 -./tools/test/h5diff/testfiles/h5diff_dset_idx1.h5 -./tools/test/h5diff/testfiles/h5diff_dset_idx2.h5 ./tools/test/h5diff/testfiles/h5diff_dtypes.h5 ./tools/test/h5diff/testfiles/h5diff_empty.h5 ./tools/test/h5diff/testfiles/h5diff_hyper1.h5 @@ -3123,12 +3145,16 @@ ./tools/test/h5stat/CMakeTests.cmake ./tools/src/misc/CMakeLists.txt ./tools/test/misc/CMakeLists.txt -./tools/test/misc/CMakeTests.cmake +./tools/test/misc/CMakeTestsClear.cmake +./tools/test/misc/CMakeTestsMkgrp.cmake +./tools/test/misc/CMakeTestsRepart.cmake ./tools/test/misc/vds/CMakeLists.txt ./tools/test/perform/CMakeLists.txt ./tools/test/perform/CMakeTests.cmake # CMake-specific User Scripts +./config/cmake/CTestScript.cmake +./config/cmake/HDF5_Examples_options.cmake ./config/cmake/scripts/CTestScript.cmake ./config/cmake/scripts/HDF5config.cmake ./config/cmake/scripts/HDF5options.cmake diff --git a/bin/output_filter.sh b/bin/output_filter.sh index 58f1c48..35af819 100644 --- a/bin/output_filter.sh +++ b/bin/output_filter.sh @@ -17,6 +17,21 @@ # Created Date: 2011/5/3 +# Comment added to address HDFFV-8270: +# As I understand it, the purpose of this file is to remove extraneous messages +# that appear in stdout and stderr on some machines that have been tested outside +# of the HDF Group realm. The purpose of this script is to filter those +# extraneous messages from stdout and stderr so that when the output files are +# compared to the expected output, the extra messages will not cause failures in +# the tests. The system messages in the comments below are out of date, meaning +# I suppose that while the script code to filter messages on the system was +# correct correct when last used, the output in the comments doesn't match the +# script code that follows. I don't currently have access to any of these +# systems to see the current output and the effect of the script code. If using +# this file in the future, please update the comments to match the scripts in use. +# Larry Knox 2017/3/15 + + # Some systems will dump some messages to stdout for various reasons. # Remove them from the stdout result file. # $1 is the file name of the file to be filtered. @@ -49,9 +49,11 @@ $Source = ""; "H5E_error_t" => "Ee", "H5E_type_t" => "Et", "H5F_close_degree_t" => "Fd", + "H5F_fspace_strategy_t" => "Ff", "H5F_file_space_type_t" => "Ff", "H5F_mem_t" => "Fm", "H5F_scope_t" => "Fs", + "H5F_fspace_type_t" => "Ft", "H5F_libver_t" => "Fv", "H5G_obj_t" => "Go", "H5G_stat_t" => "Gs", diff --git a/c++/src/H5AbstractDs.h b/c++/src/H5AbstractDs.h index eaa9d14..f0859c8 100644 --- a/c++/src/H5AbstractDs.h +++ b/c++/src/H5AbstractDs.h @@ -91,6 +91,8 @@ class H5_DLLCPP AbstractDs { private: // This member function is implemented by DataSet and Attribute - pure virtual. virtual hid_t p_get_type() const = 0; -}; -} + +}; // end of AbstractDs +} // namespace H5 + #endif // __AbstractDs_H diff --git a/c++/src/H5ArrayType.h b/c++/src/H5ArrayType.h index 15b6bb3..dd9db51 100644 --- a/c++/src/H5ArrayType.h +++ b/c++/src/H5ArrayType.h @@ -60,6 +60,8 @@ class H5_DLLCPP ArrayType : public DataType { // Default constructor ArrayType(); -}; -} + +}; // end of ArrayType +} // namespace H5 + #endif // __H5ArrayType_H diff --git a/c++/src/H5AtomType.h b/c++/src/H5AtomType.h index a1e0262..f4c70fd 100644 --- a/c++/src/H5AtomType.h +++ b/c++/src/H5AtomType.h @@ -78,6 +78,8 @@ class H5_DLLCPP AtomType : public DataType { // Constructor that takes an existing id AtomType(const hid_t existing_id); #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} + +}; // end of AtomType +} // namespace H5 + #endif // __H5AtomType_H diff --git a/c++/src/H5Attribute.cpp b/c++/src/H5Attribute.cpp index 6701f6e..c506906 100644 --- a/c++/src/H5Attribute.cpp +++ b/c++/src/H5Attribute.cpp @@ -1,7 +1,7 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * * Copyright by the Board of Trustees of the University of Illinois. * -/* All rights reserved. * + * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * diff --git a/c++/src/H5Attribute.h b/c++/src/H5Attribute.h index 0243cd9..6d6df2c 100644 --- a/c++/src/H5Attribute.h +++ b/c++/src/H5Attribute.h @@ -101,6 +101,7 @@ class H5_DLLCPP Attribute : public AbstractDs, public H5Location { // Friend function to set Attribute id. For library use only. friend void f_Attribute_setId(Attribute* attr, hid_t new_id); -}; -} +}; // end of Attribute +} // namespace H5 + #endif // __H5Attribute_H diff --git a/c++/src/H5CommonFG.cpp b/c++/src/H5CommonFG.cpp index 5c01055..f3390aa 100644 --- a/c++/src/H5CommonFG.cpp +++ b/c++/src/H5CommonFG.cpp @@ -381,4 +381,4 @@ void f_DataSet_setId(DataSet* dset, hid_t new_id) #endif // DOXYGEN_SHOULD_SKIP_THIS -} +} // end namespace diff --git a/c++/src/H5CommonFG.h b/c++/src/H5CommonFG.h index 588d49a..0c7b799 100644 --- a/c++/src/H5CommonFG.h +++ b/c++/src/H5CommonFG.h @@ -84,10 +84,8 @@ class H5_DLLCPP CommonFG { #endif // DOXYGEN_SHOULD_SKIP_THIS -}; // end of CommonFG declaration +}; // end of CommonFG +} // namespace H5 -#ifndef H5_NO_NAMESPACE -} -#endif #endif // __CommonFG_H diff --git a/c++/src/H5CompType.h b/c++/src/H5CompType.h index 222044d..042b59f 100644 --- a/c++/src/H5CompType.h +++ b/c++/src/H5CompType.h @@ -115,6 +115,8 @@ class H5_DLLCPP CompType : public DataType { // Contains common code that is used by the member functions // getMemberXxxType hid_t p_get_member_type(unsigned member_num) const; -}; -} + +}; // end of CompType +} // namespace H5 + #endif // __H5CompType_H diff --git a/c++/src/H5DataSet.h b/c++/src/H5DataSet.h index 93f9782..f9e29ff 100644 --- a/c++/src/H5DataSet.h +++ b/c++/src/H5DataSet.h @@ -130,6 +130,7 @@ class H5_DLLCPP DataSet : public H5Object, public AbstractDs { // Friend function to set DataSet id. For library use only. friend void f_DataSet_setId(DataSet* dset, hid_t new_id); -}; -} +}; // end of DataSet +} // namespace H5 + #endif // __H5DataSet_H diff --git a/c++/src/H5DataSpace.h b/c++/src/H5DataSpace.h index 969e146..281e865 100644 --- a/c++/src/H5DataSpace.h +++ b/c++/src/H5DataSpace.h @@ -149,6 +149,8 @@ class H5_DLLCPP DataSpace : public IdComponent { friend void f_DataSpace_setId(DataSpace *dspace, hid_t new_id); #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} + +}; // end of DataSpace +} // namespace H5 + #endif // __H5DataSpace_H diff --git a/c++/src/H5DataType.cpp b/c++/src/H5DataType.cpp index ae48d16..c88d6eb 100644 --- a/c++/src/H5DataType.cpp +++ b/c++/src/H5DataType.cpp @@ -744,6 +744,31 @@ bool DataType::isVariableStr() const } //-------------------------------------------------------------------------- +// Function: DataType::getCreatePlist +///\brief Returns a copy of the property list, which is for datatype +/// creation. +///\return A property list object +///\exception H5::DataTypeIException +// Programmer Binh-Minh Ribler - March, 2017 +// Description +// Currently, there is no datatype creation property list class +// in the C++ API because there is no associated functionality. +//-------------------------------------------------------------------------- +PropList DataType::getCreatePlist() const +{ + hid_t create_plist_id = H5Tget_create_plist(id); + if (create_plist_id < 0) + { + throw DataTypeIException(inMemFunc("getCreatePlist"), + "H5Tget_create_plist returns negative value"); + } + // create and return the DSetCreatPropList object + PropList create_plist; + f_PropList_setId(&create_plist, create_plist_id); + return(create_plist); +} + +//-------------------------------------------------------------------------- // Function: DataType::getId ///\brief Get the id of this datatype ///\return Datatype identifier @@ -836,4 +861,5 @@ DataType::~DataType() cerr << inMemFunc("~DataType - ") << close_error.getDetailMsg() << endl; } } + } // end namespace diff --git a/c++/src/H5DataType.h b/c++/src/H5DataType.h index fd9c17d..c2ea2a3 100644 --- a/c++/src/H5DataType.h +++ b/c++/src/H5DataType.h @@ -113,6 +113,9 @@ class H5_DLLCPP DataType : public H5Object { // Checks whether this datatype is a variable-length string. bool isVariableStr() const; + // Returns a copy of the creation property list of a datatype. + PropList getCreatePlist() const; + ///\brief Returns this class name. virtual H5std_string fromClass () const { return("DataType"); } @@ -152,6 +155,8 @@ class H5_DLLCPP DataType : public H5Object { friend void f_DataType_setId(DataType* dtype, hid_t new_id); void p_commit(hid_t loc_id, const char* name); -}; -} + +}; // end of DataType +} // namespace H5 + #endif // __H5DataType_H diff --git a/c++/src/H5DcreatProp.cpp b/c++/src/H5DcreatProp.cpp index 86d4d9c..e1be3c4 100644 --- a/c++/src/H5DcreatProp.cpp +++ b/c++/src/H5DcreatProp.cpp @@ -18,6 +18,7 @@ #include "H5Include.h" #include "H5Exception.h" #include "H5IdComponent.h" +#include "H5DataSpace.h" #include "H5PropList.h" #include "H5OcreatProp.h" #include "H5DcreatProp.h" @@ -740,6 +741,55 @@ void DSetCreatPropList::getExternal(unsigned idx, size_t name_size, char* name, } //-------------------------------------------------------------------------- +// Function: DSetCreatPropList::setVirtual +///\brief Maps elements of a virtual dataset to elements of the source +/// dataset. +///\param vspace - IN: Dataspace the virtual dataset, possibly an +/// unlimited selection +///\param src_fname - IN: Name of the HDF5 file where the source dataset +/// is located (\a char*) +///\param src_fname - IN: Path to the dataset in the file specified by +/// \a src_file_name (\a char*) +///\param sspace - IN: Dataspace with a selection applied, possibly +/// an unlimited selection +///\exception H5::PropListIException +///\par Description +/// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetVirtual +// Programmer Binh-Minh Ribler - Mar, 2017 +//-------------------------------------------------------------------------- +void DSetCreatPropList::setVirtual(const DataSpace& vspace, const char *src_fname, const char *src_dsname, const DataSpace& sspace) const +{ + herr_t ret_value = H5Pset_virtual(id, vspace.getId(), src_fname, src_dsname, sspace.getId()); + if (ret_value < 0) + { + throw PropListIException("DSetCreatPropList::setVirtual", + "H5Pset_virtual failed"); + } +} + +//-------------------------------------------------------------------------- +// Function: DSetCreatPropList::setVirtual +///\brief Maps elements of a virtual dataset to elements of the source +/// dataset. +///\param vspace - IN: Dataspace the virtual dataset, possibly an +/// unlimited selection +///\param src_fname - IN: Name of the HDF5 file where the source dataset +/// is located (\a H5std_string) +///\param src_fname - IN: Path to the dataset in the file specified by +/// \a src_file_name (\a H5std_string) +///\param sspace - IN: Dataspace with a selection applied, possibly +/// an unlimited selection +///\exception H5::PropListIException +///\par Description +/// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetVirtual +// Programmer Binh-Minh Ribler - Mar, 2017 +//-------------------------------------------------------------------------- +void DSetCreatPropList::setVirtual(const DataSpace& vspace, const H5std_string src_fname, const H5std_string src_dsname, const DataSpace& sspace) const +{ + setVirtual(vspace, src_fname.c_str(), src_dsname.c_str(), sspace); +} + +//-------------------------------------------------------------------------- // Function: DSetCreatPropList destructor ///\brief Noop destructor. // Programmer Binh-Minh Ribler - 2000 diff --git a/c++/src/H5DcreatProp.h b/c++/src/H5DcreatProp.h index b536709..f0475eb 100644 --- a/c++/src/H5DcreatProp.h +++ b/c++/src/H5DcreatProp.h @@ -20,6 +20,7 @@ namespace H5 { class DataType; +class DataSpace; /*! \class DSetCreatPropList \brief Class DSetCreatPropList inherits from ObjCreatPropList and provides @@ -116,6 +117,10 @@ class H5_DLLCPP DSetCreatPropList : public ObjCreatPropList { // Sets N-bit compression method. void setNbit() const; + // Maps elements of a virtual dataset to elements of the source dataset. + void setVirtual(const DataSpace& vspace, const char *src_fname, const char *src_dsname, const DataSpace& sspace) const; + void setVirtual(const DataSpace& vspace, const H5std_string src_fname, const H5std_string src_dsname, const DataSpace& sspace) const; + ///\brief Returns this class name. virtual H5std_string fromClass () const { return("DSetCreatPropList"); } @@ -141,6 +146,8 @@ class H5_DLLCPP DSetCreatPropList : public ObjCreatPropList { static DSetCreatPropList* getConstant(); #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} + +}; // end of DSetCreatPropList +} // namespace H5 + #endif // __H5DSCreatPropList_H diff --git a/c++/src/H5DxferProp.h b/c++/src/H5DxferProp.h index f1c363f..39d3ba8 100644 --- a/c++/src/H5DxferProp.h +++ b/c++/src/H5DxferProp.h @@ -127,6 +127,8 @@ class H5_DLLCPP DSetMemXferPropList : public PropList { static DSetMemXferPropList* getConstant(); #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} + +}; // end of DSetMemXferPropList +} // namespace H5 + #endif // __H5DSetMemXferPropList_H diff --git a/c++/src/H5EnumType.h b/c++/src/H5EnumType.h index 2fbe2cd..cd93eb5 100644 --- a/c++/src/H5EnumType.h +++ b/c++/src/H5EnumType.h @@ -78,6 +78,8 @@ class H5_DLLCPP EnumType : public DataType { EnumType(const EnumType& original); virtual ~EnumType(); -}; -} + +}; // end of EnumType +} // namespace H5 + #endif // __H5EnumType_H diff --git a/c++/src/H5Exception.h b/c++/src/H5Exception.h index 1f85711..2e68cf0 100644 --- a/c++/src/H5Exception.h +++ b/c++/src/H5Exception.h @@ -174,8 +174,8 @@ class H5_DLLCPP IdComponentException : public Exception { IdComponentException(const H5std_string& func_name, const H5std_string& message = DEFAULT_MSG); IdComponentException(); virtual ~IdComponentException() throw(); -}; -} +}; // end of IdComponentException +} // namespace H5 #endif // __H5Exception_H diff --git a/c++/src/H5FaccProp.h b/c++/src/H5FaccProp.h index ae7c7f9..8e488e5 100644 --- a/c++/src/H5FaccProp.h +++ b/c++/src/H5FaccProp.h @@ -117,6 +117,12 @@ class H5_DLLCPP FileAccPropList : public PropList { // Returns the degree for the file close behavior. H5F_close_degree_t getFcloseDegree() const; + // Sets file access property list to use the H5FD_DIRECT driver. + void setFileAccDirect(size_t boundary, size_t block_size, size_t cbuf_size) const; + + // Retrieves information about the direct file access property list. + void getFileAccDirect(size_t &boundary, size_t &block_size, size_t &cbuf_size) const; + // Sets garbage collecting references flag. void setGcReferences(unsigned gc_ref = 0) const; @@ -156,6 +162,7 @@ class H5_DLLCPP FileAccPropList : public PropList { #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} +}; // end of FileAccPropList +} // namespace H5 + #endif // __H5FileAccPropList_H diff --git a/c++/src/H5FcreatProp.cpp b/c++/src/H5FcreatProp.cpp index c490f26..893db64 100644 --- a/c++/src/H5FcreatProp.cpp +++ b/c++/src/H5FcreatProp.cpp @@ -295,7 +295,7 @@ unsigned FileCreatPropList::getIstorek() const } //-------------------------------------------------------------------------- -// Function: FileCreatPropList::setFileSpace +// Function: FileCreatPropList::setFileSpaceStrategy ///\brief Sets the strategy and the threshold value that the library /// will employ in managing file space. ///\param strategy - IN: Strategy for file space management @@ -307,62 +307,71 @@ unsigned FileCreatPropList::getIstorek() const /// changed and the existing strategy will be retained. /// If the given threshold value is zero, the property will not be /// changed and the existing threshold will be retained. -/// Valid values of \a libver_low are as follows: -/// \li \c H5F_FILE_SPACE_ALL (Default) -/// \li \c H5F_FILE_SPACE_ALL_PERSIST -/// \li \c H5F_FILE_SPACE_AGGR_VFD -/// \li \c H5F_FILE_SPACE_VFD /// For information, please see the C layer Reference Manual at: /// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetFileSpace // Programmer Binh-Minh Ribler - Feb, 2017 //-------------------------------------------------------------------------- -void FileCreatPropList::setFileSpace(H5F_file_space_type_t strategy, hsize_t threshold) const +void FileCreatPropList::setFileSpaceStrategy(H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold) const { - herr_t ret_value = H5Pset_file_space(id, strategy, threshold); + herr_t ret_value = H5Pset_file_space_strategy(id, strategy, persist, threshold); if (ret_value < 0) { - throw PropListIException("FileCreatPropList::setFileSpace", - "H5Pset_file_space failed"); + throw PropListIException("FileCreatPropList::setFileSpaceStrategy", + "H5Pset_file_space_strategy failed"); } } //-------------------------------------------------------------------------- // Function: FileCreatPropList::getFileSpaceStrategy -///\brief Returns the strategy that the library uses in managing file space. -///\return The strategy value +///\brief Retrieves the strategy, persist, and threshold that the library +/// uses in managing file space. ///\exception H5::PropListIException // Programmer Binh-Minh Ribler - Feb, 2017 //-------------------------------------------------------------------------- -H5F_file_space_type_t FileCreatPropList::getFileSpaceStrategy() const +void FileCreatPropList::getFileSpaceStrategy(H5F_fspace_strategy_t& strategy, hbool_t& persist, hsize_t& threshold) const { - H5F_file_space_type_t strategy = H5F_FILE_SPACE_ALL; - herr_t ret_value = H5Pget_file_space(id, &strategy, NULL); + herr_t ret_value = H5Pget_file_space_strategy(id, &strategy, &persist, &threshold); if (ret_value < 0) { throw PropListIException("FileCreatPropList::getFileSpaceStrategy", - "H5Pget_file_space for strategy failed"); + "H5Pget_file_space_strategy failed"); } - return(strategy); } //-------------------------------------------------------------------------- -// Function: FileCreatPropList::getFileSpaceThreshold -///\brief Returns the threshold value that the library uses in tracking -/// free space sections. -///\return The threshold value +// Function: FileCreatPropList::setFileSpacePagesize +///\brief Sets the file space page size for paged aggregation. ///\exception H5::PropListIException // Programmer Binh-Minh Ribler - Feb, 2017 //-------------------------------------------------------------------------- -hsize_t FileCreatPropList::getFileSpaceThreshold() const +void FileCreatPropList::setFileSpacePagesize(hsize_t fsp_psize) const { - hsize_t threshold = 0; - herr_t ret_value = H5Pget_file_space(id, NULL, &threshold); + herr_t ret_value = H5Pset_file_space_page_size(id, fsp_psize); if (ret_value < 0) { - throw PropListIException("FileCreatPropList::getFileSpaceThreshold", - "H5Pget_file_space for threshold failed"); + throw PropListIException("FileCreatPropList::setFileSpacePagesize", + "H5Pset_file_space_page_size failed"); } - return(threshold); +} + +//-------------------------------------------------------------------------- +// Function: FileCreatPropList::getFileSpacePagesize +///\brief Returns the file space page size for aggregating small +/// metadata or raw data. +///\return File space page size +///\exception H5::PropListIException +// Programmer Binh-Minh Ribler - Feb, 2017 +//-------------------------------------------------------------------------- +hsize_t FileCreatPropList::getFileSpacePagesize() const +{ + hsize_t fsp_psize = 0; + herr_t ret_value = H5Pget_file_space_page_size(id, &fsp_psize); + if (ret_value < 0) + { + throw PropListIException("FileCreatPropList::getFileSpacePagesize", + "H5Pget_file_space_page_size failed"); + } + return(fsp_psize); } //-------------------------------------------------------------------------- diff --git a/c++/src/H5FcreatProp.h b/c++/src/H5FcreatProp.h index 1bbaabf..6beac02 100644 --- a/c++/src/H5FcreatProp.h +++ b/c++/src/H5FcreatProp.h @@ -67,14 +67,17 @@ class H5_DLLCPP FileCreatPropList : public PropList { // Sets the strategy and the threshold value that the library will // will employ in managing file space. - void setFileSpace(H5F_file_space_type_t strategy, hsize_t threshold) const; + void setFileSpaceStrategy(H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold) const; // Returns the strategy that the library uses in managing file space. - H5F_file_space_type_t getFileSpaceStrategy() const; + void getFileSpaceStrategy(H5F_fspace_strategy_t& strategy, hbool_t& persist, hsize_t& threshold) const; + + // Sets the file space page size for paged aggregation. + void setFileSpacePagesize(hsize_t fsp_psize) const; // Returns the threshold value that the library uses in tracking free // space sections. - hsize_t getFileSpaceThreshold() const; + hsize_t getFileSpacePagesize() const; ///\brief Returns this class name. virtual H5std_string fromClass() const { return("FileCreatPropList"); } @@ -102,6 +105,7 @@ class H5_DLLCPP FileCreatPropList : public PropList { #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} +}; // end of FileCreatPropList +} // namespace H5 + #endif // __H5FileCreatPropList_H diff --git a/c++/src/H5File.cpp b/c++/src/H5File.cpp index cdb5837..bea4612 100644 --- a/c++/src/H5File.cpp +++ b/c++/src/H5File.cpp @@ -45,7 +45,7 @@ namespace H5 { using std::endl; //-------------------------------------------------------------------------- -// Function H5File default constructor +// Function H5File default constructor ///\brief Default constructor: creates a stub H5File object. // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- @@ -64,30 +64,30 @@ H5File::H5File() : Group(), id(H5I_INVALID_HID) {} ///\par Description /// Valid values of \a flags include: /// \li \c H5F_ACC_TRUNC - Truncate file, if it already exists, -/// erasing all data previously stored in -/// the file. +/// erasing all data previously stored in +/// the file. /// \li \c H5F_ACC_EXCL - Fail if file already exists. -/// \c H5F_ACC_TRUNC and \c H5F_ACC_EXCL are mutually exclusive +/// \c H5F_ACC_TRUNC and \c H5F_ACC_EXCL are mutually exclusive /// \li \c H5F_ACC_RDONLY - Open file as read-only, if it already -/// exists, and fail, otherwise +/// exists, and fail, otherwise /// \li \c H5F_ACC_RDWR - Open file for read/write, if it already -/// exists, and fail, otherwise +/// exists, and fail, otherwise ///\par /// For info on file creation in the case of an already-open file, /// please refer to the \b Special \b case section in the C layer /// Reference Manual at: /// http://www.hdfgroup.org/HDF5/doc/RM/RM_H5F.html#File-Create -// Notes With a PGI compiler (~2012-2013), the exception thrown by p_get_file -// could not be caught in the applications. Added try block here -// to catch then re-throw it. -BMR 2013/03/21 +// Notes With a PGI compiler (~2012-2013,) the exception thrown by +// p_get_file could not be caught in the applications. Added try +// block here to catch then re-throw it. -BMR 2013/03/21 // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- H5File::H5File(const char* name, unsigned int flags, const FileCreatPropList& create_plist, const FileAccPropList& access_plist) : Group(), id(H5I_INVALID_HID) { try { - p_get_file(name, flags, create_plist, access_plist); + p_get_file(name, flags, create_plist, access_plist); } catch (FileIException& open_file) { - throw open_file; + throw open_file; } } @@ -102,17 +102,17 @@ H5File::H5File(const char* name, unsigned int flags, const FileCreatPropList& cr /// FileCreatPropList::DEFAULT ///\param access_plist - IN: File access property list. Default to /// FileAccPropList::DEFAULT -// Notes With a PGI compiler (~2012-2013), the exception thrown by p_get_file -// could not be caught in the applications. Added try block here -// to catch then re-throw it. -BMR 2013/03/21 +// Notes With a PGI compiler (~2012-2013,) the exception thrown by +// p_get_file could not be caught in the applications. Added try +// block here to catch then re-throw it. -BMR 2013/03/21 // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- H5File::H5File(const H5std_string& name, unsigned int flags, const FileCreatPropList& create_plist, const FileAccPropList& access_plist) : Group(), id(H5I_INVALID_HID) { try { - p_get_file(name.c_str(), flags, create_plist, access_plist); + p_get_file(name.c_str(), flags, create_plist, access_plist); } catch (FileIException& open_file) { - throw open_file; + throw open_file; } } @@ -131,23 +131,23 @@ void H5File::p_get_file(const char* name, unsigned int flags, const FileCreatPro // create the file. if (flags & (H5F_ACC_EXCL|H5F_ACC_TRUNC)) { - hid_t create_plist_id = create_plist.getId(); - hid_t access_plist_id = access_plist.getId(); - id = H5Fcreate(name, flags, create_plist_id, access_plist_id); - if (id < 0) // throw an exception when open/create fail - { - throw FileIException("H5File constructor", "H5Fcreate failed"); - } + hid_t create_plist_id = create_plist.getId(); + hid_t access_plist_id = access_plist.getId(); + id = H5Fcreate(name, flags, create_plist_id, access_plist_id); + if (id < 0) // throw an exception when open/create fail + { + throw FileIException("H5File constructor", "H5Fcreate failed"); + } } // Open the file if none of the bits above are set. else { - hid_t access_plist_id = access_plist.getId(); - id = H5Fopen(name, flags, access_plist_id); - if (id < 0) // throw an exception when open/create fail - { - throw FileIException("H5File constructor", "H5Fopen failed"); - } + hid_t access_plist_id = access_plist.getId(); + id = H5Fopen(name, flags, access_plist_id); + if (id < 0) // throw an exception when open/create fail + { + throw FileIException("H5File constructor", "H5Fopen failed"); + } } } @@ -157,7 +157,7 @@ void H5File::p_get_file(const char* name, unsigned int flags, const FileCreatPro ///\param existing_id - IN: Id of an existing file // Programmer Binh-Minh Ribler - 2015 // Description -// Mar 29, 2015 +// Mar 29, 2015 // Added in responding to a request from user Jason Newton. // However, it is not recommended to use the private member "id" // in applications. Unlike other situations, where similar @@ -230,9 +230,9 @@ bool H5File::isHdf5(const H5std_string& name) ///\par Description /// Valid values of \a flags include: /// H5F_ACC_RDWR: Open with read/write access. If the file is -/// currently open for read-only access then it -/// will be reopened. Absence of this flag -/// implies read-only access. +/// currently open for read-only access then it +/// will be reopened. Absence of this flag +/// implies read-only access. /// /// H5F_ACC_RDONLY: Open with read only access. - default /// @@ -251,7 +251,7 @@ void H5File::openFile(const char* name, unsigned int flags, const FileAccPropLis id = H5Fopen (name, flags, access_plist_id); if (id < 0) // throw an exception when open fails { - throw FileIException("H5File::openFile", "H5Fopen failed"); + throw FileIException("H5File::openFile", "H5Fopen failed"); } } @@ -397,14 +397,14 @@ hssize_t H5File::getFreeSpace() const ///\exception H5::FileIException ///\par Description /// The valid values for \a types include: -/// \li \c H5F_OBJ_FILE - Files only -/// \li \c H5F_OBJ_DATASET - Datasets only -/// \li \c H5F_OBJ_GROUP - Groups only -/// \li \c H5F_OBJ_DATATYPE - Named datatypes only -/// \li \c H5F_OBJ_ATTR - Attributes only +/// \li \c H5F_OBJ_FILE - Files only +/// \li \c H5F_OBJ_DATASET - Datasets only +/// \li \c H5F_OBJ_GROUP - Groups only +/// \li \c H5F_OBJ_DATATYPE - Named datatypes only +/// \li \c H5F_OBJ_ATTR - Attributes only /// \li \c H5F_OBJ_ALL - All of the above, i.e., \c H5F_OBJ_FILE -/// | \c H5F_OBJ_DATASET | \c H5F_OBJ_GROUP -/// | \c H5F_OBJ_DATATYPE | \c H5F_OBJ_ATTR +/// | \c H5F_OBJ_DATASET | \c H5F_OBJ_GROUP +/// | \c H5F_OBJ_DATATYPE | \c H5F_OBJ_ATTR ///\par /// Multiple object types can be combined with the logical OR operator (|). // Programmer Binh-Minh Ribler - May 2004 @@ -425,19 +425,19 @@ ssize_t H5File::getObjCount(unsigned types) const /// groups and datatypes) in the same file. ///\param types - Type of object to retrieve the count ///\param max_objs - Maximum number of object identifiers to place -/// into obj_id_list. +/// into obj_id_list. ///\param oid_list - List of open object identifiers ///\exception H5::FileIException ///\par Description /// The valid values for \a types include: -/// \li \c H5F_OBJ_FILE - Files only -/// \li \c H5F_OBJ_DATASET - Datasets only -/// \li \c H5F_OBJ_GROUP - Groups only -/// \li \c H5F_OBJ_DATATYPE - Named datatypes only -/// \li \c H5F_OBJ_ATTR - Attributes only +/// \li \c H5F_OBJ_FILE - Files only +/// \li \c H5F_OBJ_DATASET - Datasets only +/// \li \c H5F_OBJ_GROUP - Groups only +/// \li \c H5F_OBJ_DATATYPE - Named datatypes only +/// \li \c H5F_OBJ_ATTR - Attributes only /// \li \c H5F_OBJ_ALL - All of the above, i.e., \c H5F_OBJ_FILE -/// | \c H5F_OBJ_DATASET | \c H5F_OBJ_GROUP -/// | \c H5F_OBJ_DATATYPE | \c H5F_OBJ_ATTR +/// | \c H5F_OBJ_DATASET | \c H5F_OBJ_GROUP +/// | \c H5F_OBJ_DATATYPE | \c H5F_OBJ_ATTR ///\par /// Multiple object types can be combined with the logical OR operator (|). // @@ -459,7 +459,7 @@ void H5File::getObjIDs(unsigned types, size_t max_objs, hid_t *oid_list) const /// driver. ///\param fapl - File access property list ///\param file_handle - Pointer to the file handle being used by -/// the low-level virtual file driver +/// the low-level virtual file driver ///\exception H5::FileIException ///\par Description /// For the \c FAMILY or \c MULTI drivers, \a fapl should be @@ -487,10 +487,10 @@ void H5File::getVFDHandle(const FileAccPropList& fapl, void **file_handle) const // Purpose This is an overloaded member function, kept for backward // compatibility. It differs from the above function in that it // misses const's. This wrapper will be removed in future release. -// Param fapl - File access property list -// Param file_handle - Pointer to the file handle being used by -// the low-level virtual file driver -// Exception H5::FileIException +// Param fapl - File access property list +// Param file_handle - Pointer to the file handle being used by +// the low-level virtual file driver +// Exception H5::FileIException // Programmer Binh-Minh Ribler - May 2004 // Modification // Planned for removal. -BMR, 2014/04/16 @@ -508,7 +508,7 @@ void H5File::getVFDHandle(const FileAccPropList& fapl, void **file_handle) const /// It differs from the above function only in what arguments it /// accepts. ///\param file_handle - Pointer to the file handle being used by -/// the low-level virtual file driver +/// the low-level virtual file driver ///\exception H5::FileIException // Programmer Binh-Minh Ribler - May 2004 //-------------------------------------------------------------------------- @@ -547,7 +547,7 @@ hsize_t H5File::getFileSize() const ///\brief Get the id of this file ///\return File identifier // Modification -// May 2008 - BMR +// May 2008 - BMR // Class hierarchy is revised to address bugzilla 1068. Class // AbstractDS and Attribute are moved out of H5Object. In // addition, member IdComponent::id is moved into subclasses, and @@ -562,8 +562,8 @@ hid_t H5File::getId() const #ifndef DOXYGEN_SHOULD_SKIP_THIS //-------------------------------------------------------------------------- // Function: H5File::reopen -// Purpose Reopens this file. -// Exception H5::FileIException +// Purpose Reopens this file. +// Exception H5::FileIException // Description // This function is replaced by the above function reOpen. // Programmer Binh-Minh Ribler - 2000 @@ -575,14 +575,15 @@ void H5File::reopen() //-------------------------------------------------------------------------- // Function: H5File::getLocId -// Purpose Get the id of this file +// Purpose Get the id of this file // Description // This function is a redefinition of CommonFG::getLocId. It // is used by CommonFG member functions to get the file id. // Programmer Binh-Minh Ribler - 2000 // Deprecated: -// After HDFFV-9920, the Group's methods can use getId() and getLocId() -// is kept for backward compatibility. Aug 18, 2016 -BMR +// Aug 18, 2016 -BMR +// After HDFFV-9920, the Group's methods can use getId() and +// getLocId() is kept for backward compatibility. //-------------------------------------------------------------------------- hid_t H5File::getLocId() const { @@ -626,13 +627,13 @@ void H5File::close() { if (p_valid_id(id)) { - herr_t ret_value = H5Fclose(id); - if (ret_value < 0) - { - throw FileIException("H5File::close", "H5Fclose failed"); - } - // reset the id - id = H5I_INVALID_HID; + herr_t ret_value = H5Fclose(id); + if (ret_value < 0) + { + throw FileIException("H5File::close", "H5Fclose failed"); + } + // reset the id + id = H5I_INVALID_HID; } } @@ -643,7 +644,7 @@ void H5File::close() ///\param msg - Message describing the failure ///\exception H5::FileIException // Description -// This function is also used in H5Location implementation so that +// This function is also used in H5Location implementation so that // proper exception can be thrown for file or group. The // "H5File::" will be inserted to indicate the function called is // an implementation of H5File. @@ -669,9 +670,9 @@ void H5File::throwException(const H5std_string& func_name, const H5std_string& m H5File::~H5File() { try { - close(); + close(); } catch (Exception& close_error) { - cerr << "H5File::~H5File - " << close_error.getDetailMsg() << endl; + cerr << "H5File::~H5File - " << close_error.getDetailMsg() << endl; } } diff --git a/c++/src/H5File.h b/c++/src/H5File.h index 5ac9118..df5cca6 100644 --- a/c++/src/H5File.h +++ b/c++/src/H5File.h @@ -17,7 +17,6 @@ #ifndef __H5File_H #define __H5File_H - namespace H5 { /*! \class H5File @@ -123,7 +122,8 @@ class H5_DLLCPP H5File : public Group { // constructors taking a string or a char* void p_get_file(const char* name, unsigned int flags, const FileCreatPropList& create_plist, const FileAccPropList& access_plist); -}; -} +}; // end of H5File +} // namespace H5 + #endif // __H5File_H diff --git a/c++/src/H5FloatType.h b/c++/src/H5FloatType.h index 2c925de..8affa1a 100644 --- a/c++/src/H5FloatType.h +++ b/c++/src/H5FloatType.h @@ -75,6 +75,8 @@ class H5_DLLCPP FloatType : public AtomType { // Noop destructor. virtual ~FloatType(); -}; -} + +}; // end of FloatType +} // namespace H5 + #endif // __H5FloatType_H diff --git a/c++/src/H5Group.cpp b/c++/src/H5Group.cpp index 6b143b5..bb9e05d 100644 --- a/c++/src/H5Group.cpp +++ b/c++/src/H5Group.cpp @@ -64,15 +64,64 @@ Group::Group(const Group& original) : H5Object(), CommonFG(), id(original.id) } //-------------------------------------------------------------------------- +// Function: Group::getObjId +///\brief Opens an object via object header. +///\exception H5::FileIException or H5::GroupIException +///\par Description +/// This function opens an object in a group or file, using +/// H5Oopen. Thus, an object can be opened without knowing +/// the object's type. +// Programmer Binh-Minh Ribler - March, 2017 +//-------------------------------------------------------------------------- +hid_t Group::getObjId(const char* obj_name, const PropList& plist) const +{ + hid_t ret_value = H5Oopen(getId(), obj_name, plist.getId()); + if (ret_value < 0) + { + throwException("Group::getObjId", "H5Oopen failed"); + } + return(ret_value); +} + +//-------------------------------------------------------------------------- +// Function: Group::getObjId +///\brief This is an overloaded member function, provided for convenience. +/// It takes a reference to a \c H5std_string for the object's name. +///\param obj_name - IN: Path to the object +///\exception H5::FileIException or H5::GroupIException +// Programmer Binh-Minh Ribler - March, 2017 +//-------------------------------------------------------------------------- +hid_t Group::getObjId(const H5std_string& obj_name, const PropList& plist) const +{ + return(getObjId(obj_name.c_str(), plist)); +} + +//-------------------------------------------------------------------------- +// Function: Group::closeObjId +///\brief Closes an object, which was opened with Group::getObjId +///\exception H5::FileIException or H5::GroupIException +// Programmer Binh-Minh Ribler - March, 2017 +//-------------------------------------------------------------------------- +void Group::closeObjId(hid_t obj_id) const +{ + herr_t ret_value = H5Oclose(obj_id); + if (ret_value < 0) + { + throwException("Group::closeObjId", "H5Oclose failed"); + } +} + +//-------------------------------------------------------------------------- // Function: Group::getLocId -// Purpose: Get the id of this group +// Purpose: Get the id of this group // Programmer Binh-Minh Ribler - 2000 // Description // This function is a redefinition of CommonFG::getLocId. It // is used by CommonFG member functions to get the file id. // Deprecated: -// After HDFFV-9920, the Group's methods can use getId() and getLocId() -// is kept for backward compatibility. Aug 18, 2016 -BMR +// Aug 18, 2016 -BMR +// After HDFFV-9920, the Group's methods can use getId() and +// getLocId() is kept for backward compatibility. //-------------------------------------------------------------------------- hid_t Group::getLocId() const { @@ -179,13 +228,13 @@ void Group::close() { if (p_valid_id(id)) { - herr_t ret_value = H5Gclose(id); - if (ret_value < 0) - { - throwException("Group::close", "H5Gclose failed"); - } - // reset the id - id = H5I_INVALID_HID; + herr_t ret_value = H5Gclose(id); + if (ret_value < 0) + { + throwException("Group::close", "H5Gclose failed"); + } + // reset the id + id = H5I_INVALID_HID; } } @@ -222,10 +271,10 @@ void Group::throwException(const H5std_string& func_name, const H5std_string& ms Group::~Group() { try { - close(); + close(); } catch (Exception& close_error) { - cerr << "Group::~Group - " << close_error.getDetailMsg() << endl; + cerr << "Group::~Group - " << close_error.getDetailMsg() << endl; } } diff --git a/c++/src/H5Group.h b/c++/src/H5Group.h index 96dc5c1..0f44eb9 100644 --- a/c++/src/H5Group.h +++ b/c++/src/H5Group.h @@ -54,6 +54,13 @@ class H5_DLLCPP Group : public H5Object, public CommonFG { Group(const H5Location& loc, const void* ref, H5R_type_t ref_type = H5R_OBJECT, const PropList& plist = PropList::DEFAULT); // Group(const Attribute& attr, const void* ref, H5R_type_t ref_type = H5R_OBJECT, const PropList& plist = PropList::DEFAULT); + // Opens an object within a group or a file, i.e., root group. + hid_t getObjId(const char* name, const PropList& plist = PropList::DEFAULT) const; + hid_t getObjId(const H5std_string& name, const PropList& plist = PropList::DEFAULT) const; + + // Closes an object opened by getObjId(). + void closeObjId(hid_t obj_id) const; + // default constructor Group(); @@ -77,7 +84,8 @@ class H5_DLLCPP Group : public H5Object, public CommonFG { private: hid_t id; // HDF5 group id -}; -} +}; // end of Group +} // namespace H5 + #endif // __Group_H diff --git a/c++/src/H5IdComponent.cpp b/c++/src/H5IdComponent.cpp index 574b28a..0bcc67a 100644 --- a/c++/src/H5IdComponent.cpp +++ b/c++/src/H5IdComponent.cpp @@ -194,6 +194,27 @@ hsize_t IdComponent::getNumMembers(H5I_type_t type) } //-------------------------------------------------------------------------- +// Function: isValid (static) +///\brief Checks if the given ID is valid. +///\return true if the given identifier is valid, and false, otherwise. +///\Description +/// A valid ID is one that is in use and has an application +/// reference count of at least 1. +// Programmer Binh-Minh Ribler - Mar 1, 2017 +//-------------------------------------------------------------------------- +bool IdComponent::isValid(hid_t an_id) +{ + // Call C function + htri_t ret_value = H5Iis_valid(an_id); + if (ret_value > 0) + return true; + else if (ret_value == 0) + return false; + else // Raise exception when H5Iis_valid returns a negative value + throw IdComponentException("isValid", "H5Iis_valid failed"); +} + +//-------------------------------------------------------------------------- // Function: typeExists (static) ///\brief Queries if a given type is currently registered with the /// library. diff --git a/c++/src/H5IdComponent.h b/c++/src/H5IdComponent.h index 6f57364..0603e5c 100644 --- a/c++/src/H5IdComponent.h +++ b/c++/src/H5IdComponent.h @@ -19,7 +19,6 @@ namespace H5 { -class DataSpace; /*! \class IdComponent \brief Class IdComponent provides wrappers of the C functions that operate on an HDF5 identifier. @@ -51,6 +50,9 @@ class H5_DLLCPP IdComponent { // Returns the number of members in a type. static hsize_t getNumMembers(H5I_type_t type); + // Checks if the given ID is valid. + static bool isValid(hid_t an_id); + // Determines if an type exists. static bool typeExists(H5I_type_t type); @@ -121,6 +123,6 @@ class H5_DLLCPP IdComponent { #endif // DOXYGEN_SHOULD_SKIP_THIS }; // end class IdComponent +} // namespace H5 -} #endif // __IdComponent_H diff --git a/c++/src/H5IntType.h b/c++/src/H5IntType.h index ae4b3df..1ce05c8 100644 --- a/c++/src/H5IntType.h +++ b/c++/src/H5IntType.h @@ -57,6 +57,8 @@ class H5_DLLCPP IntType : public AtomType { // Noop destructor. virtual ~IntType(); -}; -} + +}; // end of IntType +} // namespace H5 + #endif // __H5IntType_H diff --git a/c++/src/H5LaccProp.cpp b/c++/src/H5LaccProp.cpp index e9adb12..e457e83 100644 --- a/c++/src/H5LaccProp.cpp +++ b/c++/src/H5LaccProp.cpp @@ -40,7 +40,7 @@ LinkAccPropList* LinkAccPropList::DEFAULT_ = 0; // If LinkAccPropList::DEFAULT_ already points to an allocated // object, throw a PropListIException. This scenario should not // happen. -// Programmer Binh-Minh Ribler - 2015 +// Programmer Binh-Minh Ribler - December, 2016 //-------------------------------------------------------------------------- LinkAccPropList* LinkAccPropList::getConstant() { @@ -66,7 +66,7 @@ LinkAccPropList* LinkAccPropList::getConstant() // Purpose: Deletes the constant object that LinkAccPropList::DEFAULT_ // points to. // exception H5::PropListIException -// Programmer Binh-Minh Ribler - 2015 +// Programmer Binh-Minh Ribler - December, 2016 //-------------------------------------------------------------------------- void LinkAccPropList::deleteConstants() { @@ -84,7 +84,7 @@ const LinkAccPropList& LinkAccPropList::DEFAULT = *getConstant(); //-------------------------------------------------------------------------- // Function: Default Constructor ///\brief Creates a file access property list -// Programmer Binh-Minh Ribler - 2000 +// Programmer Binh-Minh Ribler - December, 2016 //-------------------------------------------------------------------------- LinkAccPropList::LinkAccPropList() : PropList(H5P_LINK_ACCESS) {} @@ -92,7 +92,7 @@ LinkAccPropList::LinkAccPropList() : PropList(H5P_LINK_ACCESS) {} // Function: LinkAccPropList copy constructor ///\brief Copy Constructor: makes a copy of the original ///\param original - IN: LinkAccPropList instance to copy -// Programmer Binh-Minh Ribler - 2000 +// Programmer Binh-Minh Ribler - December, 2016 //-------------------------------------------------------------------------- LinkAccPropList::LinkAccPropList(const LinkAccPropList& original) : PropList(original) {} @@ -100,14 +100,51 @@ LinkAccPropList::LinkAccPropList(const LinkAccPropList& original) : PropList(ori // Function: LinkAccPropList overloaded constructor ///\brief Creates a file access property list using the id of an /// existing one. -// Programmer Binh-Minh Ribler - 2000 +// Programmer Binh-Minh Ribler - December, 2016 //-------------------------------------------------------------------------- LinkAccPropList::LinkAccPropList(const hid_t plist_id) : PropList(plist_id) {} //-------------------------------------------------------------------------- +// Function: LinkAccPropList::setNumLinks +///\brief Set the number of soft or user-defined link traversals allowed +/// before the library assumes it has found a cycle and aborts the +/// traversal. +///\exception H5::PropListIException +// Programmer Binh-Minh Ribler - March 1, 2017 +//-------------------------------------------------------------------------- +void LinkAccPropList::setNumLinks(size_t nlinks) const +{ + herr_t ret_value = H5Pset_nlinks(id, nlinks); + // Throw exception if H5Pset_nlinks returns failure + if (ret_value < 0) + { + throw PropListIException("setNumLinks", "H5Pset_nlinks failed"); + } +} + +//-------------------------------------------------------------------------- +// Function: LinkAccPropList::getNumLinks +///\brief Gets the number of soft or user-defined links that can be +/// traversed before a failure occurs. +///\exception H5::PropListIException +// Programmer Binh-Minh Ribler - March 1, 2017 +//-------------------------------------------------------------------------- +size_t LinkAccPropList::getNumLinks() const +{ + size_t nlinks = 0; + herr_t ret_value = H5Pget_nlinks(id, &nlinks); + // Throw exception if H5Pget_nlinks returns failure + if (ret_value < 0) + { + throw PropListIException("getNumLinks", "H5Pget_nlinks failed"); + } + return(nlinks); +} + +//-------------------------------------------------------------------------- // Function: LinkAccPropList destructor ///\brief Noop destructor -// Programmer Binh-Minh Ribler - 2000 +// Programmer Binh-Minh Ribler - December, 2016 //-------------------------------------------------------------------------- LinkAccPropList::~LinkAccPropList() {} diff --git a/c++/src/H5LaccProp.h b/c++/src/H5LaccProp.h index 8bcdd64..f31c3fc 100644 --- a/c++/src/H5LaccProp.h +++ b/c++/src/H5LaccProp.h @@ -46,6 +46,13 @@ class H5_DLLCPP LinkAccPropList : public PropList { // using the property list id. LinkAccPropList (const hid_t plist_id); + // Sets the number of soft or user-defined links that can be + // traversed before a failure occurs. + void setNumLinks(size_t nlinks) const; + + // Gets the number of soft or user-defined link traversals allowed + size_t getNumLinks() const; + // Noop destructor virtual ~LinkAccPropList(); @@ -62,6 +69,7 @@ class H5_DLLCPP LinkAccPropList : public PropList { #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} +}; // end of LinkAccPropList +} // namespace H5 + #endif // __H5LinkAccPropList_H diff --git a/c++/src/H5Library.h b/c++/src/H5Library.h index 018ba38..76b5f82 100644 --- a/c++/src/H5Library.h +++ b/c++/src/H5Library.h @@ -69,6 +69,7 @@ class H5_DLLCPP H5Library { ~H5Library(); #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} +}; // end of H5Library +} // namespace H5 + #endif // __H5Library_H diff --git a/c++/src/H5Location.h b/c++/src/H5Location.h index 337a2b3..3a0d8ca 100644 --- a/c++/src/H5Location.h +++ b/c++/src/H5Location.h @@ -220,7 +220,7 @@ class H5_DLLCPP H5Location : public IdComponent { // Noop destructor. virtual ~H5Location(); -}; /* end class H5Location */ +}; // end of H5Location +} // namespace H5 -} #endif // __H5Location_H diff --git a/c++/src/H5Object.h b/c++/src/H5Object.h index 866d739..53f183e 100644 --- a/c++/src/H5Object.h +++ b/c++/src/H5Object.h @@ -131,7 +131,7 @@ class H5_DLLCPP H5Object : public H5Location { #endif // DOXYGEN_SHOULD_SKIP_THIS -}; /* end class H5Object */ +}; // end of H5Object +} // namespace H5 -} #endif // __H5Object_H diff --git a/c++/src/H5OcreatProp.h b/c++/src/H5OcreatProp.h index 4858e07..c9c245a 100644 --- a/c++/src/H5OcreatProp.h +++ b/c++/src/H5OcreatProp.h @@ -72,6 +72,7 @@ class H5_DLLCPP ObjCreatPropList : public PropList { #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} +}; // end of ObjCreatPropList +} // namespace H5 + #endif // __H5ObjCreatPropList_H diff --git a/c++/src/H5PredType.h b/c++/src/H5PredType.h index a8d6e37..2fd8a6b 100644 --- a/c++/src/H5PredType.h +++ b/c++/src/H5PredType.h @@ -438,6 +438,7 @@ class H5_DLLCPP PredType : public AtomType { #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} +}; // end of PredType +} // namespace H5 + #endif // __H5PredType_H diff --git a/c++/src/H5PropList.cpp b/c++/src/H5PropList.cpp index c4176c2..0df0a9f 100644 --- a/c++/src/H5PropList.cpp +++ b/c++/src/H5PropList.cpp @@ -565,8 +565,27 @@ size_t PropList::getNumProps() const ///\param name - IN: Name of property to set - \c char pointer ///\param value - IN: Void pointer to the value for the property ///\exception H5::PropListIException +// Description +// Revision svn r29815 changed 'value' to const, hence, deprecated +// the non-const setProperty. // Programmer: Binh-Minh Ribler - April, 2004 //-------------------------------------------------------------------------- +void PropList::setProperty(const char* name, const void* value) const +{ + herr_t ret_value = H5Pset(id, name, value); + if (ret_value < 0) + { + throw PropListIException(inMemFunc("setProperty"), "H5Pset failed"); + } +} + +//-------------------------------------------------------------------------- +// Function: PropList::setProperty +///\brief Deprecated due to missing const in prototype. (1.10.1) +// Programmer: Binh-Minh Ribler - March, 2017 +// Modification +// Planned for removal. -BMR, 2017/03/17 1.10.1 +//-------------------------------------------------------------------------- void PropList::setProperty(const char* name, void* value) const { herr_t ret_value = H5Pset(id, name, value); @@ -575,6 +594,7 @@ void PropList::setProperty(const char* name, void* value) const throw PropListIException(inMemFunc("setProperty"), "H5Pset failed"); } } + //-------------------------------------------------------------------------- // Function: PropList::setProperty ///\brief This is an overloaded member function, provided for convenience. @@ -582,11 +602,14 @@ void PropList::setProperty(const char* name, void* value) const /// accepts. ///\param name - IN: Name of property to set - \c char pointer ///\param charptr - IN: Char pointer to the value for the property +// Description +// Revision svn r29815 changed 'value' to const, hence, deprecated +// the non-const setProperty. // Programmer: Binh-Minh Ribler - April, 2004 //-------------------------------------------------------------------------- void PropList::setProperty(const char* name, const char* charptr) const { - herr_t ret_value = H5Pset(id, name, (void*)charptr); + herr_t ret_value = H5Pset(id, name, (const void*)charptr); if (ret_value < 0) { throw PropListIException(inMemFunc("setProperty"), "H5Pset failed"); @@ -601,6 +624,18 @@ void PropList::setProperty(const char* name, const char* charptr) const ///\param strg - IN: Value for the property is a \c H5std_string // Programmer: Binh-Minh Ribler - April, 2004 //-------------------------------------------------------------------------- +void PropList::setProperty(const char* name, const H5std_string& strg) const +{ + setProperty(name, strg.c_str()); +} + +//-------------------------------------------------------------------------- +// Function: PropList::setProperty +///\brief Deprecated due to missing const in prototype. (1.10.1) +// Programmer: Binh-Minh Ribler - March, 2017 +// Modification +// Planned for removal. -BMR, 2017/03/17 1.10.1 +//-------------------------------------------------------------------------- void PropList::setProperty(const char* name, H5std_string& strg) const { setProperty(name, strg.c_str()); @@ -615,6 +650,18 @@ void PropList::setProperty(const char* name, H5std_string& strg) const ///\param value - IN: Void pointer to the value for the property // Programmer: Binh-Minh Ribler - April, 2004 //-------------------------------------------------------------------------- +void PropList::setProperty(const H5std_string& name, const void* value) const +{ + setProperty(name.c_str(), value); +} + +//-------------------------------------------------------------------------- +// Function: PropList::setProperty +///\brief Deprecated due to missing const in prototype. (1.10.1) +// Programmer: Binh-Minh Ribler - March, 2017 +// Modification +// Planned for removal. -BMR, 2017/03/17 1.10.1 +//-------------------------------------------------------------------------- void PropList::setProperty(const H5std_string& name, void* value) const { setProperty(name.c_str(), value); @@ -629,6 +676,18 @@ void PropList::setProperty(const H5std_string& name, void* value) const ///\param strg - IN: Value for the property is a \c H5std_string // Programmer: Binh-Minh Ribler - April, 2004 //-------------------------------------------------------------------------- +void PropList::setProperty(const H5std_string& name, const H5std_string& strg) const +{ + setProperty(name.c_str(), strg.c_str()); +} + +//-------------------------------------------------------------------------- +// Function: PropList::setProperty +///\brief Deprecated due to missing const in prototype. (1.10.1) +// Programmer: Binh-Minh Ribler - March, 2017 +// Modification +// Planned for removal. -BMR, 2017/03/17 1.10.1 +//-------------------------------------------------------------------------- void PropList::setProperty(const H5std_string& name, H5std_string& strg) const { setProperty(name.c_str(), strg.c_str()); diff --git a/c++/src/H5PropList.h b/c++/src/H5PropList.h index 3977774..772e96c 100644 --- a/c++/src/H5PropList.h +++ b/c++/src/H5PropList.h @@ -78,8 +78,13 @@ class H5_DLLCPP PropList : public IdComponent { H5std_string getProperty(const H5std_string& name) const; // Set a property's value in a property list. - void setProperty(const char* name, void* value) const; void setProperty(const char* name, const char* charptr) const; + void setProperty(const char* name, const void* value) const; + void setProperty(const char* name, const H5std_string& strg) const; + void setProperty(const H5std_string& name, const void* value) const; + void setProperty(const H5std_string& name, const H5std_string& strg) const; + // Deprecated after 1.10.1, missing const + void setProperty(const char* name, void* value) const; void setProperty(const char* name, H5std_string& strg) const; void setProperty(const H5std_string& name, void* value) const; void setProperty(const H5std_string& name, H5std_string& strg) const; @@ -135,7 +140,8 @@ class H5_DLLCPP PropList : public IdComponent { friend void f_PropList_setId(PropList* plist, hid_t new_id); #endif // DOXYGEN_SHOULD_SKIP_THIS -}; -} +}; // end of PropList +} // namespace H5 + #endif // __H5PropList_H diff --git a/c++/src/H5StrType.h b/c++/src/H5StrType.h index 3272ad3..aa87bba 100644 --- a/c++/src/H5StrType.h +++ b/c++/src/H5StrType.h @@ -69,6 +69,8 @@ class H5_DLLCPP StrType : public AtomType { // Noop destructor. virtual ~StrType(); -}; -} + +}; // end of StrType +} // namespace H5 + #endif // __H5StrType_H diff --git a/c++/src/H5VarLenType.h b/c++/src/H5VarLenType.h index a93f44d..4dc0eb5 100644 --- a/c++/src/H5VarLenType.h +++ b/c++/src/H5VarLenType.h @@ -49,6 +49,8 @@ class H5_DLLCPP VarLenType : public DataType { // Default constructor VarLenType(); -}; -} + +}; // end of VarLenType +} // namespace H5 + #endif // __H5VarLenType_H diff --git a/c++/test/dsets.cpp b/c++/test/dsets.cpp index 48a0c4d..970f210 100644 --- a/c++/test/dsets.cpp +++ b/c++/test/dsets.cpp @@ -1148,6 +1148,87 @@ test_types(H5File& file) /*------------------------------------------------------------------------- + * Function: test_virtual + * + * Purpose: Tests fixed, unlimited, and printf selections in the same + * VDS + * + * Return: Success: 0 + * + * Failure: number of errors + * + * Programmer: Binh-Minh Ribler + * Friday, March 10, 2017 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +const int RANK = 2; +static herr_t +test_virtual() +{ + SUBTEST("DSetCreatPropList::setVirtual"); + + try { + // Create DCPLs + DSetCreatPropList dcpl; + DSetCreatPropList srcdcpl; + + // Set fill value + char *fill = NULL; + dcpl.setFillValue(PredType::NATIVE_INT, &fill); + + // Set chunk dimensions + hsize_t cdims[RANK]; + cdims[0] = 2; + cdims[1] = 2; + srcdcpl.setChunk(RANK, cdims); + + // Create memory space + hsize_t mdims[RANK]; + mdims[0] = 10; + mdims[1] = 10; + DataSpace memspace(RANK, mdims); + + // Get the current layout, should be default, H5D_CONTIGUOUS + H5D_layout_t layout = dcpl.getLayout(); + verify_val(layout, H5D_CONTIGUOUS, "DSetCreatPropList::getLayout", __LINE__, __FILE__); + + // Create fixed mapping + hsize_t dims[RANK]; + dims[0] = 6; + dims[1] = 6; + DataSpace vspace(RANK, dims, mdims); + + hsize_t start[RANK]; // Hyperslab start + hsize_t count[RANK]; // Hyperslab count + start[0] = start[1] = 3; + count[0] = count[1] = 3; + vspace.selectHyperslab(H5S_SELECT_SET, count, start); + + DataSpace srcspace(RANK, count); + + H5std_string src_file = "src_file_map.h5"; + H5std_string src_dset = "src_dset_fixed"; + dcpl.setVirtual(vspace, src_file, src_dset, srcspace); + + // Get and verify the new layout + layout = dcpl.getLayout(); + verify_val(layout, H5D_VIRTUAL, "DSetCreatPropList::getLayout", __LINE__, __FILE__); + + PASSED(); + return 0; + } // end top try block + + catch (Exception& E) + { + return -1; + } +} // test_virtual + + +/*------------------------------------------------------------------------- * Function: test_dset * * Purpose Tests the dataset interface (H5D) @@ -1195,6 +1276,7 @@ void test_dset() nerrors += test_nbit_compression(file) < 0 ? 1:0; nerrors += test_multiopen (file) < 0 ? 1:0; nerrors += test_types(file) < 0 ? 1:0; + nerrors += test_virtual() < 0 ? 1:0; // Close group "emit diagnostics". grp.close(); diff --git a/c++/test/tfile.cpp b/c++/test/tfile.cpp index 261beab..715bb30 100644 --- a/c++/test/tfile.cpp +++ b/c++/test/tfile.cpp @@ -812,17 +812,17 @@ const H5std_string FILE7("tfile7.h5"); * *------------------------------------------------------------------------- */ +const hsize_t FSP_SIZE_DEF = 4096; +const hsize_t FSP_SIZE512 = 512; static void test_file_info() { // Output message about test being performed SUBTEST("File general information"); - hsize_t in_threshold = 2; // Free space section threshold to set */ - hsize_t out_threshold = 0; // Free space section threshold to get */ + hsize_t out_threshold = 0; // Free space section threshold to get + hbool_t out_persist = FALSE;// Persist free-space read // File space handling strategy - H5F_file_space_type_t in_strategy = H5F_FILE_SPACE_ALL; - // File space handling strategy - H5F_file_space_type_t out_strategy = H5F_FILE_SPACE_DEFAULT; + H5F_fspace_strategy_t out_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; try { // Create a file using default properties. @@ -841,12 +841,30 @@ static void test_file_info() // Create file creation property list. FileCreatPropList fcpl; + // Retrieve file space information. + fcpl.getFileSpaceStrategy(out_strategy, out_persist, out_threshold); + + // Verify file space information. + verify_val(out_strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5File::getFileInfo", __LINE__, __FILE__); + verify_val(out_persist, FALSE, "H5File::getFileInfo", __LINE__, __FILE__); + verify_val(out_threshold, 1, "H5File::getFileInfo", __LINE__, __FILE__); + + /* Retrieve file space page size */ + hsize_t out_fsp_psize = fcpl.getFileSpacePagesize(); + verify_val(out_fsp_psize, FSP_SIZE_DEF, "FileCreatPropList::getFileSpacePagesize", __LINE__, __FILE__); + // Set various file information. fcpl.setUserblock(F2_USERBLOCK_SIZE); fcpl.setSizes(F2_OFFSET_SIZE, F2_LENGTH_SIZE); fcpl.setSymk(F2_SYM_INTERN_K, F2_SYM_LEAF_K); fcpl.setIstorek(F2_ISTORE); - fcpl.setFileSpace(in_strategy, in_threshold); + + hsize_t threshold = 5; // Free space section threshold to set + hbool_t persist = TRUE; // Persist free-space to set + H5F_fspace_strategy_t strategy = H5F_FSPACE_STRATEGY_PAGE; + + fcpl.setFileSpaceStrategy(strategy, persist, threshold); + fcpl.setFileSpacePagesize(FSP_SIZE512); // Creating a file with the non-default file creation property list // should create a version 1 superblock @@ -901,11 +919,13 @@ static void test_file_info() */ // Get and verify the file space info from the creation property list */ - out_strategy = fcpl2.getFileSpaceStrategy(); - verify_val(static_cast<unsigned>(out_strategy), static_cast<unsigned>(in_strategy), "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__); + fcpl2.getFileSpaceStrategy(out_strategy, out_persist, out_threshold); + verify_val(out_strategy, strategy, "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__); + verify_val(out_persist, persist, "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__); + verify_val(out_threshold, threshold, "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__); - out_threshold = fcpl2.getFileSpaceThreshold(); - verify_val(static_cast<unsigned>(out_threshold), static_cast<unsigned>(in_threshold), "FileCreatPropList::getFileSpaceThreshold", __LINE__, __FILE__); + out_fsp_psize = fcpl2.getFileSpacePagesize(); + verify_val(out_fsp_psize, FSP_SIZE512, "FileCreatPropList::getFileSpacePagesize", __LINE__, __FILE__); PASSED(); } // end of try block diff --git a/c++/test/titerate.cpp b/c++/test/titerate.cpp index fce42fa..34e6892 100644 --- a/c++/test/titerate.cpp +++ b/c++/test/titerate.cpp @@ -503,9 +503,9 @@ void test_iterate() FileAccPropList fapl; fapl.setLibverBounds(H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); - test_iter_group(fapl); // Test iterating groups - test_HDFFV_9920(); // Test the fix of HDFFV-9920 - //test_iter_attr(fapl); // Test iterating attributes + test_iter_group(fapl); // Test iterating groups + test_HDFFV_9920(); // Test the fix of HDFFV-9920 + //test_iter_attr(fapl); // Test iterating attributes } // test_iterate diff --git a/c++/test/tlinks.cpp b/c++/test/tlinks.cpp index c217718..0bf6ebf 100644 --- a/c++/test/tlinks.cpp +++ b/c++/test/tlinks.cpp @@ -142,11 +142,8 @@ const char *FILENAME[] = { #define H5L_DIM2 100 /* Creation order macros */ -#define CORDER_GROUP_NAME "corder_group" #define CORDER_SOFT_GROUP_NAME "corder_soft_group" #define CORDER_NLINKS 18 -#define CORDER_ITER_STOP 3 -#define CORDER_EST_ENTRY_LEN 9 /* Timestamp macros */ #define TIMESTAMP_GROUP_1 "timestamp1" @@ -341,7 +338,7 @@ static const char *FILENAME[] = { static void test_basic_links(hid_t fapl_id, hbool_t new_format) { hsize_t size[1] = {1}; - char filename[NAME_BUF_SIZE]; + char filename[NAME_BUF_SIZE]; // Use the file access template id to create a file access prop. list. FileAccPropList fapl(fapl_id); @@ -441,6 +438,55 @@ static void test_basic_links(hid_t fapl_id, hbool_t new_format) /*------------------------------------------------------------------------- + * Function: test_num_links + * + * Purpose Test setting and getting limit of number of links + * + * Return Success: 0 + * + * Failure: -1 + * + * Programmer Binh-Minh Ribler + * October 16, 2009 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void test_num_links(hid_t fapl_id, hbool_t new_format) +{ + char filename[NAME_BUF_SIZE]; + + if(new_format) + SUBTEST("Setting number of links (w/new group format)") + else + SUBTEST("Setting number of links") + + try + { + // Use the file access template id to create a file access prop. list. + FileAccPropList fapl(fapl_id); + + h5_fixname(FILENAME[0], fapl_id, filename, sizeof filename); + H5File file(filename, H5F_ACC_RDWR, FileCreatPropList::DEFAULT, fapl); + + LinkAccPropList lapl; + size_t nlinks = 5; + lapl.setNumLinks(nlinks); + + // Read it back and verify + size_t read_nlinks = lapl.getNumLinks(); + verify_val(read_nlinks, nlinks, "LinkAccPropList::setNumLinks", __LINE__, __FILE__); + + PASSED(); + } // end of try block + catch (Exception& E) + { + issue_fail_msg("test_num_links()", __LINE__, __FILE__, E.getCDetailMsg()); + } +} // test_num_links + +/*------------------------------------------------------------------------- * Function: test_links * * Purpose Test links diff --git a/c++/test/tobject.cpp b/c++/test/tobject.cpp index 6003dd6..3c9c1b0 100644 --- a/c++/test/tobject.cpp +++ b/c++/test/tobject.cpp @@ -32,6 +32,7 @@ using namespace H5; #include "h5cpputil.h" // C++ utilility header file const H5std_string FILE_OBJECTS("tobjects.h5"); +const H5std_string FILE_OBJHDR("tobject_header.h5"); const H5std_string GROUP1("Top Group"); const H5std_string GROUP1_PATH("/Top Group"); const H5std_string GROUP1_1("Sub-Group 1.1"); @@ -395,7 +396,126 @@ static void test_get_objtype() issue_fail_msg("test_get_objtype", __LINE__, __FILE__); } } // test_get_objtype + +/*------------------------------------------------------------------------- + * Function: test_open_object_header + * + * Purpose Test Group::getObjId function. + * + * Return None + * + * Programmer Binh-Minh Ribler (use C version) + * March, 2017 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +const H5std_string GROUPNAME("group"); +const H5std_string DTYPENAME("group/datatype"); +const H5std_string DTYPENAME_INGRP("datatype"); +const H5std_string DSETNAME("dataset"); +#define RANK 2 +#define DIM0 5 +#define DIM1 10 +static void test_open_object_header() +{ + hsize_t dims[2]; + H5G_info_t ginfo; /* Group info struct */ + + // Output message about test being performed + SUBTEST("Group::getObjId"); + + try { + // Create file FILE1 + H5File file1(FILE_OBJHDR, H5F_ACC_TRUNC); + /* Create a group, dataset, and committed datatype within the file */ + + // Create a group in the root group + Group grp(file1.createGroup(GROUPNAME)); + grp.close(); + // Commit the type inside the file + IntType dtype(PredType::NATIVE_INT); + dtype.commit(file1, DTYPENAME); + dtype.close(); + + // Create a new dataset + dims[0] = DIM0; + dims[1] = DIM1; + DataSpace dspace(RANK, dims); + DataSet dset(file1.createDataSet(DSETNAME, PredType::NATIVE_INT, dspace)); + + // Close dataset and dataspace + dset.close(); + dspace.close(); + + // Now make sure that getObjId can open all three types of objects + hid_t obj_grp = file1.getObjId(GROUPNAME); + hid_t obj_dtype = file1.getObjId(DTYPENAME); + hid_t obj_dset = file1.getObjId(DSETNAME); + + // Make sure that each is the right kind of ID + H5I_type_t id_type = IdComponent::getHDFObjType(obj_grp); + verify_val(id_type, H5I_GROUP, "H5Iget_type for group ID", __LINE__, __FILE__); + id_type = IdComponent::getHDFObjType(obj_dtype); + verify_val(id_type, H5I_DATATYPE, "H5Iget_type for datatype ID", __LINE__, __FILE__); + id_type = IdComponent::getHDFObjType(obj_dset); + verify_val(id_type, H5I_DATASET, "H5Iget_type for dataset ID", __LINE__, __FILE__); + + /* Do something more complex with each of the IDs to make sure */ + + Group grp2(obj_grp); + hsize_t num_objs = grp2.getNumObjs(); + verify_val(num_objs, 1, "H5Gget_info", __LINE__, __FILE__); + // There should be one object, the datatype + + // Close datatype object opened from the file + file1.closeObjId(obj_dtype); + + dset.setId(obj_dset); + dspace = dset.getSpace(); + bool is_simple = dspace.isSimple(); + dspace.close(); + + // Open datatype object from the group + obj_dtype = grp2.getObjId(DTYPENAME_INGRP); + + dtype.setId(obj_dtype); + H5T_class_t type_class = dtype.getClass(); + verify_val(type_class, H5T_INTEGER, "H5Tget_class", __LINE__, __FILE__); + dtype.close(); + + // Close datatype object + grp2.closeObjId(obj_dtype); + + // Close the group object + file1.closeObjId(obj_grp); + + // Try doing something with group, the ID should still work + num_objs = grp2.getNumObjs(); + verify_val(num_objs, 1, "H5Gget_info", __LINE__, __FILE__); + + // Close the cloned group + grp2.close(); + + PASSED(); + } // end of try block + // catch invalid action exception + catch (InvalidActionException& E) + { + cerr << " in InvalidActionException" << endl; + cerr << " *FAILED*" << endl; + cerr << " <<< " << E.getDetailMsg() << " >>>" << endl << endl; + } + // catch all other exceptions + catch (Exception& E) + { + cerr << " in Exception" << endl; + issue_fail_msg("test_file_name()", __LINE__, __FILE__, E.getCDetailMsg()); + } +} /* test_open_object_header() */ + /*------------------------------------------------------------------------- * Function: test_objects * @@ -421,6 +541,7 @@ void test_object() test_existance(); // Test check for object existance test_get_objname_ontypes(); // Test get object name from types test_get_objtype(); // Test get object type + test_open_object_header(); // Test object header functions (H5O) } // test_objects diff --git a/c++/test/trefer.cpp b/c++/test/trefer.cpp index d8a0d5f..c232809 100644 --- a/c++/test/trefer.cpp +++ b/c++/test/trefer.cpp @@ -16,7 +16,7 @@ /***************************************************************************** FILE trefer.cpp - HDF5 C++ testing the functionalities associated with the C - Reference interface (H5R) + Reference interface (H5R) ***************************************************************************/ #ifdef OLD_HEADER_FILENAME @@ -93,7 +93,7 @@ test_reference_params(void) file1 = new H5File (FILE1, H5F_ACC_TRUNC); // Create dataspace for datasets - hsize_t dims1[] = {SPACE1_DIM1}; + hsize_t dims1[] = {SPACE1_DIM1}; DataSpace sid1(SPACE1_RANK, dims1); // Create a group @@ -172,7 +172,7 @@ test_reference_params(void) catch (Exception& E) { issue_fail_msg("test_reference_param()",__LINE__,__FILE__, - E.getCFuncName(), E.getCDetailMsg()); + E.getCFuncName(), E.getCDetailMsg()); } if(file1) @@ -209,7 +209,7 @@ static void test_reference_obj(void) file1 = new H5File (FILE1, H5F_ACC_TRUNC); // Create dataspace for datasets - hsize_t dims1[] = {SPACE1_DIM1}; + hsize_t dims1[] = {SPACE1_DIM1}; DataSpace sid1(SPACE1_RANK, dims1); // Create dataset access property list @@ -364,7 +364,7 @@ static void test_reference_obj(void) catch (Exception& E) { issue_fail_msg("test_reference_obj()",__LINE__,__FILE__, - E.getCFuncName(), E.getCDetailMsg()); + E.getCFuncName(), E.getCDetailMsg()); } if(file1) @@ -493,7 +493,7 @@ test_reference_group(void) catch (Exception& E) { issue_fail_msg("test_reference_group()",__LINE__,__FILE__, - E.getCFuncName(), E.getCDetailMsg()); + E.getCFuncName(), E.getCDetailMsg()); } if(file1) @@ -523,10 +523,10 @@ test_reference_region_1D(void) SUBTEST("1-D Dataset Region Reference Functions"); try { - hdset_reg_ref_t *wbuf, // buffer to write to disk + hdset_reg_ref_t *wbuf, // buffer to write to disk *rbuf; // buffer read from disk - uint8_t *dwbuf, // Buffer for writing numeric data to disk - *drbuf; // Buffer for reading numeric data from disk + uint8_t *dwbuf, // Buffer for writing numeric data to disk + *drbuf; // Buffer for reading numeric data from disk // Allocate write & read buffers wbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)SPACE1_DIM1); @@ -538,7 +538,7 @@ test_reference_region_1D(void) H5File file1(FILE2, H5F_ACC_TRUNC); // Create dataspace for datasets - hsize_t dims3[] = {SPACE3_DIM1}; + hsize_t dims3[] = {SPACE3_DIM1}; DataSpace sid3(SPACE3_RANK, dims3); // Create dataset access property list @@ -558,7 +558,7 @@ test_reference_region_1D(void) dset3.close(); // Create dataspace for datasets - hsize_t dims1[] = {SPACE1_DIM1}; + hsize_t dims1[] = {SPACE1_DIM1}; DataSpace sid1(SPACE1_RANK, dims1); // Create a dataset @@ -785,7 +785,7 @@ test_reference_region_1D(void) catch (Exception& E) { issue_fail_msg("test_reference_region_1D()",__LINE__,__FILE__, - E.getCFuncName(), E.getCDetailMsg()); + E.getCFuncName(), E.getCDetailMsg()); } } /* test_reference_region_1D() */ diff --git a/c++/test/ttypes.cpp b/c++/test/ttypes.cpp index 0cc8918..22db539 100644 --- a/c++/test/ttypes.cpp +++ b/c++/test/ttypes.cpp @@ -248,12 +248,28 @@ static void test_query() index = tid2.getMemberIndex("ORANGE"); verify_val(index, 3, "EnumType::getMemberIndex()", __LINE__, __FILE__); - // Commit compound datatype and close it + // Commit compound datatype, and test getting the datatype creation + // prop list, then close it tid1.commit(file, CompT_NAME); + PropList tcpl = tid1.getCreatePlist(); + if (!IdComponent::isValid(tcpl.getId())) + { + // Throw an invalid action exception + throw InvalidActionException("H5Object::createAttribute", "Datatype creation property list is not valid"); + } + tcpl.close(); tid1.close(); - // Commit enumeration datatype and close it + // Commit enumeration datatype, and test getting the datatype creation + // prop list, then close it tid2.commit(file, EnumT_NAME); + tcpl = tid2.getCreatePlist(); + if (!IdComponent::isValid(tcpl.getId())) + { + // Throw an invalid action exception + throw InvalidActionException("H5Object::createAttribute", "Datatype creation property list is not valid"); + } + tcpl.close(); tid2.close(); // Open the datatypes for query diff --git a/c++/test/tvlstr.cpp b/c++/test/tvlstr.cpp index 5dee1a2..859867b 100644 --- a/c++/test/tvlstr.cpp +++ b/c++/test/tvlstr.cpp @@ -57,7 +57,7 @@ static void *test_vlstr_alloc_custom(size_t size, void *info) { void *ret_value=NULL; // Pointer to return size_t *mem_used=(size_t *)info; // Get the pointer to the memory used - size_t extra; // Extra space needed + size_t extra; // Extra space needed /* * This weird contortion is required on the DEC Alpha to keep the @@ -91,7 +91,7 @@ static void test_vlstr_free_custom(void *_mem, void *info) { unsigned char *mem; size_t *mem_used=(size_t *)info; // Get the pointer to the memory used - size_t extra; // Extra space needed + size_t extra; // Extra space needed /* * This weird contortion is required on the DEC Alpha to keep the @@ -257,7 +257,7 @@ static void test_vlstring_array_dataset() for (ii = 0; ii < SPACE1_DIM1; ii++) { if(HDstrcmp(string_ds_check[ii], string_ds_array[ii])!=0) - TestErrPrintf("Line %d: Dataset data different: written=%s,read=%s\n",__LINE__, string_ds_array[ii], string_ds_check[ii]); + TestErrPrintf("Line %d: Dataset data different: written=%s,read=%s\n",__LINE__, string_ds_array[ii], string_ds_check[ii]); HDfree(string_ds_check[ii]); } @@ -351,23 +351,26 @@ static void test_vlstrings_special() hsize_t ii; // counting variable for (ii=0; ii<SPACE1_DIM1; ii++) if(rdata[ii]!=NULL) - TestErrPrintf("VL doesn't match!, rdata[%d]=%p\n",(int)ii,rdata[ii]); + TestErrPrintf("VL doesn't match!, rdata[%d]=%p\n",(int)ii,rdata[ii]); // Write dataset to disk, then read it back. dataset.write(wdata, vlst); dataset.read(rdata, vlst); // Compare data read in. - for (ii = 0; ii < SPACE1_DIM1; ii++) { + for (ii = 0; ii < SPACE1_DIM1; ii++) + { size_t wlen = HDstrlen(wdata[ii]); size_t rlen = HDstrlen(rdata[ii]); - if(wlen != rlen) { - TestErrPrintf("VL data lengths don't match!, strlen(wdata[%d])=%u, strlen(rdata[%d])=%u\n", (int)ii, (unsigned)wlen, (int)ii, (unsigned)rlen); - continue; + if(wlen != rlen) + { + TestErrPrintf("VL data lengths don't match!, strlen(wdata[%d])=%u, strlen(rdata[%d])=%u\n", (int)ii, (unsigned)wlen, (int)ii, (unsigned)rlen); + continue; } // end if - if(HDstrcmp(wdata[ii],rdata[ii]) != 0) { - TestErrPrintf("VL data values don't match!, wdata[%d]=%s, rdata[%d]=%s\n", (int)ii, wdata[ii], (int)ii, rdata[ii]); - continue; + if(HDstrcmp(wdata[ii],rdata[ii]) != 0) + { + TestErrPrintf("VL data values don't match!, wdata[%d]=%s, rdata[%d]=%s\n", (int)ii, wdata[ii], (int)ii, rdata[ii]); + continue; } // end if } // end for @@ -384,7 +387,7 @@ static void test_vlstrings_special() // Create the property list and set the fill value for the second // dataset. DSetCreatPropList dcpl; - char *fill = NULL; // Fill value + char *fill = NULL; // Fill value dcpl.setFillValue(vlst, &fill); dataset = file1.createDataSet("Dataset4", vlst, sid1, dcpl); @@ -566,12 +569,12 @@ static void test_compact_vlstring() hsize_t i; for (i=0; i<SPACE1_DIM1; i++) { if (HDstrlen(wdata[i])!=strlen(rdata[i])) { - TestErrPrintf("VL data length don't match!, strlen(wdata[%d])=%d, strlen(rdata[%d])=%d\n",(int)i,(int)strlen(wdata[i]),(int)i,(int)strlen(rdata[i])); - continue; + TestErrPrintf("VL data length don't match!, strlen(wdata[%d])=%d, strlen(rdata[%d])=%d\n",(int)i,(int)strlen(wdata[i]),(int)i,(int)strlen(rdata[i])); + continue; } // end if if (HDstrcmp(wdata[i],rdata[i]) != 0) { - TestErrPrintf("VL data values don't match!, wdata[%d]=%s, rdata[%d]=%s\n",(int)i,wdata[i],(int)i,rdata[i]); - continue; + TestErrPrintf("VL data values don't match!, wdata[%d]=%s, rdata[%d]=%s\n",(int)i,wdata[i],(int)i,rdata[i]); + continue; } // end if } // end for @@ -800,7 +803,7 @@ static void test_vlstring_array_attribute() for (ii = 0; ii < SPACE1_DIM1; ii++) { if(HDstrcmp(string_att_check[ii], string_att_array[ii])!=0) - TestErrPrintf("Line %d: Attribute data different: written=%s,read=%s\n",__LINE__, string_att_check[ii], string_att_check[ii]); + TestErrPrintf("Line %d: Attribute data different: written=%s,read=%s\n",__LINE__, string_att_check[ii], string_att_check[ii]); HDfree(string_att_check[ii]); // note: no need for std::string test } @@ -821,7 +824,7 @@ static void test_vlstring_array_attribute() /* Helper routine for test_vl_rewrite() */ static void write_scalar_dset(H5File& file, DataType& type, DataSpace& space, - char *name, char *data) + char *name, char *data) { DataSet dset; try { @@ -839,7 +842,7 @@ static void write_scalar_dset(H5File& file, DataType& type, DataSpace& space, /* Helper routine for test_vl_rewrite() */ static void read_scalar_dset(H5File& file, DataType& type, DataSpace& space, - char *name, char *data) + char *name, char *data) { char *data_read; DataSet dset; diff --git a/config/cmake/CTestScript.cmake b/config/cmake/CTestScript.cmake new file mode 100755 index 0000000..d53d7a4 --- /dev/null +++ b/config/cmake/CTestScript.cmake @@ -0,0 +1,152 @@ +cmake_minimum_required (VERSION 3.2.2 FATAL_ERROR) +######################################################## +# For any comments please contact cdashhelp@hdfgroup.org +# +######################################################## +# ----------------------------------------------------------- +# -- Get environment +# ----------------------------------------------------------- +if (NOT SITE_OS_NAME) + ## machine name not provided - attempt to discover with uname + ## -- set hostname + ## -------------------------- + find_program (HOSTNAME_CMD NAMES hostname) + exec_program (${HOSTNAME_CMD} ARGS OUTPUT_VARIABLE HOSTNAME) + set (CTEST_SITE "${HOSTNAME}${CTEST_SITE_EXT}") + find_program (UNAME NAMES uname) + macro (getuname name flag) + exec_program ("${UNAME}" ARGS "${flag}" OUTPUT_VARIABLE "${name}") + endmacro () + + getuname (osname -s) + getuname (osrel -r) + getuname (cpu -m) + message (STATUS "Dashboard script uname output: ${osname}-${osrel}-${cpu}\n") + + set (CTEST_BUILD_NAME "${osname}-${osrel}-${cpu}") + if (SITE_BUILDNAME_SUFFIX) + set (CTEST_BUILD_NAME "${SITE_BUILDNAME_SUFFIX}-${CTEST_BUILD_NAME}") + endif () + set (BUILD_OPTIONS "${ADD_BUILD_OPTIONS}") +else () + ## machine name provided + ## -------------------------- + if (CMAKE_HOST_UNIX) + set(CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_OS_BITS}-${SITE_COMPILER_NAME}-${SITE_COMPILER_VERSION}") + else () + set(CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_COMPILER_NAME}") + endif () + if (SITE_BUILDNAME_SUFFIX) + set(CTEST_BUILD_NAME "${CTEST_BUILD_NAME}-${SITE_BUILDNAME_SUFFIX}") + endif () + set (BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDNAME:STRING=${CTEST_BUILD_NAME}") +endif () + +#----------------------------------------------------------------------------- +# MAC machines need special option +#----------------------------------------------------------------------------- +if (APPLE) + # Compiler choice + execute_process (COMMAND xcrun --find cc OUTPUT_VARIABLE XCODE_CC OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process (COMMAND xcrun --find c++ OUTPUT_VARIABLE XCODE_CXX OUTPUT_STRIP_TRAILING_WHITESPACE) + set (ENV{CC} "${XCODE_CC}") + set (ENV{CXX} "${XCODE_CXX}") + + if (NOT NO_MAC_FORTRAN) + # Shared fortran is not supported, build static + set (BUILD_OPTIONS "${BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=OFF -DCMAKE_ANSI_CFLAGS:STRING=-fPIC") + else () + set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=OFF") + endif () + + set (BUILD_OPTIONS "${BUILD_OPTIONS} -DCTEST_USE_LAUNCHERS:BOOL=ON -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=OFF") +endif () + +#----------------------------------------------------------------------------- +set (NEED_REPOSITORY_CHECKOUT 0) +set (CTEST_CMAKE_COMMAND "\"${CMAKE_COMMAND}\"") +if (CTEST_USE_TAR_SOURCE) + ## Uncompress source if tar or zip file provided + ## -------------------------- + if (WIN32) + message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip]") + execute_process (COMMAND ${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip RESULT_VARIABLE rv) + else () + message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}/${CTEST_USE_TAR_SOURCE}.tar]") + execute_process (COMMAND ${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}/${CTEST_USE_TAR_SOURCE}.tar RESULT_VARIABLE rv) + endif () + + if (NOT rv EQUAL 0) + message (STATUS "extracting... [error-(${rv}) clean up]") + file (REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}") + message (FATAL_ERROR "error: extract of ${CTEST_USE_TAR_SOURCE} failed") + endif () +endif () + +#----------------------------------------------------------------------------- +## Clear the build directory +## -------------------------- +set (CTEST_START_WITH_EMPTY_BINARY_DIRECTORY TRUE) +if (EXISTS "${CTEST_BINARY_DIRECTORY}" AND IS_DIRECTORY "${CTEST_BINARY_DIRECTORY}") + ctest_empty_binary_directory (${CTEST_BINARY_DIRECTORY}) +else () + file (MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}") +endif () + +# Use multiple CPU cores to build +include (ProcessorCount) +ProcessorCount (N) +if (NOT N EQUAL 0) + if (NOT WIN32) + set (CTEST_BUILD_FLAGS -j${N}) + endif () + set (ctest_test_args ${ctest_test_args} PARALLEL_LEVEL ${N}) +endif () + +#----------------------------------------------------------------------------- +# Initialize the CTEST commands +#------------------------------ +set (CTEST_CONFIGURE_COMMAND + "${CTEST_CMAKE_COMMAND} -C \"${CTEST_SOURCE_DIRECTORY}/config/cmake/cacheinit.cmake\" -DCMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_SOURCE_DIRECTORY}\"" +) +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +## -- set output to english +set ($ENV{LC_MESSAGES} "en_EN") + +#----------------------------------------------------------------------------- + configure_file(${CTEST_SOURCE_DIRECTORY}/config/cmake/CTestCustom.cmake ${CTEST_BINARY_DIRECTORY}/CTestCustom.cmake) + ctest_read_custom_files ("${CTEST_BINARY_DIRECTORY}") +#----------------------------------------------------------------------------- + ## NORMAL process + ## -- LOCAL_SUBMIT reports to CDash server + ## -------------------------- + ctest_start (Experimental) + ctest_configure (BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) + if (LOCAL_SUBMIT) + ctest_submit (PARTS Configure Notes) + endif () + if (${res} LESS 0 OR ${res} GREATER 0) + file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Configure: ${res}\n") + endif () + + ctest_build (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND RETURN_VALUE res NUMBER_ERRORS errval) + if (LOCAL_SUBMIT) + ctest_submit (PARTS Build) + endif () + if (${res} LESS 0 OR ${res} GREATER 0 OR ${errval} GREATER 0) + file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed ${errval} Build: ${res}\n") + endif () + + ctest_test (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args} RETURN_VALUE res) + if (LOCAL_SUBMIT) + ctest_submit (PARTS Test) + endif() + if (${res} LESS 0 OR ${res} GREATER 0) + file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n") + endif () + if (${res} LESS 0 OR ${res} GREATER 0) + message (FATAL_ERROR "tests FAILED") + endif () +#----------------------------------------------------------------------------- diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 5dad4f3..55de8f8 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -41,17 +41,41 @@ /* As FC_FUNC, but for C identifiers containing underscores. */ #define @H5_FC_FUNC_@ -/* Define Fortran Maximum Real Decimal Precision */ -#cmakedefine H5_PAC_FC_MAX_REAL_PRECISION @H5_PAC_FC_MAX_REAL_PRECISION@ - -/* Define C Maximum Real Decimal Precision */ -#cmakedefine H5_PAC_C_MAX_REAL_PRECISION @H5_PAC_C_MAX_REAL_PRECISION@ +/* Define if Fortran C_LONG_DOUBLE is different from C_DOUBLE */ +#cmakedefine H5_FORTRAN_C_LONG_DOUBLE_IS_UNIQUE @H5_FORTRAN_C_LONG_DOUBLE_IS_UNIQUE@ /* Define if we have Fortran C_LONG_DOUBLE */ #cmakedefine H5_FORTRAN_HAVE_C_LONG_DOUBLE @H5_FORTRAN_HAVE_C_LONG_DOUBLE@ -/* Determine if __float128 is available */ -#cmakedefine H5_HAVE_FLOAT128 @H5_HAVE_FLOAT128@ +/* Define if we have Fortran intrinsic C_SIZEOF */ +#cmakedefine H5_FORTRAN_HAVE_C_SIZEOF @H5_FORTRAN_HAVE_C_SIZEOF@ + +/* Define if we have Fortran intrinsic SIZEOF */ +#cmakedefine H5_FORTRAN_HAVE_SIZEOF @H5_FORTRAN_HAVE_SIZEOF@ + +/* Define if we have Fortran intrinsic STORAGE_SIZE */ +#cmakedefine H5_FORTRAN_HAVE_STORAGE_SIZE @H5_FORTRAN_HAVE_STORAGE_SIZE@ + +/* Determine the size of C long double */ +#cmakedefine H5_FORTRAN_SIZEOF_LONG_DOUBLE @H5_FORTRAN_SIZEOF_LONG_DOUBLE@ + +/* Define Fortran compiler ID */ +#cmakedefine H5_Fortran_COMPILER_ID @H5_Fortran_COMPILER_ID@ + +/* Define valid Fortran INTEGER KINDs */ +#cmakedefine H5_H5CONFIG_F_IKIND @H5_HH5_H5CONFIG_F_NUM_RKIND5CONFIG_F_IKIND@ + +/* Define number of valid Fortran INTEGER KINDs */ +#cmakedefine H5_H5CONFIG_F_NUM_IKIND @H5_H5CONFIG_F_NUM_IKIND@ + +/* Define number of valid Fortran REAL KINDs */ +#cmakedefine H5_H5CONFIG_F_NUM_RKIND @H5_H5CONFIG_F_NUM_RKIND@ + +/* Define valid Fortran REAL KINDs */ +#cmakedefine H5_H5CONFIG_F_RKIND @H5_H5CONFIG_F_RKIND@ + +/* Define valid Fortran REAL KINDs Sizeof */ +#cmakedefine H5_H5CONFIG_F_RKIND_SIZEOF @H5_H5CONFIG_F_RKIND_SIZEOF@ /* Define to 1 if you have the `alarm' function. */ #cmakedefine H5_HAVE_ALARM @H5_HAVE_ALARM@ @@ -81,7 +105,7 @@ /* Define to 1 if you have the `difftime' function. */ #cmakedefine H5_HAVE_DIFFTIME @H5_HAVE_DIFFTIME@ -/* Define if the direct I/O virtual file driver should be compiled */ +/* Define if the direct I/O virtual file driver (VFD) should be compiled */ #cmakedefine H5_HAVE_DIRECT @H5_HAVE_DIRECT@ /* Define to 1 if you have the <dirent.h> header file. */ @@ -108,6 +132,9 @@ /* Define if support for szip filter is enabled */ #cmakedefine H5_HAVE_FILTER_SZIP @H5_HAVE_FILTER_SZIP@ +/* Determine if __float128 is available */ +#cmakedefine H5_HAVE_FLOAT128 @H5_HAVE_FLOAT128@ + /* Define to 1 if you have the `flock' function. */ #cmakedefine H5_HAVE_FLOCK @H5_HAVE_FLOCK@ @@ -162,7 +189,7 @@ /* Define if the compiler understands inline */ #cmakedefine H5_HAVE_INLINE @H5_HAVE_INLINE@ -/* Define if library will contain instrumentation to detect correct +/* Define if parallel library will contain instrumentation to detect correct optimization operation */ #cmakedefine H5_HAVE_INSTRUMENTED_LIBRARY @H5_HAVE_INSTRUMENTED_LIBRARY@ @@ -244,8 +271,8 @@ /* Define to 1 if you have the <pthread.h> header file. */ #cmakedefine H5_HAVE_PTHREAD_H @H5_HAVE_PTHREAD_H@ -/* Define to 1 if you have the 'InitOnceExecuteOnce' function. */ -#cmakedefine H5_HAVE_WIN_THREADS @H5_HAVE_WIN_THREADS@ +/* Define to 1 if you have the <quadmath.h> header file. */ +#cmakedefine H5_HAVE_QUADMATH_H @H5_HAVE_QUADMATH_H@ /* Define to 1 if you have the `random' function. */ #cmakedefine H5_HAVE_RANDOM @H5_HAVE_RANDOM@ @@ -254,10 +281,10 @@ #cmakedefine H5_HAVE_RAND_R @H5_HAVE_RAND_R@ /* Define to 1 if you have the `round' function. */ -#cmakedefine H5_HAVE_ROUNDF @H5_HAVE_ROUNDF@ +#cmakedefine H5_HAVE_ROUND @H5_HAVE_ROUND@ /* Define to 1 if you have the `roundf' function. */ -#cmakedefine H5_HAVE_ROUND @H5_HAVE_ROUND@ +#cmakedefine H5_HAVE_ROUNDF @H5_HAVE_ROUNDF@ /* Define to 1 if you have the `setjmp' function. */ #cmakedefine H5_HAVE_SETJMP @H5_HAVE_SETJMP@ @@ -292,6 +319,9 @@ /* Define if `struct stat' has the `st_blocks' field */ #cmakedefine H5_HAVE_STAT_ST_BLOCKS @H5_HAVE_STAT_ST_BLOCKS@ +/* Define to 1 if you have the <stdbool.h> header file. */ +#cmakedefine H5_HAVE_STDBOOL_H @H5_HAVE_STDBOOL_H@ + /* Define to 1 if you have the <stddef.h> header file. */ #cmakedefine H5_HAVE_STDDEF_H @H5_HAVE_STDDEF_H@ @@ -307,6 +337,12 @@ /* Define to 1 if you have the `strdup' function. */ #cmakedefine H5_HAVE_STRDUP @H5_HAVE_STRDUP@ +/* Define to 1 if you have the `strtoll' function. */ +#cmakedefine H5_HAVE_STRTOLL @H5_HAVE_STRTOLL@ + +/* Define to 1 if you have the `strtoull' function. */ +#cmakedefine H5_HAVE_STRTOULL @H5_HAVE_STRTOULL@ + /* Define to 1 if you have the <strings.h> header file. */ #cmakedefine H5_HAVE_STRINGS_H @H5_HAVE_STRINGS_H@ @@ -325,6 +361,9 @@ /* Define to 1 if you have the `system' function. */ #cmakedefine H5_HAVE_SYSTEM @H5_HAVE_SYSTEM@ +/* Define to 1 if you have the <sys/file.h> header file. */ +#cmakedefine H5_HAVE_SYS_FILE_H @H5_HAVE_SYS_FILE_H@ + /* Define to 1 if you have the <sys/ioctl.h> header file. */ #cmakedefine H5_HAVE_SYS_IOCTL_H @H5_HAVE_SYS_IOCTL_H@ @@ -379,6 +418,9 @@ /* Define to 1 if you have the `waitpid' function. */ #cmakedefine H5_HAVE_WAITPID @H5_HAVE_WAITPID@ +/* Define to 1 if you have the 'InitOnceExecuteOnce' function. */ +#cmakedefine H5_HAVE_WIN_THREADS @H5_HAVE_WIN_THREADS@ + /* Define if your system has window style path name. */ #cmakedefine H5_HAVE_WINDOW_PATH @H5_HAVE_WINDOW_PATH@ @@ -419,10 +461,12 @@ with special algorithm. */ #cmakedefine H5_LONG_TO_LDOUBLE_SPECIAL @H5_LONG_TO_LDOUBLE_SPECIAL@ -/* Define to the sub-directory in which libtool stores uninstalled libraries. - */ +/* Define to the sub-directory where libtool stores uninstalled libraries. */ #cmakedefine H5_LT_OBJDIR @H5_LT_OBJDIR@ +/* Define to enable internal memory allocation sanity checking. */ +/* #cmakedefine H5_MEMORY_ALLOC_SANITY_CHECK @H5_MEMORY_ALLOC_SANITY_CHECK@ ** Define in CMakeLists.txt */ + /* Define if the metadata trace file code is to be compiled in */ #cmakedefine H5_METADATA_TRACE_FILE @H5_METADATA_TRACE_FILE@ @@ -453,9 +497,18 @@ /* Define to the version of this package. */ #define H5_PACKAGE_VERSION "@HDF5_PACKAGE_VERSION_STRING@" +/* Determine the maximum decimal precision in C */ +#define H5_PAC_C_MAX_REAL_PRECISION @H5_PAC_C_MAX_REAL_PRECISION@ + +/* Define Fortran Maximum Real Decimal Precision */ +#cmakedefine H5_PAC_FC_MAX_REAL_PRECISION @H5_PAC_FC_MAX_REAL_PRECISION@ + /* Width for printf() for type `long long' or `__int64', use `ll' */ #cmakedefine H5_PRINTF_LL_WIDTH @H5_PRINTF_LL_WIDTH@ +/* The size of `bool', as computed by sizeof. */ +#cmakedefine H5_SIZEOF_BOOL @H5_SIZEOF_BOOL@ + /* The size of `char', as computed by sizeof. */ #cmakedefine H5_SIZEOF_CHAR @H5_SIZEOF_CHAR@ @@ -589,14 +642,14 @@ /* The size of `unsigned', as computed by sizeof. */ #cmakedefine H5_SIZEOF_UNSIGNED @H5_SIZEOF_UNSIGNED@ -/* The size of `__int64', as computed by sizeof. */ -#define H5_SIZEOF___INT64 @H5_SIZEOF___INT64@ +/* The size of `_Quad', as computed by sizeof. */ +#define H5_SIZEOF__QUAD @H5_SIZEOF__QUAD@ -/* Define to 1 if you have the <stdbool.h> header file. */ -#cmakedefine H5_HAVE_STDBOOL_H @H5_HAVE_STDBOOL_H@ +/* The size of `__float128', as computed by sizeof. */ +#define H5_SIZEOF___FLOAT128 @H5_SIZEOF___FLOAT128@ -/* The size of `bool', as computed by sizeof. */ -#cmakedefine H5_SIZEOF_BOOL @H5_SIZEOF_BOOL@ +/* The size of `__int64', as computed by sizeof. */ +#define H5_SIZEOF___INT64 @H5_SIZEOF___INT64@ /* Define to 1 if you have the ANSI C header files. */ #cmakedefine H5_STDC_HEADERS @H5_STDC_HEADERS@ diff --git a/config/cmake/HDF5_Examples.cmake.in b/config/cmake/HDF5_Examples.cmake.in index eadff5f..04f72a0 100644 --- a/config/cmake/HDF5_Examples.cmake.in +++ b/config/cmake/HDF5_Examples.cmake.in @@ -5,102 +5,75 @@ cmake_minimum_required(VERSION 3.2.2 FATAL_ERROR) # ctest -S HDF5_Examples.cmake,OPTION=VALUE -C Release -V -O test.log ############################################################################################################### -set(CTEST_CMAKE_GENERATOR "@CMAKE_GENERATOR@") -set(CTEST_DASHBOARD_ROOT ${CTEST_SCRIPT_DIRECTORY}) +set (CTEST_CMAKE_GENERATOR "@CMAKE_GENERATOR@") +set (CTEST_DASHBOARD_ROOT ${CTEST_SCRIPT_DIRECTORY}) # handle input parameters to script. #INSTALLDIR - HDF5 root folder #CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo -#CTEST_SOURCE_NAME - name of source folder; HDF4Examples -#STATIC_ONLY - Default is YES -#FORTRAN_LIBRARIES - Default is NO -#JAVA_LIBRARIES - Default is NO -##NO_MAC_FORTRAN - set to TRUE to allow shared libs on a Mac) +#CTEST_SOURCE_NAME - name of source folder; HDF5Examples if (DEFINED CTEST_SCRIPT_ARG) - # transform ctest script arguments of the form - # script.ctest,var1=value1,var2=value2 - # to variables with the respective names set to the respective values - string (REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}") - foreach (current_var ${script_args}) - if ("${current_var}" MATCHES "^([^=]+)=(.+)$") - set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}") - endif () - endforeach () -endif () -if (NOT DEFINED INSTALLDIR) - set (INSTALLDIR "@CMAKE_INSTALL_PREFIX@") + # transform ctest script arguments of the form + # script.ctest,var1=value1,var2=value2 + # to variables with the respective names set to the respective values + string (REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}") + foreach (current_var ${script_args}) + if ("${current_var}" MATCHES "^([^=]+)=(.+)$") + set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}") + endif () + endforeach () endif () + +################################################################### +### Following Line is one of [Release, RelWithDebInfo, Debug] ##### +set (CTEST_CONFIGURATION_TYPE "$ENV{CMAKE_CONFIG_TYPE}") if (NOT DEFINED CTEST_CONFIGURATION_TYPE) - set (CTEST_CONFIGURATION_TYPE "Release") -endif () -if (NOT DEFINED CTEST_SOURCE_NAME) - set (CTEST_SOURCE_NAME "HDF5Examples") + set (CTEST_CONFIGURATION_TYPE "Release") endif () -if (NOT DEFINED STATIC_ONLY) - set (STATICONLYLIBRARIES "YES") -else () - set (STATICONLYLIBRARIES "NO") -endif () -if (NOT DEFINED FORTRAN_LIBRARIES) - set (FORTRANLIBRARIES "NO") -else () - set (FORTRANLIBRARIES "YES") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCTEST_CONFIGURATION_TYPE:STRING=${CTEST_CONFIGURATION_TYPE}") +################################################################## + +if (NOT DEFINED INSTALLDIR) + set (INSTALLDIR "@CMAKE_INSTALL_PREFIX@") endif () -if (NOT DEFINED JAVA_LIBRARIES) - set (JAVALIBRARIES "NO") -else () - set (JAVALIBRARIES "YES") + +if (NOT DEFINED CTEST_SOURCE_NAME) + set (CTEST_SOURCE_NAME "HDF5Examples") endif () + if (NOT DEFINED HDF_LOCAL) - set (CDASH_LOCAL "NO") + set (CDASH_LOCAL "NO") else () - set (CDASH_LOCAL "YES") + set (CDASH_LOCAL "YES") endif () if(NOT DEFINED CTEST_SITE) - set (CTEST_SITE "local") + set (CTEST_SITE "local") endif () -if(NOT DEFINED CTEST_BUILD_NAME) - set (CTEST_BUILD_NAME "examples") +if (NOT DEFINED CTEST_BUILD_NAME) + set (CTEST_BUILD_NAME "examples") endif () set (BUILD_OPTIONS "${BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDNAME:STRING=${CTEST_BUILD_NAME}") #TAR_SOURCE - name of tarfile #if (NOT DEFINED TAR_SOURCE) -# set (CTEST_USE_TAR_SOURCE "HDF5Examples-1.10.1-Source") +# set (CTEST_USE_TAR_SOURCE "HDF5Examples-1.10.5-Source") #endif () ############################################################################################################### -# Adjust the following SET Commands as needed -############################################################################################################### if (WIN32) - if (${STATICONLYLIBRARIES}) - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=OFF") - endif () + set (SITE_OS_NAME "Windows") set (ENV{HDF5_DIR} "${INSTALLDIR}/cmake") set (CTEST_BINARY_NAME ${CTEST_SOURCE_NAME}\\build) set (CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_SOURCE_NAME}") set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_BINARY_NAME}") else () - if (${STATICONLYLIBRARIES}) - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=OFF -DCMAKE_ANSI_CFLAGS:STRING=-fPIC") - endif () set (ENV{HDF5_DIR} "${INSTALLDIR}/share/cmake") set (ENV{LD_LIBRARY_PATH} "${INSTALLDIR}/lib") set (CTEST_BINARY_NAME ${CTEST_SOURCE_NAME}/build) set (CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_SOURCE_NAME}") set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_BINARY_NAME}") endif () -if (${FORTRANLIBRARIES}) - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF_BUILD_FORTRAN:BOOL=ON") -else () - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF_BUILD_FORTRAN:BOOL=OFF") -endif () -if(${JAVALIBRARIES}) - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF_BUILD_JAVA:BOOL=ON") -else () - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF_BUILD_JAVA:BOOL=OFF") -endif () -if(${CDASH_LOCAL}) +if (${CDASH_LOCAL}) set (BUILD_OPTIONS "${BUILD_OPTIONS} -DCDASH_LOCAL:BOOL=ON") endif () set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF5_PACKAGE_NAME:STRING=@HDF5_PACKAGE@@HDF_PACKAGE_EXT@") @@ -110,102 +83,10 @@ set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF5_PACKAGE_NAME:STRING=@HDF5_PACKAGE@@H # ############################################################################################################### -#----------------------------------------------------------------------------- -# MAC machines need special option -#----------------------------------------------------------------------------- -if (APPLE) - # Compiler choice - execute_process (COMMAND xcrun --find cc OUTPUT_VARIABLE XCODE_CC OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process (COMMAND xcrun --find c++ OUTPUT_VARIABLE XCODE_CXX OUTPUT_STRIP_TRAILING_WHITESPACE) - set (ENV{CC} "${XCODE_CC}") - set (ENV{CXX} "${XCODE_CXX}") - if (NOT NO_MAC_FORTRAN) - # Shared fortran is not supported, build static - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=OFF -DCMAKE_ANSI_CFLAGS:STRING=-fPIC") - else () - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DHDF_BUILD_FORTRAN:BOOL=OFF") - endif () - set (BUILD_OPTIONS "${BUILD_OPTIONS} -DCTEST_USE_LAUNCHERS:BOOL=ON -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=OFF") -endif () - -#----------------------------------------------------------------------------- -set (CTEST_CMAKE_COMMAND "\"${CMAKE_COMMAND}\"") -## -------------------------- -if (CTEST_USE_TAR_SOURCE) - ## Uncompress source if tar or zip file provided - ## -------------------------- - if (WIN32) - message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_USE_TAR_SOURCE}.zip]") - execute_process (COMMAND ${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip RESULT_VARIABLE rv) - else () - message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_USE_TAR_SOURCE}.tar]") - execute_process (COMMAND ${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}/${CTEST_USE_TAR_SOURCE}.tar RESULT_VARIABLE rv) - endif () - - if (NOT rv EQUAL 0) - message (STATUS "extracting... [error-(${rv}) clean up]") - file (REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}") - message (FATAL_ERROR "error: extract of ${CTEST_SOURCE_NAME} failed") - endif () -endif() - -#----------------------------------------------------------------------------- -## Clear the build directory -## -------------------------- -set (CTEST_START_WITH_EMPTY_BINARY_DIRECTORY TRUE) -if (EXISTS "${CTEST_BINARY_DIRECTORY}" AND IS_DIRECTORY "${CTEST_BINARY_DIRECTORY}") - ctest_empty_binary_directory (${CTEST_BINARY_DIRECTORY}) +if (WIN32) + include (${CTEST_SCRIPT_DIRECTORY}\\HDF5_Examples_options.cmake) + include (${CTEST_SCRIPT_DIRECTORY}\\CTestScript.cmake) else () - file (MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}") -endif () - -# Use multiple CPU cores to build -include (ProcessorCount) -ProcessorCount (N) -if (NOT N EQUAL 0) - if (NOT WIN32) - set (CTEST_BUILD_FLAGS -j${N}) - endif () - set (ctest_test_args ${ctest_test_args} PARALLEL_LEVEL ${N}) -endif () -set (CTEST_CONFIGURE_COMMAND - "${CTEST_CMAKE_COMMAND} -C \"${CTEST_SOURCE_DIRECTORY}/config/cmake/cacheinit.cmake\" -DCMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_SOURCE_DIRECTORY}\"" -) - -#----------------------------------------------------------------------------- -## -- set output to english -set ($ENV{LC_MESSAGES} "en_EN") - -#----------------------------------------------------------------------------- -configure_file (${CTEST_SOURCE_DIRECTORY}/config/cmake/CTestCustom.cmake ${CTEST_BINARY_DIRECTORY}/CTestCustom.cmake) -ctest_read_custom_files ("${CTEST_BINARY_DIRECTORY}") -## NORMAL process -## -------------------------- -ctest_start (Experimental) -ctest_configure (BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) -if (${res} LESS 0 OR ${res} GREATER 0) - file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Configure: ${res}\n") -endif () -if (LOCAL_SUBMIT) - ctest_submit (PARTS Configure Notes) -endif () -ctest_build (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND APPEND RETURN_VALUE res NUMBER_ERRORS errval) -if (${res} LESS 0 OR ${res} GREATER 0 OR ${errval} GREATER 0) - file(APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed ${errval} Build: ${res}\n") -endif () -if (LOCAL_SUBMIT) - ctest_submit (PARTS Build) -endif () -ctest_test (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args} RETURN_VALUE res) -if (${res} LESS 0 OR ${res} GREATER 0) - file(APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n") -endif () -if (LOCAL_SUBMIT) - ctest_submit (PARTS Test) -endif () -if (${res} LESS 0 OR ${res} GREATER 0) - message (FATAL_ERROR "tests FAILED") + include (${CTEST_SCRIPT_DIRECTORY}/HDF_Examples_options.cmake) + include (${CTEST_SCRIPT_DIRECTORY}/CTestScript.cmake) endif () -#----------------------------------------------------------------------------- -############################################################################################################## -message (STATUS "DONE") diff --git a/config/cmake/HDF5_Examples_options.cmake b/config/cmake/HDF5_Examples_options.cmake new file mode 100755 index 0000000..a8f0c8e --- /dev/null +++ b/config/cmake/HDF5_Examples_options.cmake @@ -0,0 +1,48 @@ +############################################################################################# +#### Change default configuration of options in config/cmake/cacheinit.cmake file ### +#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ### +#### DEFAULT: ### +#### BUILD_SHARED_LIBS:BOOL=OFF ### +#### HDF_BUILD_C:BOOL=ON ### +#### HDF_BUILD_CXX:BOOL=OFF ### +#### HDF_BUILD_FORTRAN:BOOL=OFF ### +#### HDF_BUILD_JAVA:BOOL=OFF ### +#### BUILD_TESTING:BOOL=OFF ### +#### HDF_ENABLE_PARALLEL:BOOL=OFF ### +#### HDF_ENABLE_THREADSAFE:BOOL=OFF ### +############################################################################################# + +### uncomment/comment and change the following lines for other configuration options +### build with shared libraries +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=ON") + +############################################################################################# +#### languages #### +### disable C builds +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_BUILD_C:BOOL=OFF") + +### enable C++ builds +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_BUILD_CXX:BOOL=ON") + +### enable Fortran builds +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_BUILD_FORTRAN:BOOL=ON") + +### enable JAVA builds +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_BUILD_JAVA:BOOL=ON") + +############################################################################################# +### enable parallel program builds +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_ENABLE_PARALLEL:BOOL=ON") + + +############################################################################################# +### enable threadsafe program builds +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_ENABLE_THREADSAFE:BOOL=ON") + + +############################################################################################# +### enable test program builds, requires reference files in testfiles subdirectory +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_TESTING:BOOL=ON") +#set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCOMPARE_TESTING:BOOL=ON") + +############################################################################################# diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake index dca9631..cb92933 100755 --- a/config/cmake/scripts/CTestScript.cmake +++ b/config/cmake/scripts/CTestScript.cmake @@ -33,12 +33,12 @@ else () ## machine name provided
## --------------------------
if (CMAKE_HOST_UNIX)
- set(CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_OS_BITS}-${SITE_COMPILER_NAME}-${SITE_COMPILER_VERSION}")
+ set (CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_OS_BITS}-${SITE_COMPILER_NAME}-${SITE_COMPILER_VERSION}")
else ()
- set(CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_COMPILER_NAME}")
+ set (CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_COMPILER_NAME}")
endif ()
if (SITE_BUILDNAME_SUFFIX)
- set(CTEST_BUILD_NAME "${CTEST_BUILD_NAME}-${SITE_BUILDNAME_SUFFIX}")
+ set (CTEST_BUILD_NAME "${CTEST_BUILD_NAME}-${SITE_BUILDNAME_SUFFIX}")
endif ()
set (BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDNAME:STRING=${CTEST_BUILD_NAME}")
endif ()
@@ -185,17 +185,22 @@ endforeach () #-----------------------------------------------------------------------------
# Initialize the CTEST commands
#------------------------------
+if(CMAKE_GENERATOR_TOOLSET)
+ set(CTEST_CONFIGURE_TOOLSET "-T${CMAKE_GENERATOR_TOOLSET}")
+else ()
+ set(CTEST_CONFIGURE_TOOLSET "")
+endif()
if (LOCAL_MEMCHECK_TEST)
find_program (CTEST_MEMORYCHECK_COMMAND NAMES valgrind)
set (CTEST_CONFIGURE_COMMAND
- "${CTEST_CMAKE_COMMAND} -C \"${CTEST_SOURCE_DIRECTORY}/config/cmake/mccacheinit.cmake\" -DCMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_SOURCE_DIRECTORY}\""
+ "${CTEST_CMAKE_COMMAND} -C \"${CTEST_SOURCE_DIRECTORY}/config/cmake/mccacheinit.cmake\" -DCMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_CONFIGURE_TOOLSET}\" \"${CTEST_SOURCE_DIRECTORY}\""
)
else ()
if (LOCAL_COVERAGE_TEST)
find_program (CTEST_COVERAGE_COMMAND NAMES gcov)
endif ()
set (CTEST_CONFIGURE_COMMAND
- "${CTEST_CMAKE_COMMAND} -C \"${CTEST_SOURCE_DIRECTORY}/config/cmake/cacheinit.cmake\" -DCMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_SOURCE_DIRECTORY}\""
+ "${CTEST_CMAKE_COMMAND} -C \"${CTEST_SOURCE_DIRECTORY}/config/cmake/cacheinit.cmake\" -DCMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_CONFIGURE_TOOLSET}\" \"${CTEST_SOURCE_DIRECTORY}\""
)
endif ()
diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 55fecfb..f9dada9 100755 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -4,7 +4,7 @@ ### ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201264 -C Release -VV -O hdf5.log ###
#############################################################################################
-cmake_minimum_required(VERSION 3.2.2 FATAL_ERROR)
+cmake_minimum_required (VERSION 3.2.2 FATAL_ERROR)
############################################################################
# Usage:
# ctest -S HDF5config.cmake,OPTION=VALUE -C Release -VV -O test.log
@@ -27,8 +27,8 @@ cmake_minimum_required(VERSION 3.2.2 FATAL_ERROR) # NO_MAC_FORTRAN - Yes to be SHARED on a Mac
##############################################################################
-set(CTEST_SOURCE_VERSION 1.10.1)
-set(CTEST_SOURCE_VERSEXT "")
+set (CTEST_SOURCE_VERSION 1.10.1)
+set (CTEST_SOURCE_VERSEXT "")
##############################################################################
# handle input parameters to script.
@@ -40,139 +40,139 @@ set(CTEST_SOURCE_VERSEXT "") #FORTRAN_LIBRARIES - Default is NO
#JAVA_LIBRARIES - Default is NO
#NO_MAC_FORTRAN - set to TRUE to allow shared libs on a Mac
-if(DEFINED CTEST_SCRIPT_ARG)
+if (DEFINED CTEST_SCRIPT_ARG)
# transform ctest script arguments of the form
# script.ctest,var1=value1,var2=value2
# to variables with the respective names set to the respective values
- string(REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}")
- foreach(current_var ${script_args})
+ string (REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}")
+ foreach (current_var ${script_args})
if ("${current_var}" MATCHES "^([^=]+)=(.+)$")
- set("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}")
+ set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}")
endif ()
endforeach ()
endif ()
# build generator must be defined
-if(NOT DEFINED BUILD_GENERATOR)
- message(FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2015, VS201564, VS2013, VS201364, VS2012, or VS201264")
+if (NOT DEFINED BUILD_GENERATOR)
+ message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2015, VS201564, VS2013, VS201364, VS2012, or VS201264")
else ()
- if(${BUILD_GENERATOR} STREQUAL "Unix")
- set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
- elseif(${BUILD_GENERATOR} STREQUAL "VS2015")
- set(CTEST_CMAKE_GENERATOR "Visual Studio 14 2015")
- elseif(${BUILD_GENERATOR} STREQUAL "VS201564")
- set(CTEST_CMAKE_GENERATOR "Visual Studio 14 2015 Win64")
- elseif(${BUILD_GENERATOR} STREQUAL "VS2013")
- set(CTEST_CMAKE_GENERATOR "Visual Studio 12 2013")
- elseif(${BUILD_GENERATOR} STREQUAL "VS201364")
- set(CTEST_CMAKE_GENERATOR "Visual Studio 12 2013 Win64")
- elseif(${BUILD_GENERATOR} STREQUAL "VS2012")
- set(CTEST_CMAKE_GENERATOR "Visual Studio 11 2012")
- elseif(${BUILD_GENERATOR} STREQUAL "VS201264")
- set(CTEST_CMAKE_GENERATOR "Visual Studio 11 2012 Win64")
+ if (${BUILD_GENERATOR} STREQUAL "Unix")
+ set (CTEST_CMAKE_GENERATOR "Unix Makefiles")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS2015")
+ set (CTEST_CMAKE_GENERATOR "Visual Studio 14 2015")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS201564")
+ set (CTEST_CMAKE_GENERATOR "Visual Studio 14 2015 Win64")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS2013")
+ set (CTEST_CMAKE_GENERATOR "Visual Studio 12 2013")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS201364")
+ set (CTEST_CMAKE_GENERATOR "Visual Studio 12 2013 Win64")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS2012")
+ set (CTEST_CMAKE_GENERATOR "Visual Studio 11 2012")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS201264")
+ set (CTEST_CMAKE_GENERATOR "Visual Studio 11 2012 Win64")
else ()
- message(FATAL_ERROR "Invalid BUILD_GENERATOR must be - Unix, VS2015, VS201564, VS2013, VS201364, VS2012, or VS201264")
+ message (FATAL_ERROR "Invalid BUILD_GENERATOR must be - Unix, VS2015, VS201564, VS2013, VS201364, VS2012, or VS201264")
endif ()
endif ()
###################################################################
### Following Line is one of [Release, RelWithDebInfo, Debug] #####
-set(CTEST_CONFIGURATION_TYPE "$ENV{CMAKE_CONFIG_TYPE}")
+set (CTEST_CONFIGURATION_TYPE "$ENV{CMAKE_CONFIG_TYPE}")
###################################################################
-if(NOT DEFINED INSTALLDIR)
- if(WIN32)
- set(INSTALLDIR "C:/Program Files/HDF_Group/HDF5/${CTEST_SOURCE_VERSION}")
+if (NOT DEFINED INSTALLDIR)
+ if (WIN32)
+ set (INSTALLDIR "C:/Program Files/HDF_Group/HDF5/${CTEST_SOURCE_VERSION}")
else ()
- set(INSTALLDIR "${CTEST_SCRIPT_DIRECTORY}/HDF_Group/HDF5/${CTEST_SOURCE_VERSION}")
+ set (INSTALLDIR "${CTEST_SCRIPT_DIRECTORY}/HDF_Group/HDF5/${CTEST_SOURCE_VERSION}")
endif ()
endif ()
-if(NOT DEFINED CTEST_CONFIGURATION_TYPE)
- set(CTEST_CONFIGURATION_TYPE "Release")
+if (NOT DEFINED CTEST_CONFIGURATION_TYPE)
+ set (CTEST_CONFIGURATION_TYPE "Release")
endif ()
-if(NOT DEFINED CTEST_SOURCE_NAME)
- set(CTEST_SOURCE_NAME "hdf5-${CTEST_SOURCE_VERSION}${CTEST_SOURCE_VERSEXT}")
+if (NOT DEFINED CTEST_SOURCE_NAME)
+ set (CTEST_SOURCE_NAME "hdf5-${CTEST_SOURCE_VERSION}${CTEST_SOURCE_VERSEXT}")
endif ()
-if(NOT DEFINED STATIC_ONLY)
- set(STATICONLYLIBRARIES "YES")
+if (NOT DEFINED STATIC_ONLY)
+ set (STATICONLYLIBRARIES "YES")
else ()
- set(STATICONLYLIBRARIES "NO")
+ set (STATICONLYLIBRARIES "NO")
endif ()
-if(NOT DEFINED FORTRAN_LIBRARIES)
- set(FORTRANLIBRARIES "NO")
+if (NOT DEFINED FORTRAN_LIBRARIES)
+ set (FORTRANLIBRARIES "NO")
else ()
- set(FORTRANLIBRARIES "YES")
+ set(FORTRANLIBRARIES "YES")
endif ()
-if(NOT DEFINED JAVA_LIBRARIES)
- set(JAVALIBRARIES "NO")
+if (NOT DEFINED JAVA_LIBRARIES)
+ set (JAVALIBRARIES "NO")
else ()
- set(JAVALIBRARIES "YES")
+ set (JAVALIBRARIES "YES")
endif ()
-set(CTEST_BINARY_NAME "build")
-set(CTEST_DASHBOARD_ROOT "${CTEST_SCRIPT_DIRECTORY}")
-if(WIN32)
- set(CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_SOURCE_NAME}")
- set(CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_BINARY_NAME}")
+set (CTEST_BINARY_NAME "build")
+set (CTEST_DASHBOARD_ROOT "${CTEST_SCRIPT_DIRECTORY}")
+if (WIN32)
+ set (CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_SOURCE_NAME}")
+ set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_BINARY_NAME}")
else ()
- set(CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_SOURCE_NAME}")
- set(CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_BINARY_NAME}")
+ set (CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_SOURCE_NAME}")
+ set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_BINARY_NAME}")
endif ()
###################################################################
######### Following describes compiler ############
-if(WIN32)
- set(SITE_OS_NAME "Windows")
- set(SITE_OS_VERSION "WIN7")
- if(${BUILD_GENERATOR} STREQUAL "VS201564")
- set(SITE_OS_BITS "64")
- set(SITE_COMPILER_NAME "vs2015")
- set(SITE_COMPILER_VERSION "14")
- elseif(${BUILD_GENERATOR} STREQUAL "VS2015")
- set(SITE_OS_BITS "32")
- set(SITE_COMPILER_NAME "vs2015")
- set(SITE_COMPILER_VERSION "14")
- elseif(${BUILD_GENERATOR} STREQUAL "VS201364")
- set(SITE_OS_BITS "64")
- set(SITE_COMPILER_NAME "vs2013")
- set(SITE_COMPILER_VERSION "12")
- elseif(${BUILD_GENERATOR} STREQUAL "VS2013")
- set(SITE_OS_BITS "32")
- set(SITE_COMPILER_NAME "vs2013")
- set(SITE_COMPILER_VERSION "12")
- elseif(${BUILD_GENERATOR} STREQUAL "VS201264")
- set(SITE_OS_BITS "64")
- set(SITE_COMPILER_NAME "vs2012")
- set(SITE_COMPILER_VERSION "11")
- elseif(${BUILD_GENERATOR} STREQUAL "VS2012")
- set(SITE_OS_BITS "32")
- set(SITE_COMPILER_NAME "vs2012")
- set(SITE_COMPILER_VERSION "11")
+if (WIN32)
+ set (SITE_OS_NAME "Windows")
+ set (SITE_OS_VERSION "WIN7")
+ if (${BUILD_GENERATOR} STREQUAL "VS201564")
+ set (SITE_OS_BITS "64")
+ set (SITE_COMPILER_NAME "vs2015")
+ set (SITE_COMPILER_VERSION "14")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS2015")
+ set (SITE_OS_BITS "32")
+ set (SITE_COMPILER_NAME "vs2015")
+ set (SITE_COMPILER_VERSION "14")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS201364")
+ set (SITE_OS_BITS "64")
+ set (SITE_COMPILER_NAME "vs2013")
+ set (SITE_COMPILER_VERSION "12")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS2013")
+ set (SITE_OS_BITS "32")
+ set (SITE_COMPILER_NAME "vs2013")
+ set (SITE_COMPILER_VERSION "12")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS201264")
+ set (SITE_OS_BITS "64")
+ set (SITE_COMPILER_NAME "vs2012")
+ set (SITE_COMPILER_VERSION "11")
+ elseif (${BUILD_GENERATOR} STREQUAL "VS2012")
+ set (SITE_OS_BITS "32")
+ set (SITE_COMPILER_NAME "vs2012")
+ set (SITE_COMPILER_VERSION "11")
endif ()
## Set the following to unique id your computer ##
- set(CTEST_SITE "WIN7${BUILD_GENERATOR}.XXXX")
+ set (CTEST_SITE "WIN7${BUILD_GENERATOR}.XXXX")
else ()
- set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+ set (CTEST_CMAKE_GENERATOR "Unix Makefiles")
## Set the following to unique id your computer ##
- if(APPLE)
- set(CTEST_SITE "MAC.XXXX")
+ if (APPLE)
+ set (CTEST_SITE "MAC.XXXX")
else ()
- set(CTEST_SITE "LINUX.XXXX")
+ set (CTEST_SITE "LINUX.XXXX")
endif ()
- if(APPLE)
- execute_process(COMMAND xcrun --find cc OUTPUT_VARIABLE XCODE_CC OUTPUT_STRIP_TRAILING_WHITESPACE)
- execute_process(COMMAND xcrun --find c++ OUTPUT_VARIABLE XCODE_CXX OUTPUT_STRIP_TRAILING_WHITESPACE)
- set(ENV{CC} "${XCODE_CC}")
- set(ENV{CXX} "${XCODE_CXX}")
- set(CTEST_USE_LAUNCHERS 1)
- set(RR_WARNINGS_COMMON "-Wno-format-nonliteral -Wno-cast-align -Wno-unused -Wno-unused-variable -Wno-unused-function -Wno-self-assign -Wno-unused-parameter -Wno-sign-compare")
- set(RR_WARNINGS_C "${RR_WARNINGS_COMMON} -Wno-deprecated-declarations -Wno-uninitialized")
- set(RR_WARNINGS_CXX "${RR_WARNINGS_COMMON} -Woverloaded-virtual -Wshadow -Wwrite-strings -Wc++11-compat")
- set(RR_FLAGS_COMMON "-g -O0 -fstack-protector-all -D_FORTIFY_SOURCE=2")
- set(RR_FLAGS_C "${RR_FLAGS_COMMON}")
- set(RR_FLAGS_CXX "${RR_FLAGS_COMMON}")
- set(ENV{CFLAGS} "${RR_WARNINGS_C} ${RR_FLAGS_C}")
- set(ENV{CXXFLAGS} "${RR_WARNINGS_CXX} ${RR_FLAGS_CXX}")
+ if (APPLE)
+ execute_process (COMMAND xcrun --find cc OUTPUT_VARIABLE XCODE_CC OUTPUT_STRIP_TRAILING_WHITESPACE)
+ execute_process (COMMAND xcrun --find c++ OUTPUT_VARIABLE XCODE_CXX OUTPUT_STRIP_TRAILING_WHITESPACE)
+ set (ENV{CC} "${XCODE_CC}")
+ set (ENV{CXX} "${XCODE_CXX}")
+ set (CTEST_USE_LAUNCHERS 1)
+ set (RR_WARNINGS_COMMON "-Wno-format-nonliteral -Wno-cast-align -Wno-unused -Wno-unused-variable -Wno-unused-function -Wno-self-assign -Wno-unused-parameter -Wno-sign-compare")
+ set (RR_WARNINGS_C "${RR_WARNINGS_COMMON} -Wno-deprecated-declarations -Wno-uninitialized")
+ set (RR_WARNINGS_CXX "${RR_WARNINGS_COMMON} -Woverloaded-virtual -Wshadow -Wwrite-strings -Wc++11-compat")
+ set (RR_FLAGS_COMMON "-g -O0 -fstack-protector-all -D_FORTIFY_SOURCE=2")
+ set (RR_FLAGS_C "${RR_FLAGS_COMMON}")
+ set (RR_FLAGS_CXX "${RR_FLAGS_COMMON}")
+ set (ENV{CFLAGS} "${RR_WARNINGS_C} ${RR_FLAGS_C}")
+ set (ENV{CXXFLAGS} "${RR_WARNINGS_CXX} ${RR_FLAGS_CXX}")
endif ()
endif ()
###################################################################
@@ -180,99 +180,99 @@ endif () ###################################################################
######### Following is for submission to CDash ############
###################################################################
-set(MODEL "Experimental")
+set (MODEL "Experimental")
###################################################################
###################################################################
##### Following controls CDash submission #####
-#set(LOCAL_SUBMIT "TRUE")
+#set (LOCAL_SUBMIT "TRUE")
##### Following controls test process #####
-#set(LOCAL_SKIP_TEST "TRUE")
-#set(LOCAL_MEMCHECK_TEST "TRUE")
-#set(LOCAL_COVERAGE_TEST "TRUE")
+#set (LOCAL_SKIP_TEST "TRUE")
+#set (LOCAL_MEMCHECK_TEST "TRUE")
+#set (LOCAL_COVERAGE_TEST "TRUE")
##### Following controls cpack command #####
-#set(LOCAL_NO_PACKAGE "TRUE")
+#set (LOCAL_NO_PACKAGE "TRUE")
##### Following controls source update #####
-#set(LOCAL_UPDATE "TRUE")
-set(REPOSITORY_URL "https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5.git")
-set(REPOSITORY_BRANCH "develop")
+#set (LOCAL_UPDATE "TRUE")
+set (REPOSITORY_URL "https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5.git")
+set (REPOSITORY_BRANCH "develop")
#uncomment to use a compressed source file: *.tar on linux or mac *.zip on windows
#set(CTEST_USE_TAR_SOURCE "${CTEST_SOURCE_VERSION}")
###################################################################
###################################################################
-if(${STATICONLYLIBRARIES})
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=OFF")
+if (${STATICONLYLIBRARIES})
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_SHARED_LIBS:BOOL=OFF")
######### Following describes computer ############
## following is optional to describe build ##
- set(SITE_BUILDNAME_SUFFIX "STATIC")
+ set (SITE_BUILDNAME_SUFFIX "STATIC")
endif ()
###################################################################
#### fortran ####
-if(${FORTRANLIBRARIES})
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=ON")
+if (${FORTRANLIBRARIES})
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=ON")
### enable Fortran 2003 depends on HDF5_BUILD_FORTRAN
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_F2003:BOOL=ON")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_F2003:BOOL=ON")
else ()
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=OFF")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=OFF")
### enable Fortran 2003 depends on HDF5_BUILD_FORTRAN
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_F2003:BOOL=OFF")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_F2003:BOOL=OFF")
endif ()
#### java ####
-if(${JAVALIBRARIES})
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=ON")
+if (${JAVALIBRARIES})
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=ON")
else ()
- set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=OFF")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=OFF")
endif ()
### change install prefix
-set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_INSTALL_PREFIX:PATH=${INSTALLDIR}")
-set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCTEST_CONFIGURATION_TYPE:STRING=$ENV{CMAKE_CONFIG_TYPE}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_INSTALL_PREFIX:PATH=${INSTALLDIR}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCTEST_CONFIGURATION_TYPE:STRING=$ENV{CMAKE_CONFIG_TYPE}")
###################################################################
-if(WIN32)
- set(BINFILEBASE "HDF5-${CTEST_SOURCE_VERSION}${CTEST_SOURCE_VERSEXT}-win${SITE_OS_BITS}")
- include(${CTEST_SCRIPT_DIRECTORY}\\HDF5options.cmake)
- include(${CTEST_SCRIPT_DIRECTORY}\\CTestScript.cmake)
- if(EXISTS "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.exe")
- file(COPY "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.exe" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+if (WIN32)
+ set (BINFILEBASE "HDF5-${CTEST_SOURCE_VERSION}${CTEST_SOURCE_VERSEXT}-win${SITE_OS_BITS}")
+ include (${CTEST_SCRIPT_DIRECTORY}\\HDF5options.cmake)
+ include (${CTEST_SCRIPT_DIRECTORY}\\CTestScript.cmake)
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.exe")
+ file (COPY "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.exe" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
- if(EXISTS "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.msi")
- file(COPY "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.msi" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.msi")
+ file (COPY "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.msi" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
- if(EXISTS "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.zip")
- file(COPY "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.zip" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.zip")
+ file (COPY "${CTEST_BINARY_DIRECTORY}\\${BINFILEBASE}.zip" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
else ()
- set(BINFILEBASE "HDF5-${CTEST_SOURCE_VERSION}${CTEST_SOURCE_VERSEXT}")
- include(${CTEST_SCRIPT_DIRECTORY}/HDF5options.cmake)
- include(${CTEST_SCRIPT_DIRECTORY}/CTestScript.cmake)
- if(APPLE)
- if(EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.dmg")
- file(COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.dmg" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ set (BINFILEBASE "HDF5-${CTEST_SOURCE_VERSION}${CTEST_SOURCE_VERSEXT}")
+ include (${CTEST_SCRIPT_DIRECTORY}/HDF5options.cmake)
+ include (${CTEST_SCRIPT_DIRECTORY}/CTestScript.cmake)
+ if (APPLE)
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.dmg")
+ file (COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.dmg" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
- if(EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.tar.gz")
- file(COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.tar.gz" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.tar.gz")
+ file (COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.tar.gz" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
- if(EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.sh")
- file(COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.sh" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.sh")
+ file (COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Darwin.sh" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
else ()
- if(CYGWIN)
- if(EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.sh")
- file(COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.sh" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (CYGWIN)
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.sh")
+ file (COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.sh" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
- if(EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.tar.gz")
- file(COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.tar.gz" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.tar.gz")
+ file (COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-CYGWIN.tar.gz" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
else ()
- if(EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.sh")
- file(COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.sh" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.sh")
+ file (COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.sh" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
- if(EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.tar.gz")
- file(COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.tar.gz" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
+ if (EXISTS "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.tar.gz")
+ file (COPY "${CTEST_BINARY_DIRECTORY}/${BINFILEBASE}-Linux.tar.gz" DESTINATION ${CTEST_SCRIPT_DIRECTORY})
endif ()
endif ()
endif ()
diff --git a/config/cmake/scripts/HDF5options.cmake b/config/cmake/scripts/HDF5options.cmake index a1bf88d..bbbb8dc 100755 --- a/config/cmake/scripts/HDF5options.cmake +++ b/config/cmake/scripts/HDF5options.cmake @@ -6,6 +6,10 @@ ### uncomment/comment and change the following lines for other configuration options ############################################################################################# +#### alternate toolsets #### +#set(CMAKE_GENERATOR_TOOLSET "Intel C++ Compiler 17.0") + +############################################################################################# #### ext libraries #### ### ext libs from tgz @@ -31,7 +35,7 @@ set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRING ### disable packaging #set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_NO_PACKAGES:BOOL=ON") -### Create install package with external libraries (szip, zlib, jpeg) +### Create install package with external libraries (szip, zlib) set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACKAGE_EXTLIBS:BOOL=ON") ############################################################################################# diff --git a/config/cmake_ext_mod/ConfigureChecks.cmake b/config/cmake_ext_mod/ConfigureChecks.cmake index 2e99c94..a0ec8e3 100644 --- a/config/cmake_ext_mod/ConfigureChecks.cmake +++ b/config/cmake_ext_mod/ConfigureChecks.cmake @@ -198,18 +198,19 @@ endmacro () #----------------------------------------------------------------------------- # Check for the existence of certain header files #----------------------------------------------------------------------------- -CHECK_INCLUDE_FILE_CONCAT ("sys/resource.h" ${HDF_PREFIX}_HAVE_SYS_RESOURCE_H) -CHECK_INCLUDE_FILE_CONCAT ("sys/time.h" ${HDF_PREFIX}_HAVE_SYS_TIME_H) -CHECK_INCLUDE_FILE_CONCAT ("unistd.h" ${HDF_PREFIX}_HAVE_UNISTD_H) +CHECK_INCLUDE_FILE_CONCAT ("sys/file.h" ${HDF_PREFIX}_HAVE_SYS_FILE_H) CHECK_INCLUDE_FILE_CONCAT ("sys/ioctl.h" ${HDF_PREFIX}_HAVE_SYS_IOCTL_H) -CHECK_INCLUDE_FILE_CONCAT ("sys/stat.h" ${HDF_PREFIX}_HAVE_SYS_STAT_H) +CHECK_INCLUDE_FILE_CONCAT ("sys/resource.h" ${HDF_PREFIX}_HAVE_SYS_RESOURCE_H) CHECK_INCLUDE_FILE_CONCAT ("sys/socket.h" ${HDF_PREFIX}_HAVE_SYS_SOCKET_H) +CHECK_INCLUDE_FILE_CONCAT ("sys/stat.h" ${HDF_PREFIX}_HAVE_SYS_STAT_H) +CHECK_INCLUDE_FILE_CONCAT ("sys/time.h" ${HDF_PREFIX}_HAVE_SYS_TIME_H) CHECK_INCLUDE_FILE_CONCAT ("sys/types.h" ${HDF_PREFIX}_HAVE_SYS_TYPES_H) -CHECK_INCLUDE_FILE_CONCAT ("stddef.h" ${HDF_PREFIX}_HAVE_STDDEF_H) -CHECK_INCLUDE_FILE_CONCAT ("setjmp.h" ${HDF_PREFIX}_HAVE_SETJMP_H) CHECK_INCLUDE_FILE_CONCAT ("features.h" ${HDF_PREFIX}_HAVE_FEATURES_H) CHECK_INCLUDE_FILE_CONCAT ("dirent.h" ${HDF_PREFIX}_HAVE_DIRENT_H) +CHECK_INCLUDE_FILE_CONCAT ("setjmp.h" ${HDF_PREFIX}_HAVE_SETJMP_H) +CHECK_INCLUDE_FILE_CONCAT ("stddef.h" ${HDF_PREFIX}_HAVE_STDDEF_H) CHECK_INCLUDE_FILE_CONCAT ("stdint.h" ${HDF_PREFIX}_HAVE_STDINT_H) +CHECK_INCLUDE_FILE_CONCAT ("unistd.h" ${HDF_PREFIX}_HAVE_UNISTD_H) # IF the c compiler found stdint, check the C++ as well. On some systems this # file will be found by C but not C++, only do this test IF the C++ compiler @@ -527,6 +528,8 @@ CHECK_FUNCTION_EXISTS (sigprocmask ${HDF_PREFIX}_HAVE_SIGPROCMASK) CHECK_FUNCTION_EXISTS (snprintf ${HDF_PREFIX}_HAVE_SNPRINTF) CHECK_FUNCTION_EXISTS (srandom ${HDF_PREFIX}_HAVE_SRANDOM) CHECK_FUNCTION_EXISTS (strdup ${HDF_PREFIX}_HAVE_STRDUP) +CHECK_FUNCTION_EXISTS (strtoll ${HDF_PREFIX}_HAVE_STRTOLL) +CHECK_FUNCTION_EXISTS (strtoull ${HDF_PREFIX}_HAVE_STRTOULL) CHECK_FUNCTION_EXISTS (symlink ${HDF_PREFIX}_HAVE_SYMLINK) CHECK_FUNCTION_EXISTS (system ${HDF_PREFIX}_HAVE_SYSTEM) @@ -585,6 +588,7 @@ macro (HDF_CXX_FUNCTION_TEST OTHER_TEST) HAVE_UNISTD_H HAVE_SYS_TYPES_H HAVE_SYS_SOCKET_H + HAVE_SYS_FILE_H ) if ("${${HDF_PREFIX}_${def}}") set (MACRO_CHECK_FUNCTION_DEFINITIONS "${MACRO_CHECK_FUNCTION_DEFINITIONS} -D${def}") diff --git a/configure.ac b/configure.ac index 2af7988..3e918db 100644 --- a/configure.ac +++ b/configure.ac @@ -1851,6 +1851,7 @@ AC_CHECK_FUNCS([frexpl gethostname getrusage gettimeofday]) AC_CHECK_FUNCS([lstat rand_r random setsysinfo]) AC_CHECK_FUNCS([signal longjmp setjmp siglongjmp sigsetjmp sigprocmask]) AC_CHECK_FUNCS([snprintf srandom strdup symlink system]) +AC_CHECK_FUNCS([strtoll strtoull]) AC_CHECK_FUNCS([tmpfile asprintf vasprintf vsnprintf waitpid]) AC_CHECK_FUNCS([roundf lroundf llroundf round lround llround]) diff --git a/fortran/test/tH5F.F90 b/fortran/test/tH5F.F90 index 8334b30..ea8c53f 100644 --- a/fortran/test/tH5F.F90 +++ b/fortran/test/tH5F.F90 @@ -729,7 +729,7 @@ CONTAINS CALL h5fget_freespace_f(fid, free_space, error) CALL check("h5fget_freespace_f",error,total_error) - if(error .eq.0 .and. free_space .ne. 0) then + if(error .eq.0 .and. free_space .ne. 1248) then total_error = total_error + 1 write(*,*) "1: Wrong amount of free space reported, ", free_space endif @@ -745,7 +745,7 @@ CONTAINS ! Check the free space now CALL h5fget_freespace_f(fid, free_space, error) CALL check("h5fget_freespace_f",error,total_error) - if(error .eq.0 .and. free_space .ne. 0) then + if(error .eq.0 .and. free_space .ne. 216) then total_error = total_error + 1 write(*,*) "2: Wrong amount of free space reported, ", free_space endif @@ -757,7 +757,7 @@ CONTAINS ! Check the free space now CALL h5fget_freespace_f(fid, free_space, error) CALL check("h5fget_freespace_f",error,total_error) - if(error .eq.0 .and. free_space .ne. 0) then + if(error .eq.0 .and. free_space .ne. 1248) then total_error = total_error + 1 write(*,*) "3: Wrong amount of free space reported, ", free_space endif diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 02d35ba..5b35d17 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -5356,39 +5356,24 @@ public class H5 implements java.io.Serializable { throws HDF5LibraryException, IllegalArgumentException; /** - * H5Pset_file_space sets the file space management strategy for the file associated with fcpl_id to strategy. + * H5Pset_file_space_strategy sets the file space management strategy for the file associated with fcpl_id to strategy. * There are four strategies that applications can select and they are described in the Parameters section. * * @param fcpl_id * IN: File creation property list identifier * @param strategy * IN: The strategy for file space management. - * Passing a value of zero (0) indicates that the value of strategy is not to be modified. - * H5F_FILE_SPACE_ALL_PERSIST - * With this strategy, the free-space managers track the free space that results from the - * manipulation of HDF5 objects in the HDF5 file. The free space information is saved when the - * file is closed, and reloaded when the file is reopened. When space is needed for file metadata - * or raw data, the HDF5 library first requests space from the library's free-space managers. - * If the request is not satisfied, the library requests space from the aggregators. If the request - * is still not satisfied, the library requests space from the virtual file driver. That is, the - * library will use all of the mechanisms for allocating space. - * H5F_FILE_SPACE_ALL (Default file space management strategy) - * With this strategy, the free-space managers track the free space that results from the manipulation - * of HDF5 objects in the HDF5 file. The free space information is NOT saved when the file is closed - * and the free space that exists upon file closing becomes unaccounted space in the file. - * Like the previous strategy, the library will try all of the mechanisms for allocating space. When - * space is needed for file metadata or raw data, the library first requests space from the free-space - * managers. If the request is not satisfied, the library requests space from the aggregators. If the - * request is still not satisfied, the library requests space from the virtual file driver. - * H5F_FILE_SPACE_AGGR_VFD - * With this strategy, the library does not track free space that results from the manipulation of HDF5 - * obejcts in the HDF5 file and the free space becomes unaccounted space in the file. - * When space is needed for file metadata or raw data, the library first requests space from the - * aggregators. If the request is not satisfied, the library requests space from the virtual file driver. - * H5F_FILE_SPACE_VFD - * With this strategy, the library does not track free space that results from the manipulation of HDF5 - * obejcts in the HDF5 file and the free space becomes unaccounted space in the file. - * When space is needed for file metadata or raw data, the library requests space from the virtual file driver. + * H5F_FSPACE_STRATEGY_FSM_AGGR + * Mechanisms: free-space managers, aggregators, and virtual file drivers + * This is the library default when not set. + * H5F_FSPACE_STRATEGY_PAGE + * Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers + * H5F_FSPACE_STRATEGY_AGGR + * Mechanisms: aggregators and virtual file drivers + * H5F_FSPACE_STRATEGY_NONE + * Mechanisms: virtual file drivers + * @param persist + * IN: True to persist free-space. * @param threshold * IN: The free-space section threshold. The library default is 1, which is to track all free-space sections. * Passing a value of zero (0) indicates that the value of threshold is not to be modified. @@ -5399,28 +5384,100 @@ public class H5 implements java.io.Serializable { * - Invalid values of max_list and min_btree. * **/ - public synchronized static native void H5Pset_file_space(long fcpl_id, int strategy, long threshold) + public synchronized static native void H5Pset_file_space_strategy(long fcpl_id, int strategy, boolean persist, long threshold) throws HDF5LibraryException, IllegalArgumentException; /** - * H5Pget_file_space provides the means for applications to manage the HDF5 file's file space for their specific needs. + * H5Pget_file_space_strategy provides the means for applications to manage the HDF5 file's file space strategy for their specific needs. * * @param fcpl_id * IN: File creation property list identifier - * @param strategy - * IN/OUT: The current file space management strategy in use for the file. NULL, strategy not queried. + * @param persist + * IN/OUT: The current free-space persistence. NULL, persist not queried. * @param threshold * IN/OUT: The current free-space section threshold. NULL, threshold not queried. * + * @return the current free-space strategy. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native int H5Pget_file_space_strategy(long fcpl_id, boolean[] persist, long[] threshold) + throws HDF5LibraryException, IllegalArgumentException; + + /** + * H5Pget_file_space_strategy_persist provides the means for applications to manage the HDF5 file's file space strategy for their specific needs. + * + * @param fcpl_id + * IN: File creation property list identifier + * + * @return the current free-space persistence. + * * @exception HDF5LibraryException * - Error from the HDF-5 Library. * @exception IllegalArgumentException * - Invalid values of max_list and min_btree. * **/ - public synchronized static native void H5Pget_file_space(long fcpl_id, int[] strategy, long[] threshold) + public synchronized static native boolean H5Pget_file_space_strategy_persist(long fcpl_id) throws HDF5LibraryException, IllegalArgumentException; + /** + * H5Pget_file_space_strategy_threshold provides the means for applications to manage the HDF5 file's file space strategy for their specific needs. + * + * @param fcpl_id + * IN: File creation property list identifier + * + * @return the current free-space section threshold. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native long H5Pget_file_space_strategy_threshold(long fcpl_id) + throws HDF5LibraryException, IllegalArgumentException; + + /** + * H5Pset_file_space_page_size retrieves the file space page size for aggregating small metadata or raw data. + * + * @param fcpl_id + * IN: File creation property list identifier + * @param page_size + * IN: the file space page size. + * + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native void H5Pset_file_space_page_size(long fcpl_id, long page_size) + throws HDF5LibraryException, IllegalArgumentException; + + /** + * H5Pget_file_space_page_size Sets the file space page size for paged aggregation. + * + * @param fcpl_id + * IN: File creation property list identifier + * + * @return the current file space page size. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native long H5Pget_file_space_page_size(long fcpl_id) + throws HDF5LibraryException, IllegalArgumentException; + + // File access property list (FAPL) routines /** diff --git a/java/src/hdf/hdf5lib/HDF5Constants.java b/java/src/hdf/hdf5lib/HDF5Constants.java index acd3eb5..9773100 100644 --- a/java/src/hdf/hdf5lib/HDF5Constants.java +++ b/java/src/hdf/hdf5lib/HDF5Constants.java @@ -219,12 +219,12 @@ public class HDF5Constants { public static final int H5F_SCOPE_GLOBAL = H5F_SCOPE_GLOBAL(); public static final int H5F_SCOPE_LOCAL = H5F_SCOPE_LOCAL(); public static final int H5F_UNLIMITED = H5F_UNLIMITED(); - public static final int H5F_FILE_SPACE_DEFAULT = H5F_FILE_SPACE_DEFAULT(); - public static final int H5F_FILE_SPACE_ALL_PERSIST = H5F_FILE_SPACE_ALL_PERSIST(); - public static final int H5F_FILE_SPACE_ALL = H5F_FILE_SPACE_ALL(); - public static final int H5F_FILE_SPACE_AGGR_VFD = H5F_FILE_SPACE_AGGR_VFD(); - public static final int H5F_FILE_SPACE_VFD = H5F_FILE_SPACE_VFD(); - public static final int H5F_FILE_SPACE_NTYPES = H5F_FILE_SPACE_NTYPES(); + + public static final int H5F_FSPACE_STRATEGY_FSM_AGGR = H5F_FSPACE_STRATEGY_FSM_AGGR(); + public static final int H5F_FSPACE_STRATEGY_AGGR = H5F_FSPACE_STRATEGY_AGGR(); + public static final int H5F_FSPACE_STRATEGY_PAGE = H5F_FSPACE_STRATEGY_PAGE(); + public static final int H5F_FSPACE_STRATEGY_NONE = H5F_FSPACE_STRATEGY_NONE(); + public static final int H5F_FSPACE_STRATEGY_NTYPES = H5F_FSPACE_STRATEGY_NTYPES(); public static final long H5FD_CORE = H5FD_CORE(); public static final long H5FD_DIRECT = H5FD_DIRECT(); @@ -1024,17 +1024,15 @@ public class HDF5Constants { private static native final int H5F_UNLIMITED(); - private static native final int H5F_FILE_SPACE_DEFAULT(); - - private static native final int H5F_FILE_SPACE_ALL_PERSIST(); + private static native final int H5F_FSPACE_STRATEGY_FSM_AGGR(); - private static native final int H5F_FILE_SPACE_ALL(); + private static native final int H5F_FSPACE_STRATEGY_AGGR(); - private static native final int H5F_FILE_SPACE_AGGR_VFD(); + private static native final int H5F_FSPACE_STRATEGY_PAGE(); - private static native final int H5F_FILE_SPACE_VFD(); + private static native final int H5F_FSPACE_STRATEGY_NONE(); - private static native final int H5F_FILE_SPACE_NTYPES(); + private static native final int H5F_FSPACE_STRATEGY_NTYPES(); private static native final long H5FD_CORE(); diff --git a/java/src/jni/h5Constants.c b/java/src/jni/h5Constants.c index ccd0b91..c6aa16c 100644 --- a/java/src/jni/h5Constants.c +++ b/java/src/jni/h5Constants.c @@ -403,17 +403,15 @@ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_HDF5Constants_H5F_1UNLIMITED(JNIEnv *env, jclass cls) { return (jint)H5F_UNLIMITED; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1DEFAULT(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_DEFAULT; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1FSM_1AGGR(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_FSM_AGGR; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1ALL_1PERSIST(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_ALL_PERSIST; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1AGGR(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_AGGR; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1ALL(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_ALL; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1PAGE(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_PAGE; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1AGGR_1VFD(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_AGGR_VFD; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1NONE(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_NONE; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1VFD(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_VFD; } -JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1NTYPES(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_NTYPES; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1NTYPES(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_NTYPES; } JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5FD_1CORE(JNIEnv *env, jclass cls) { return H5FD_CORE; } diff --git a/java/src/jni/h5pImp.c b/java/src/jni/h5pImp.c index 1368139..f39f0e6 100644 --- a/java/src/jni/h5pImp.c +++ b/java/src/jni/h5pImp.c @@ -4883,61 +4883,130 @@ Java_hdf_hdf5lib_H5_H5Pset_1virtual_1printf_1gap /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_file_space - * Signature: (J[I[J)V + * Method: H5Pget_file_space_strategy + * Signature: (J[Z[J)I */ -JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pget_1file_1space - (JNIEnv *env, jclass clss, jlong fcpl_id, jintArray strategy, jlongArray threshold) +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy + (JNIEnv *env, jclass clss, jlong fcpl_id, jbooleanArray persist, jlongArray threshold) { herr_t status = -1; - jint *thestrategyArray = NULL; + H5F_fspace_strategy_t thestrategy = H5F_FSPACE_STRATEGY_FSM_AGGR; /* Library default */ jlong *thethresholdArray = NULL; + jboolean *thepersistArray = NULL; jboolean isCopy; - if (strategy) { - thestrategyArray = (jint*)ENVPTR->GetIntArrayElements(ENVPAR strategy, &isCopy); - if (thestrategyArray == NULL) { - h5JNIFatalError(env, "H5Pget_file_space: strategy not pinned"); - return; + if (persist) { + thepersistArray = (jboolean*)ENVPTR->GetBooleanArrayElements(ENVPAR persist, &isCopy); + if (thepersistArray == NULL) { + h5JNIFatalError(env, "H5Pget_file_space: persist not pinned"); + return -1; } } if (threshold) { thethresholdArray = (jlong*)ENVPTR->GetLongArrayElements(ENVPAR threshold, &isCopy); if (thethresholdArray == NULL) { - if (strategy) ENVPTR->ReleaseIntArrayElements(ENVPAR strategy, thestrategyArray, JNI_ABORT); + if (persist) ENVPTR->ReleaseBooleanArrayElements(ENVPAR persist, thepersistArray, JNI_ABORT); h5JNIFatalError(env, "H5Pget_file_space: threshold not pinned"); - return; + return -1; } /* end if */ } /* end if */ - status = H5Pget_file_space((hid_t)fcpl_id, (H5F_file_space_type_t*)thestrategyArray, (hsize_t*)thethresholdArray); + status = H5Pget_file_space_strategy((hid_t)fcpl_id, &thestrategy, (hbool_t*)thepersistArray, (hsize_t*)thethresholdArray); if (status < 0) { - if (strategy) ENVPTR->ReleaseIntArrayElements(ENVPAR strategy, thestrategyArray, JNI_ABORT); + if (persist) ENVPTR->ReleaseBooleanArrayElements(ENVPAR persist, thepersistArray, JNI_ABORT); if (threshold) ENVPTR->ReleaseLongArrayElements(ENVPAR threshold, thethresholdArray, JNI_ABORT); h5libraryError(env); } /* end if */ else { - if (strategy) ENVPTR->ReleaseIntArrayElements(ENVPAR strategy, thestrategyArray, 0); + if (persist) ENVPTR->ReleaseBooleanArrayElements(ENVPAR persist, thepersistArray, 0); if (threshold) ENVPTR->ReleaseLongArrayElements(ENVPAR threshold, thethresholdArray, 0); } /* end else */ -} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space */ + return (jint)thestrategy; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_persist + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1persist + (JNIEnv *env, jclass clss, jlong fcpl_id) +{ + herr_t status = -1; + hbool_t thepersist = FALSE; + + status = H5Pget_file_space_strategy((hid_t)fcpl_id, NULL, &thepersist, NULL); + + if (status < 0) + h5libraryError(env); + + return (jboolean)thepersist; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1persist */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_threshold + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1threshold + (JNIEnv *env, jclass clss, jlong fcpl_id) +{ + herr_t status = -1; + hsize_t thethreshold; + + status = H5Pget_file_space_strategy((hid_t)fcpl_id, NULL, NULL, &thethreshold); + + if (status < 0) + h5libraryError(env); + + return (jlong)thethreshold; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1threshold */ /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_file_space - * Signature: (JIJ)V + * Method: H5Pset_file_space_strategy + * Signature: (JIZJ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1file_1space - (JNIEnv *env, jclass clss, jlong fcpl_id, jint strategy, jlong threshold) +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1strategy + (JNIEnv *env, jclass clss, jlong fcpl_id, jint strategy, jboolean persist, jlong threshold) { - if (H5Pset_file_space((hid_t)fcpl_id, (H5F_file_space_type_t)strategy, (hsize_t)threshold) < 0) + if (H5Pset_file_space_strategy((hid_t)fcpl_id, (H5F_fspace_strategy_t)strategy, (hbool_t)persist, (hsize_t)threshold) < 0) h5libraryError(env); -} /* end Java_hdf_hdf5lib_H5_H5Pset_1file_1space */ +} /* end Java_hdf_hdf5lib_H5_H5Pset_file_space_strategy */ +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_space_page_size + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1page_1size + (JNIEnv *env, jclass clss, jlong fcpl_id, jlong fsp_size) +{ + if (H5Pset_file_space_page_size((hid_t)fcpl_id, (hsize_t)fsp_size) < 0) + h5libraryError(env); +} + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_page_size + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1page_1size + (JNIEnv *env, jclass clss, jlong fcpl_id) +{ + hsize_t fsp_size = 0; + if (H5Pget_file_space_page_size((hid_t)fcpl_id, &fsp_size) < 0) + h5libraryError(env); + return (jlong)fsp_size; +} static herr_t H5P_cls_create_cb diff --git a/java/src/jni/h5pImp.h b/java/src/jni/h5pImp.h index 927501e..03b0e03 100644 --- a/java/src/jni/h5pImp.h +++ b/java/src/jni/h5pImp.h @@ -1385,22 +1385,57 @@ Java_hdf_hdf5lib_H5_H5Pset_1virtual_1printf_1gap /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_file_space - * Signature: (J[I[J)V + * Method: H5Pget_file_space_strategy + * Signature: (J[Z[J)I + */ +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy +(JNIEnv *, jclass, jlong, jbooleanArray, jlongArray); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_persist + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1persist +(JNIEnv *, jclass, jlong); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_threshold + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1threshold +(JNIEnv *, jclass, jlong); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_space_strategy + * Signature: (JIZJ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pget_1file_1space -(JNIEnv *, jclass, jlong, jintArray, jlongArray); +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1strategy +(JNIEnv *, jclass, jlong, jint, jboolean, jlong); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_page_size + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1page_1size +(JNIEnv *, jclass, jlong); /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_file_space - * Signature: (JIJ)V + * Method: H5Pset_file_space_page_size + * Signature: (JJ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1file_1space -(JNIEnv *, jclass, jlong, jint, jlong); +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1page_1size +(JNIEnv *, jclass, jlong, jlong); /* * Class: hdf_hdf5lib_H5 diff --git a/java/test/JUnit-interface.txt b/java/test/JUnit-interface.txt index 34ec02a..cae8cef 100644 --- a/java/test/JUnit-interface.txt +++ b/java/test/JUnit-interface.txt @@ -416,6 +416,7 @@ JUnit version 4.11 .testH5P_layout .testH5Pget_link_creation_order .testH5Pset_shared_mesg_nindexes_InvalidHIGHnindexes +.testH5P_file_space_page_size .testH5Pget_shared_mesg_index_Invalid_indexnum .testH5Pset_data_transform_NullExpression .testH5Pset_elink_prefix_null @@ -438,9 +439,9 @@ JUnit version 4.11 .testH5Pset_shared_mesg_index .testH5Pset_copy_object .testH5Pset_link_creation_order_trackedPLUSindexed +.testH5P_file_space_strategy .testH5Pset_copy_object_invalidobject .testH5Pset_est_link_info_InvalidValues -.testH5P_file_space .testH5Pset_local_heap_size_hint .testH5Pget_est_link_info .testH5Pset_scaleoffset @@ -638,7 +639,7 @@ JUnit version 4.11 Time: XXXX -OK (636 tests) +OK (637 tests) HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): #000: (file name) line (number) in H5Fopen(): can't set access and transfer property lists diff --git a/java/test/TestH5P.java b/java/test/TestH5P.java index b8a4376..fa8b525 100644 --- a/java/test/TestH5P.java +++ b/java/test/TestH5P.java @@ -1198,25 +1198,40 @@ public class TestH5P { } @Test - public void testH5P_file_space() { + public void testH5P_file_space_strategy() { long[] threshold = {0}; - int[] strategy = {0}; + boolean[] persist = {false}; + int strategy = 0; try { - H5.H5Pget_file_space(fcpl_id, strategy, threshold); - assertTrue("strategy: "+strategy[0], strategy[0] == HDF5Constants.H5F_FILE_SPACE_ALL); + strategy = H5.H5Pget_file_space_strategy(fcpl_id, persist, threshold); + assertTrue("strategy(default): "+strategy, strategy == HDF5Constants.H5F_FSPACE_STRATEGY_FSM_AGGR); + assertTrue("persist(default): "+persist[0], persist[0] == false); + assertTrue("theshold(default): "+threshold[0], threshold[0] == 1); + H5.H5Pset_file_space_strategy(fcpl_id, HDF5Constants.H5F_FSPACE_STRATEGY_PAGE, true, 1); + strategy = H5.H5Pget_file_space_strategy(fcpl_id, persist, threshold); + assertTrue("strategy: "+strategy, strategy == HDF5Constants.H5F_FSPACE_STRATEGY_PAGE); + assertTrue("persist: "+persist[0], persist[0] == true); assertTrue("theshold: "+threshold[0], threshold[0] == 1); - H5.H5Pset_file_space(fcpl_id, HDF5Constants.H5F_FILE_SPACE_ALL_PERSIST, 10); - H5.H5Pget_file_space(fcpl_id, strategy, threshold); - assertTrue("strategy: "+strategy[0], strategy[0] == HDF5Constants.H5F_FILE_SPACE_ALL_PERSIST); - assertTrue("theshold: "+threshold[0], threshold[0] == 10); - H5.H5Pset_file_space(fcpl_id, HDF5Constants.H5F_FILE_SPACE_VFD, 0); - H5.H5Pget_file_space(fcpl_id, strategy, threshold); - assertTrue("strategy: "+strategy[0], strategy[0] == HDF5Constants.H5F_FILE_SPACE_VFD); - assertTrue("theshold: "+threshold[0], threshold[0] == 10); } catch (Throwable err) { err.printStackTrace(); - fail("testH5P_file_space: " + err); + fail("testH5P_file_space_strategy: " + err); + } + } + + @Test + public void testH5P_file_space_page_size() { + long page_size = 0; + try { + page_size = H5.H5Pget_file_space_page_size(fcpl_id); + assertTrue("page_size(default): "+page_size, page_size == 4096); + H5.H5Pset_file_space_page_size(fcpl_id, 512); + page_size = H5.H5Pget_file_space_page_size(fcpl_id); + assertTrue("page_size: "+page_size, page_size == 512); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5P_file_space_page_size: " + err); } } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef361b2..9321bbd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -195,6 +195,7 @@ set (H5F_SOURCES ${HDF5_SRC_DIR}/H5Fmpi.c ${HDF5_SRC_DIR}/H5Fquery.c ${HDF5_SRC_DIR}/H5Fsfile.c + ${HDF5_SRC_DIR}/H5Fspace.c ${HDF5_SRC_DIR}/H5Fsuper.c ${HDF5_SRC_DIR}/H5Fsuper_cache.c ${HDF5_SRC_DIR}/H5Ftest.c @@ -504,6 +505,15 @@ set (H5P_HDRS ) IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SOURCES}" ) +set (H5PB_SOURCES + ${HDF5_SRC_DIR}/H5PB.c +) + +set (H5PB_HDRS + ${HDF5_SRC_DIR}/H5PBpkg.h +) +IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" ) + set (H5PL_SOURCES ${HDF5_SRC_DIR}/H5PL.c ) @@ -699,6 +709,7 @@ set (common_SRCS ${H5MP_SOURCES} ${H5O_SOURCES} ${H5P_SOURCES} + ${H5PB_SOURCES} ${H5PL_SOURCES} ${H5R_SOURCES} ${H5UC_SOURCES} @@ -739,6 +750,7 @@ set (H5_PUBLIC_HEADERS ${H5MP_HDRS} ${H5O_HDRS} ${H5P_HDRS} + ${H5PB_HDRS} ${H5PL_HDRS} ${H5R_HDRS} ${H5S_HDRS} @@ -777,6 +789,7 @@ set (H5_PRIVATE_HEADERS ${HDF5_SRC_DIR}/H5MPprivate.h ${HDF5_SRC_DIR}/H5Oprivate.h ${HDF5_SRC_DIR}/H5Pprivate.h + ${HDF5_SRC_DIR}/H5PBprivate.h ${HDF5_SRC_DIR}/H5PLprivate.h ${HDF5_SRC_DIR}/H5UCprivate.h ${HDF5_SRC_DIR}/H5Rprivate.h @@ -353,6 +353,44 @@ H5AC_term_package(void) /*------------------------------------------------------------------------- + * + * Function: H5AC_cache_image_pending() + * + * Purpose: Debugging function that tests to see if the load of a + * metadata cache image load is pending (i.e. will be executed + * on the next protect or insert) + * + * Returns TRUE if a cache image load is pending, and FALSE + * if not. Throws an assertion failure on error. + * + * Return: TRUE if a cache image load is pending, and FALSE otherwise. + * + * Programmer: John Mainzer, 1/10/17 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +hbool_t +H5AC_cache_image_pending(const H5F_t *f) +{ + H5C_t *cache_ptr; + hbool_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + + ret_value = H5C_cache_image_pending(cache_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_cache_image_pending() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_create * * Purpose: Initialize the cache just after a file is opened. The @@ -501,7 +539,7 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co /* Turn on metadata cache logging, if being used */ if(H5F_USE_MDC_LOGGING(f)) { if(H5C_set_up_logging(f->shared->cache, H5F_MDC_LOG_LOCATION(f), H5F_START_MDC_LOG_ON_ACCESS(f)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "mdc logging setup failed") + HGOTO_ERROR(H5E_CACHE, H5E_CANTINIT, FAIL, "mdc logging setup failed") /* Write the log header regardless of current logging status */ if(H5AC__write_create_cache_log_msg(f->shared->cache) < 0) @@ -510,9 +548,9 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co /* Set the cache parameters */ if(H5AC_set_cache_auto_resize_config(f->shared->cache, config_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed") + HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "auto resize configuration failed") - /* don't need to get the current H5C image config here since the + /* Don't need to get the current H5C image config here since the * cache has just been created, and thus f->shared->cache->image_ctl * must still set to its initial value (H5C__DEFAULT_CACHE_IMAGE_CTL). * Note that this not true as soon as control returns to the application @@ -522,9 +560,8 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co int_ci_config.generate_image = image_config_ptr->generate_image; int_ci_config.save_resize_status = image_config_ptr->save_resize_status; int_ci_config.entry_ageout = image_config_ptr->entry_ageout; - if(H5C_set_cache_image_config(f, f->shared->cache, &int_ci_config) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed") + HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "auto resize configuration failed") done: #ifdef H5_HAVE_PARALLEL @@ -836,6 +873,46 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_force_cache_image_load() + * + * Purpose: On rare occasions, it is necessary to run + * H5MF_tidy_self_referential_fsm_hack() prior to the first + * metadata cache access. This is a problem as if there is a + * cache image at the end of the file, that routine will + * discard it. + * + * We solve this issue by calling this function, which will + * load the cache image and then call + * H5MF_tidy_self_referential_fsm_hack() to discard it. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 1/11/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_force_cache_image_load(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if(H5C_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't load cache image") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_force_cache_image_load() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_get_entry_status * * Purpose: Given a file address, determine whether the metadata @@ -3120,6 +3197,48 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_unsettle_ring() + * + * Purpose: Advise the metadata cache that the specified free space + * manager ring is no longer settled (if it was on entry). + * + * If the target free space manager ring is already + * unsettled, do nothing, and return SUCCEED. + * + * If the target free space manager ring is settled, and + * we are not in the process of a file shutdown, mark + * the ring as unsettled, and return SUCCEED. + * + * If the target free space manager is settled, and we + * are in the process of a file shutdown, post an error + * message, and return FAIL. + * + * Note that this function simply passes the call on to + * the metadata cache proper, and returns the result. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/15/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_unsettle_ring(H5F_t * f, H5C_ring_t ring) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if(FAIL == (ret_value = H5C_unsettle_ring(f, ring))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_unsettle_ring() failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_unsettle_ring() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_remove_entry() * * Purpose: Remove an entry from the cache. Must be not protected, pinned, diff --git a/src/H5ACdbg.c b/src/H5ACdbg.c index 8ca5102..6073288 100644 --- a/src/H5ACdbg.c +++ b/src/H5ACdbg.c @@ -101,6 +101,7 @@ H5AC_stats(const H5F_t *f) FUNC_LEAVE_NOAPI(SUCCEED) } /* H5AC_stats() */ +#ifndef NDEBUG /*------------------------------------------------------------------------- * Function: H5AC_dump_cache @@ -133,6 +134,7 @@ H5AC_dump_cache(const H5F_t *f) done: FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_dump_cache() */ +#endif /* NDEBUG */ /*------------------------------------------------------------------------- diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 1fe6456..1f8299b 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -451,6 +451,8 @@ H5_DLL herr_t H5AC_validate_config(H5AC_cache_config_t *config_ptr); H5_DLL herr_t H5AC_load_cache_image_on_next_protect(H5F_t *f, haddr_t addr, hsize_t len, hbool_t rw); H5_DLL herr_t H5AC_validate_cache_image_config(H5AC_cache_image_config_t *config_ptr); +H5_DLL hbool_t H5AC_cache_image_pending(const H5F_t *f); +H5_DLL herr_t H5AC_force_cache_image_load(H5F_t * f, hid_t dxpl_id); /* Tag & Ring routines */ H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag); @@ -464,6 +466,7 @@ H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dx H5AC_ring_t *orig_ring); H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring); H5_DLL herr_t H5AC_unsettle_entry_ring(void *entry); +H5_DLL herr_t H5AC_unsettle_ring(H5F_t * f, H5AC_ring_t ring); H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag); @@ -482,8 +485,8 @@ H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr); /* Debugging functions */ H5_DLL herr_t H5AC_stats(const H5F_t *f); -H5_DLL herr_t H5AC_dump_cache(const H5F_t *f); #ifndef NDEBUG +H5_DLL herr_t H5AC_dump_cache(const H5F_t *f); H5_DLL herr_t H5AC_get_entry_ptr_from_addr(const H5F_t *f, haddr_t addr, void **entry_ptr_ptr); H5_DLL herr_t H5AC_flush_dependency_exists(H5F_t *f, haddr_t parent_addr, @@ -386,6 +386,7 @@ H5C_create(size_t max_cache_size, cache_ptr->cache_full = FALSE; cache_ptr->size_decreased = FALSE; cache_ptr->resize_in_progress = FALSE; + cache_ptr->msic_in_progress = FALSE; (cache_ptr->resize_ctl).version = H5C__CURR_AUTO_SIZE_CTL_VER; (cache_ptr->resize_ctl).rpt_fcn = NULL; @@ -731,7 +732,8 @@ herr_t H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id) { H5C_t * cache_ptr; - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t image_generated = FALSE; /* Whether a cache image was generated */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -755,9 +757,48 @@ H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id) HDassert(cache_ptr->pl_len == 0); /* Prepare cache image */ - if(H5C__prep_image_for_file_close(f, dxpl_id) < 0) + if(H5C__prep_image_for_file_close(f, dxpl_id, &image_generated) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create cache image") +#ifdef H5_HAVE_PARALLEL + if(!image_generated && cache_ptr->aux_ptr != NULL && f->shared->fs_persist) { + /* If persistent free space managers are enabled, flushing the + * metadata cache may result in the deletion, insertion, and/or + * dirtying of entries. + * + * This is a problem in PHDF5, as it breaks two invariants of + * our management of the metadata cache across all processes: + * + * 1) Entries will not be dirtied, deleted, inserted, or moved + * during flush in the parallel case. + * + * 2) All processes contain the same set of dirty metadata + * entries on entry to a sync point. + * + * To solve this problem for the persistent free space managers, + * serialize the metadata cache on all processes prior to the + * first sync point on file shutdown. The shutdown warning is + * a convenient location for this call. + * + * This is sufficient since: + * + * 1) FSM settle routines are only invoked on file close. Since + * serialization make the same settle calls as flush on file + * close, and since the close warning is issued after all + * non FSM related space allocations and just before the + * first sync point on close, this call will leave the caches + * in a consistant state across the processes if they were + * consistant before. + * + * 2) Since the FSM settle routines are only invoked once during + * file close, invoking them now will prevent their invocation + * during a flush, and thus avoid any resulting entrie dirties, + * deletions, insertion, or moves during the flush. + */ + if(H5C__serialize_cache(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTSERIALIZE, FAIL, "serialization of the cache failed") + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -925,12 +966,6 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is protected") if(entry_ptr->is_pinned) HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is pinned") -#ifdef H5_HAVE_PARALLEL - if(entry_ptr->coll_access) { - entry_ptr->coll_access = FALSE; - H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ /* If we get this far, call H5C__flush_single_entry() with the * H5C__FLUSH_INVALIDATE_FLAG and the H5C__FLUSH_CLEAR_ONLY_FLAG. @@ -1465,10 +1500,15 @@ H5C_insert_entry(H5F_t * f, entry_ptr->prefetched = FALSE; entry_ptr->prefetch_type_id = 0; entry_ptr->age = 0; + entry_ptr->prefetched_dirty = FALSE; #ifndef NDEBUG /* debugging field */ entry_ptr->serialization_count = 0; #endif /* NDEBUG */ + entry_ptr->tl_next = NULL; + entry_ptr->tl_prev = NULL; + entry_ptr->tag_info = NULL; + /* Apply tag to newly inserted entry */ if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Cannot tag metadata entry") @@ -2437,7 +2477,7 @@ H5C_protect(H5F_t * f, marked as collective, and is clean, it is possible that other processes will not have it in its cache and will expect a bcast of the entry from process 0. So process 0 - will bcast the entry to all other ranks. Ranks that do have + will bcast the entry to all other ranks. Ranks that _do_ have the entry in their cache still have to participate in the bcast. */ #ifdef H5_HAVE_PARALLEL @@ -3537,6 +3577,73 @@ done: /*------------------------------------------------------------------------- + * Function: H5C_unsettle_ring() + * + * Purpose: Advise the metadata cache that the specified free space + * manager ring is no longer settled (if it was on entry). + * + * If the target free space manager ring is already + * unsettled, do nothing, and return SUCCEED. + * + * If the target free space manager ring is settled, and + * we are not in the process of a file shutdown, mark + * the ring as unsettled, and return SUCCEED. + * + * If the target free space manager is settled, and we + * are in the process of a file shutdown, post an error + * message, and return FAIL. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/15/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring) +{ + H5C_t * cache_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + HDassert((H5C_RING_RDFSM == ring) || (H5C_RING_MDFSM == ring)); + cache_ptr = f->shared->cache; + HDassert(H5C__H5C_T_MAGIC == cache_ptr->magic); + + switch(ring) { + case H5C_RING_RDFSM: + if(cache_ptr->rdfsm_settled) { + if(cache_ptr->close_warning_received) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected rdfsm ring unsettle") + cache_ptr->rdfsm_settled = FALSE; + } /* end if */ + break; + + case H5C_RING_MDFSM: + if(cache_ptr->mdfsm_settled) { + if(cache_ptr->close_warning_received) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected mdfsm ring unsettle") + cache_ptr->mdfsm_settled = FALSE; + } /* end if */ + break; + + default: + HDassert(FALSE); /* this should be un-reachable */ + break; + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_unsettle_ring() */ + + +/*------------------------------------------------------------------------- * Function: H5C_validate_resize_config() * * Purpose: Run a sanity check on the specified sections of the @@ -4586,7 +4693,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, ( (entry_ptr->type)->id != H5AC_EPOCH_MARKER_ID ) && ( bytes_evicted < eviction_size_limit ) ) { - hbool_t corked = FALSE; + hbool_t skipping_entry = FALSE; HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert( ! (entry_ptr->is_protected) ); @@ -4600,9 +4707,11 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, prev_is_dirty = prev_ptr->is_dirty; if(entry_ptr->is_dirty ) { + HDassert(!entry_ptr->prefetched_dirty); + /* dirty corked entry is skipped */ if(entry_ptr->tag_info && entry_ptr->tag_info->corked) - corked = TRUE; + skipping_entry = TRUE; else { /* reset entries_removed_counter and * last_entry_removed_ptr prior to the call to @@ -4622,16 +4731,22 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, restart_scan = TRUE; } /* end else */ } /* end if */ - else { + else if(!entry_ptr->prefetched_dirty) { bytes_evicted += entry_ptr->size; if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0 ) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") - } + } /* end else-if */ + else { + HDassert(!entry_ptr->is_dirty); + HDassert(entry_ptr->prefetched_dirty); + + skipping_entry = TRUE; + } /* end else */ if(prev_ptr != NULL) { - if(corked) /* dirty corked entry is skipped */ + if(skipping_entry) entry_ptr = prev_ptr; else if(restart_scan || (prev_ptr->is_dirty != prev_is_dirty) || (prev_ptr->next != next_ptr) @@ -4691,10 +4806,10 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, prev_ptr = entry_ptr->prev; - if(!(entry_ptr->is_dirty)) { + if(!(entry_ptr->is_dirty) && !(entry_ptr->prefetched_dirty)) if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry") - } /* end if */ + /* just skip the entry if it is dirty, as we can't do * anything with it now since we can't write. * @@ -5999,6 +6114,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, hbool_t write_entry; /* internal flag */ hbool_t destroy_entry; /* internal flag */ hbool_t generate_image; /* internal flag */ + hbool_t update_page_buffer; /* internal flag */ hbool_t was_dirty; hbool_t suppress_image_entry_writes = FALSE; hbool_t suppress_image_entry_frees = FALSE; @@ -6024,6 +6140,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, del_from_slist_on_destroy = ((flags & H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) != 0); during_flush = ((flags & H5C__DURING_FLUSH_FLAG) != 0); generate_image = ((flags & H5C__GENERATE_IMAGE_FLAG) != 0); + update_page_buffer = ((flags & H5C__UPDATE_PAGE_BUFFER_FLAG) != 0); /* Set the flag for destroying the entry, based on the 'take ownership' * and 'destroy' flags @@ -6033,10 +6150,6 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, else destroy_entry = destroy; -#ifdef H5_HAVE_PARALLEL - HDassert(FALSE == entry_ptr->coll_access); -#endif - /* we will write the entry to disk if it exists, is dirty, and if the * clear only flag is not set. */ @@ -6232,9 +6345,11 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, * * 2) Delete it from the skip list if requested. * - * 3) Update the replacement policy for eviction + * 3) Delete it from the collective read access list. * - * 4) Remove it from the tag list for this object + * 4) Update the replacement policy for eviction + * + * 5) Remove it from the tag list for this object * * Finally, if the destroy_entry flag is set, discard the * entry. @@ -6244,6 +6359,14 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, if(entry_ptr->in_slist && del_from_slist_on_destroy) H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) +#ifdef H5_HAVE_PARALLEL + /* Check for collective read access flag */ + if(entry_ptr->coll_access) { + entry_ptr->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ + H5C__UPDATE_RP_FOR_EVICTION(cache_ptr, entry_ptr, FAIL) /* Remove entry from tag list */ @@ -6290,7 +6413,8 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") /* Propagate the clean flag up the flush dependency chain if appropriate */ - HDassert(entry_ptr->flush_dep_ndirty_children == 0); + if(entry_ptr->flush_dep_ndirty_children != 0) + HDassert(entry_ptr->flush_dep_ndirty_children == 0); if(entry_ptr->flush_dep_nparents > 0) if(H5C__mark_flush_dep_clean(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag") @@ -6432,6 +6556,19 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, } /* end else */ } /* if (destroy) */ + /* Check if we have to update the page buffer with cleared entries + * so it doesn't go out of date + */ + if(update_page_buffer) { + /* Sanity check */ + HDassert(!destroy); + HDassert(entry_ptr->image_ptr); + + if(f->shared->page_buf && f->shared->page_buf->page_size >= entry_ptr->size) + if(H5PB_update_entry(f->shared->page_buf, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) > 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Failed to update PB with metadata cache") + } /* end if */ + if(cache_ptr->log_flush) if((cache_ptr->log_flush)(cache_ptr, entry_addr, was_dirty, flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "log_flush callback failed") @@ -6822,10 +6959,15 @@ H5C_load_entry(H5F_t * f, entry->prefetched = FALSE; entry->prefetch_type_id = 0; entry->age = 0; + entry->prefetched_dirty = FALSE; #ifndef NDEBUG /* debugging field */ entry->serialization_count = 0; #endif /* NDEBUG */ + entry->tl_next = NULL; + entry->tl_prev = NULL; + entry->tag_info = NULL; + H5C__RESET_CACHE_ENTRY_STATS(entry); ret_value = thing; @@ -6864,13 +7006,6 @@ done: * Thus the function simply does its best, returning success * unless an error is encountered. * - * The primary_dxpl_id and secondary_dxpl_id parameters - * specify the dxpl_ids used on the first write occasioned - * by the call (primary_dxpl_id), and on all subsequent - * writes (secondary_dxpl_id). This is useful in the metadata - * cache, but may not be needed elsewhere. If so, just use the - * same dxpl_id for both parameters. - * * Observe that this function cannot occasion a read. * * Return: Non-negative on success/Negative on failure. @@ -6886,11 +7021,13 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, H5C_t * cache_ptr = f->shared->cache; #if H5C_COLLECT_CACHE_STATS int32_t clean_entries_skipped = 0; + int32_t dirty_pf_entries_skipped = 0; int32_t total_entries_scanned = 0; #endif /* H5C_COLLECT_CACHE_STATS */ uint32_t entries_examined = 0; uint32_t initial_list_len; size_t empty_space; + hbool_t reentrant_call = FALSE; hbool_t prev_is_dirty = FALSE; hbool_t didnt_flush_entry = FALSE; hbool_t restart_scan; @@ -6908,6 +7045,18 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert(cache_ptr->index_size == (cache_ptr->clean_index_size + cache_ptr->dirty_index_size)); + /* check to see if cache_ptr->msic_in_progress is TRUE. If it, this + * is a re-entrant call via a client callback called in the make + * space in cache process. To avoid an infinite recursion, set + * reentrant_call to TRUE, and goto done. + */ + if(cache_ptr->msic_in_progress) { + reentrant_call = TRUE; + HGOTO_DONE(SUCCEED); + } /* end if */ + + cache_ptr->msic_in_progress = TRUE; + if ( write_permitted ) { restart_scan = FALSE; initial_list_len = cache_ptr->LRU_list_len; @@ -6954,7 +7103,8 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, didnt_flush_entry = TRUE; } else if ( ( (entry_ptr->type)->id != H5AC_EPOCH_MARKER_ID ) && - ( ! entry_ptr->flush_in_progress ) ) { + ( ! entry_ptr->flush_in_progress ) && + ( ! entry_ptr->prefetched_dirty ) ) { didnt_flush_entry = FALSE; @@ -6980,13 +7130,6 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, cache_ptr->entries_removed_counter = 0; cache_ptr->last_entry_removed_ptr = NULL; -#ifdef H5_HAVE_PARALLEL - if(TRUE == entry_ptr->coll_access) { - entry_ptr->coll_access = FALSE; - H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) - } /* end if */ -#endif - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") @@ -7020,10 +7163,16 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, } else { - /* Skip epoch markers and entries that are in the process - * of being flushed. + /* Skip epoch markers, entries that are in the process + * of being flushed, and entries marked as prefetched_dirty + * (occurs in the R/O case only). */ didnt_flush_entry = TRUE; + +#if H5C_COLLECT_CACHE_STATS + if(entry_ptr->prefetched_dirty) + dirty_pf_entries_skipped++; +#endif /* H5C_COLLECT_CACHE_STATS */ } if ( prev_ptr != NULL ) { @@ -7088,6 +7237,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, cache_ptr->calls_to_msic++; cache_ptr->total_entries_skipped_in_msic += clean_entries_skipped; + cache_ptr->total_dirty_pf_entries_skipped_in_msic += dirty_pf_entries_skipped; cache_ptr->total_entries_scanned_in_msic += total_entries_scanned; if ( clean_entries_skipped > cache_ptr->max_entries_skipped_in_msic ) { @@ -7095,6 +7245,9 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, cache_ptr->max_entries_skipped_in_msic = clean_entries_skipped; } + if(dirty_pf_entries_skipped > cache_ptr->max_dirty_pf_entries_skipped_in_msic) + cache_ptr->max_dirty_pf_entries_skipped_in_msic = dirty_pf_entries_skipped; + if ( total_entries_scanned > cache_ptr->max_entries_scanned_in_msic ) { cache_ptr->max_entries_scanned_in_msic = total_entries_scanned; @@ -7163,6 +7316,12 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, } done: + /* Sanity checks */ + HDassert(cache_ptr->msic_in_progress); + if(!reentrant_call) + cache_ptr->msic_in_progress = FALSE; + HDassert((!reentrant_call) || (cache_ptr->msic_in_progress)); + FUNC_LEAVE_NOAPI(ret_value) } /* H5C__make_space_in_cache() */ @@ -8723,12 +8882,21 @@ H5C_remove_entry(void *_entry) /* Update the cache internal data structures as appropriate for a destroy. * Specifically: * 1) Delete it from the index - * 2) Update the replacement policy for eviction - * 3) Remove it from the tag list for this object + * 2) Delete it from the collective read access list + * 3) Update the replacement policy for eviction + * 4) Remove it from the tag list for this object */ H5C__DELETE_FROM_INDEX(cache, entry, FAIL) +#ifdef H5_HAVE_PARALLEL + /* Check for collective read access flag */ + if(entry->coll_access) { + entry->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache, entry, FAIL) + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ + H5C__UPDATE_RP_FOR_EVICTION(cache, entry, FAIL) /* Remove entry from tag list */ diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index eb5f123..a955eaf 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -70,6 +70,7 @@ /*******************/ +#ifndef NDEBUG /*------------------------------------------------------------------------- * Function: H5C_dump_cache @@ -175,6 +176,85 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5C_dump_cache() */ +#endif /* NDEBUG */ + +#ifndef NDEBUG + +/*------------------------------------------------------------------------- + * Function: H5C_dump_cache_LRU + * + * Purpose: Print a summary of the contents of the metadata cache + * LRU for debugging purposes. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/10/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name) +{ + H5C_cache_entry_t * entry_ptr; + int i = 0; + + FUNC_ENTER_NOAPI_NOERR + + /* Sanity check */ + HDassert(cache_ptr != NULL); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_name != NULL ); + + HDfprintf(stdout, "\n\nDump of metadata cache LRU \"%s\"\n", cache_name); + HDfprintf(stdout, "LRU len = %d, LRU size = %d\n", + cache_ptr->LRU_list_len, (int)(cache_ptr->LRU_list_size)); + HDfprintf(stdout, "index_size = %d, max_cache_size = %d, delta = %d\n\n", + (int)(cache_ptr->index_size), (int)(cache_ptr->max_cache_size), + (int)(cache_ptr->max_cache_size) - (int)(cache_ptr->index_size)); + + /* Print header */ + HDfprintf(stdout, "Entry "); + HDfprintf(stdout, "| Address "); + HDfprintf(stdout, "| Tag "); + HDfprintf(stdout, "| Size "); + HDfprintf(stdout, "| Ring "); + HDfprintf(stdout, "| Type "); + HDfprintf(stdout, "| Dirty"); + HDfprintf(stdout, "\n"); + + HDfprintf(stdout, "----------------------------------------------------------------------------------------------------------------\n"); + + entry_ptr = cache_ptr->LRU_head_ptr; + while(entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + /* Print entry */ + HDfprintf(stdout, "%s%5d ", cache_ptr->prefix, i); + HDfprintf(stdout, " 0x%16llx ", (long long)(entry_ptr->addr)); + + if(NULL == entry_ptr->tag_info) + HDfprintf(stdout, " %16s ", "N/A"); + else + HDfprintf(stdout, " 0x%16llx ", + (long long)(entry_ptr->tag_info->tag)); + + HDfprintf(stdout, " %5lld ", (long long)(entry_ptr->size)); + HDfprintf(stdout, " %d ", (int)(entry_ptr->ring)); + HDfprintf(stdout, " %2d %-32s ", (int)(entry_ptr->type->id), + (entry_ptr->type->name)); + HDfprintf(stdout, " %d", (int)(entry_ptr->is_dirty)); + HDfprintf(stdout, "\n"); + + i++; + entry_ptr = entry_ptr->next; + } /* end while */ + + HDfprintf(stdout, "----------------------------------------------------------------------------------------------------------------\n"); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5C_dump_cache_LRU() */ +#endif /* NDEBUG */ /*------------------------------------------------------------------------- @@ -385,6 +465,7 @@ H5C_stats(H5C_t * cache_ptr, double average_successful_search_depth = 0.0f; double average_failed_search_depth = 0.0f; double average_entries_skipped_per_calls_to_msic = 0.0f; + double average_dirty_pf_entries_skipped_per_call_to_msic = 0.0f; double average_entries_scanned_per_calls_to_msic = 0.0f; #endif /* H5C_COLLECT_CACHE_STATS */ herr_t ret_value = SUCCEED; /* Return value */ @@ -620,6 +701,17 @@ H5C_stats(H5C_t * cache_ptr, (long)(cache_ptr->max_entries_skipped_in_msic)); if(cache_ptr->calls_to_msic > 0) + average_dirty_pf_entries_skipped_per_call_to_msic = + (((double)(cache_ptr->total_dirty_pf_entries_skipped_in_msic)) / + ((double)(cache_ptr->calls_to_msic))); + + HDfprintf(stdout, + "%s MSIC: Average/max dirty pf entries skipped = %lf / %ld\n", + cache_ptr->prefix, + average_dirty_pf_entries_skipped_per_call_to_msic, + (long)(cache_ptr->max_dirty_pf_entries_skipped_in_msic)); + + if(cache_ptr->calls_to_msic > 0) average_entries_scanned_per_calls_to_msic = (((double)(cache_ptr->total_entries_scanned_in_msic)) / ((double)(cache_ptr->calls_to_msic))); @@ -887,12 +979,14 @@ H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) cache_ptr->max_pel_len = 0; cache_ptr->max_pel_size = (size_t)0; - cache_ptr->calls_to_msic = 0; - cache_ptr->total_entries_skipped_in_msic = 0; - cache_ptr->total_entries_scanned_in_msic = 0; - cache_ptr->max_entries_skipped_in_msic = 0; - cache_ptr->max_entries_scanned_in_msic = 0; - cache_ptr->entries_scanned_to_make_space = 0; + cache_ptr->calls_to_msic = 0; + cache_ptr->total_entries_skipped_in_msic = 0; + cache_ptr->total_dirty_pf_entries_skipped_in_msic = 0; + cache_ptr->total_entries_scanned_in_msic = 0; + cache_ptr->max_entries_skipped_in_msic = 0; + cache_ptr->max_dirty_pf_entries_skipped_in_msic = 0; + cache_ptr->max_entries_scanned_in_msic = 0; + cache_ptr->entries_scanned_to_make_space = 0; cache_ptr->slist_scan_restarts = 0; cache_ptr->LRU_scan_restarts = 0; diff --git a/src/H5Cimage.c b/src/H5Cimage.c index 1da2545..fc58dac 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -147,6 +147,40 @@ H5FL_DEFINE(H5C_cache_entry_t); /*------------------------------------------------------------------------- + * + * Function: H5C_cache_image_pending() + * + * Purpose: Tests to see if the load of a metadata cache image + * load is pending (i.e. will be executed on the next + * protect or insert) + * + * Returns TRUE if a cache image load is pending, and FALSE + * if not. Throws an assertion failure on error. + * + * Return: TRUE if a cache image load is pending, and FALSE otherwise. + * + * Programmer: John Mainzer, 6/18/16 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5C_cache_image_pending(const H5C_t *cache_ptr) +{ + hbool_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + ret_value = (cache_ptr->load_image && !cache_ptr->image_loaded); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_cache_image_pending() */ + + +/*------------------------------------------------------------------------- * Function: H5C_cache_image_status() * * Purpose: Examine the metadata cache associated with the supplied @@ -489,7 +523,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, */ HDassert(!((type->flags & H5C__CLASS_SKIP_READS) && (type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG))); - HDassert(H5F_addr_defined(addr)); HDassert(type->get_initial_load_size); HDassert(type->deserialize); @@ -498,6 +531,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, * relationships now. The client will restore the relationship(s) with * the deserialized entry if appropriate. */ + HDassert(pf_entry_ptr->fd_parent_count == pf_entry_ptr->flush_dep_nparents); for(i = (int)(pf_entry_ptr->fd_parent_count) - 1; i >= 0; i--) { HDassert(pf_entry_ptr->flush_dep_parent); HDassert(pf_entry_ptr->flush_dep_parent[i]); @@ -525,7 +559,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, */ if(pf_entry_ptr->fd_child_count > 0) { if(NULL == (fd_children = (H5C_cache_entry_t **)H5MM_calloc(sizeof(H5C_cache_entry_t **) * (size_t)(pf_entry_ptr->fd_child_count + 1)))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for fd child ptr array") + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for fd child ptr array") if(H5C__destroy_pf_entry_child_flush_deps(cache_ptr, pf_entry_ptr, fd_children) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNDEPEND, FAIL, "can't destroy pf entry child flush dependency(s).") @@ -544,7 +578,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, */ if(NULL == (thing = type->deserialize(pf_entry_ptr->image_ptr, len, udata, &dirty))) HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't deserialize image") - ds_entry_ptr = (H5C_cache_entry_t *)thing; /* In general, an entry should be clean just after it is loaded. @@ -583,8 +616,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, ds_entry_ptr->image_ptr = pf_entry_ptr->image_ptr; ds_entry_ptr->image_up_to_date = !dirty; ds_entry_ptr->type = type; - ds_entry_ptr->is_dirty = dirty | - pf_entry_ptr->is_dirty; + ds_entry_ptr->is_dirty = dirty | pf_entry_ptr->is_dirty; ds_entry_ptr->dirtied = FALSE; ds_entry_ptr->is_protected = FALSE; ds_entry_ptr->is_read_only = FALSE; @@ -626,7 +658,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, pf_entry_ptr->coll_prev = NULL; #endif /* H5_HAVE_PARALLEL */ - /* initialize cache image related fields */ + /* Initialize cache image related fields */ ds_entry_ptr->include_in_image = FALSE; ds_entry_ptr->lru_rank = 0; ds_entry_ptr->image_dirty = FALSE; @@ -638,6 +670,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, ds_entry_ptr->prefetched = FALSE; ds_entry_ptr->prefetch_type_id = 0; ds_entry_ptr->age = 0; + ds_entry_ptr->prefetched_dirty = pf_entry_ptr->prefetched_dirty; +#ifndef NDEBUG /* debugging field */ + ds_entry_ptr->serialization_count = 0; +#endif /* NDEBUG */ H5C__RESET_CACHE_ENTRY_STATS(ds_entry_ptr); @@ -666,7 +702,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, * and H5C__FLUSH_CLEAR_ONLY_FLAG flags set. */ pf_entry_ptr->image_ptr = NULL; - if ( pf_entry_ptr->is_dirty ) { + if(pf_entry_ptr->is_dirty) { HDassert(pf_entry_ptr->in_slist); flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; } /* end if */ @@ -684,7 +720,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, H5C__INSERT_IN_INDEX(cache_ptr, ds_entry_ptr, FAIL) HDassert(!ds_entry_ptr->in_slist); - if(ds_entry_ptr->is_dirty) H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, ds_entry_ptr, FAIL) @@ -699,16 +734,13 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, (ds_entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_LOAD, ds_entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry loaded into cache") - /* restore flush dependencies with the flush dependency children of + /* Restore flush dependencies with the flush dependency children of * of the prefetched entry. Note that we must protect *ds_entry_ptr * before the call to avoid triggering sanity check failures, and * then unprotect it afterwards. */ i = 0; if(fd_children != NULL) { - int j; - hbool_t found; - H5C__UPDATE_RP_FOR_PROTECT(cache_ptr, ds_entry_ptr, FAIL) ds_entry_ptr->is_protected = TRUE; while(fd_children[i] != NULL) { @@ -718,15 +750,22 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, HDassert((fd_children[i])->fd_parent_count > 0); HDassert((fd_children[i])->fd_parent_addrs); - j = 0; - found = FALSE; - while((j < (int)((fd_children[i])->fd_parent_count)) && (!found)) { - if((fd_children[i])->fd_parent_addrs[j] == ds_entry_ptr->addr) - found = TRUE; +#ifndef NDEBUG + { + int j; + hbool_t found; + + j = 0; + found = FALSE; + while((j < (int)((fd_children[i])->fd_parent_count)) && (!found)) { + if((fd_children[i])->fd_parent_addrs[j] == ds_entry_ptr->addr) + found = TRUE; - j++; - } /* end while */ - HDassert(found); + j++; + } /* end while */ + HDassert(found); + } +#endif /* NDEBUG */ if(H5C_create_flush_dependency(ds_entry_ptr, fd_children[i]) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "Can't restore child flush dependency") @@ -735,7 +774,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, } /* end while */ H5C__UPDATE_RP_FOR_UNPROTECT(cache_ptr, ds_entry_ptr, FAIL); - ds_entry_ptr->is_protected = FALSE; } /* end if ( fd_children != NULL ) */ HDassert((unsigned)i == ds_entry_ptr->fd_child_count); @@ -828,6 +866,54 @@ H5C__free_image_entries_array(H5C_t * cache_ptr) /*------------------------------------------------------------------------- + * Function: H5C_force_cache_image_load() + * + * Purpose: On rare occasions, it is necessary to run + * H5MF_tidy_self_referential_fsm_hack() prior to the first + * metadata cache access. This is a problem as if there is a + * cache image at the end of the file, that routine will + * discard it. + * + * We solve this issue by calling this function, which will + * load the cache image and then call + * H5MF_tidy_self_referential_fsm_hack() to discard it. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 1/11/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_force_cache_image_load(H5F_t *f, hid_t dxpl_id) +{ + H5C_t *cache_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->load_image); + + /* Load the cache image, if requested */ + if(cache_ptr->load_image) { + cache_ptr->load_image = FALSE; + if(H5C__load_cache_image(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "can't load cache image") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_force_cache_image_load() */ + + +/*------------------------------------------------------------------------- * Function: H5C_get_cache_image_config * * Purpose: Copy the current configuration for cache image generation @@ -1235,7 +1321,7 @@ H5C__image_entry_cmp(const void *_entry1, const void *_entry2) *------------------------------------------------------------------------- */ herr_t -H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) +H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id, hbool_t *image_generated) { H5C_t * cache_ptr = NULL; haddr_t eoa_frag_addr = HADDR_UNDEF; @@ -1251,6 +1337,7 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) cache_ptr = f->shared->cache; HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(image_generated); /* If the file is opened and closed without any access to * any group or data set, it is possible that the cache image (if @@ -1262,6 +1349,26 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "can't load cache image") } /* end if */ + /* Before we start to generate the cache image (if requested), verify + * that the superblock supports superblock extension messages, and + * silently cancel any request for a cache image if it does not. + * + * Ideally, we would do this when the cache image is requested, + * but the necessary information is not necessary available at that + * time -- hence this last minute check. + * + * Note that under some error conditions, the superblock will be + * undefined in this case as well -- if so, assume that the + * superblock does not support superblock extension messages. + */ + if((NULL == f->shared->sblock) || + (f->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2)) { + H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL; + + cache_ptr->image_ctl = default_image_ctl; + HDassert(!(cache_ptr->image_ctl.generate_image)); + } /* end if */ + /* Generate the cache image, if requested */ if(cache_ptr->image_ctl.generate_image) { /* Create the cache image super block extension message. @@ -1355,6 +1462,15 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) (hsize_t)(cache_ptr->image_data_len), &eoa_frag_addr, &eoa_frag_size))) HGOTO_ERROR(H5E_CACHE, H5E_NOSPACE, FAIL, "can't allocate file space for metadata cache image") + /* Make note of the eoa after allocation of the cache image + * block. This value is used for sanity checking when we + * shutdown the self referential free space managers after + * we destroy the metadata cache. + */ + HDassert(HADDR_UNDEF == f->shared->eoa_post_mdci_fsalloc); + if(HADDR_UNDEF == (f->shared->eoa_post_mdci_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + /* For now, drop any fragment left over from the allocation of the * image block on the ground. A fragment should only be returned * if the underlying file alignment is greater than 1. @@ -1386,7 +1502,7 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) */ if(cache_ptr->image_ctl.flags & H5C_CI__GEN_MDC_IMAGE_BLK) if(H5C__write_cache_image_superblock_msg(f, dxpl_id, FALSE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "update of cache image SB mesg failed.") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "update of cache image SB mesg failed") /* At this point: * @@ -1440,10 +1556,13 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) */ if(cache_ptr->image_ctl.flags & H5C_CI__GEN_MDC_IMAGE_BLK) if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_MDCI_MSG_ID) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove MDC image msg from superblock ext.") + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove MDC image msg from superblock ext") cache_ptr->image_ctl.generate_image = FALSE; } /* end else */ + + /* Indicate that a cache image was generated */ + *image_generated = TRUE; } /* end if */ done: @@ -1458,6 +1577,18 @@ done: * image_ctl field of *cache_ptr. Make adjustments for * changes in configuration as required. * + * If the file is open read only, silently + * force the cache image configuration to its default + * (which disables construction of a cache image). + * + * Note that in addition to being inapplicable in the + * read only case, cache image is also inapplicable if + * the superblock does not support superblock extension + * messages. Unfortunately, this information need not + * be available at this point. Thus we check for this + * later, in H5C_prep_for_file_close() and cancel the + * cache image request if appropriate. + * * Fail if the new configuration is invalid. * * Return: SUCCEED on success, and FAIL on failure. @@ -1483,26 +1614,13 @@ H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr, /* Check arguments */ if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad cache_ptr on entry") - if(config_ptr == NULL) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL config_ptr on entry") - if(config_ptr->version != H5C__CURR_CACHE_IMAGE_CTL_VER) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Unknown config version") - /* check general configuration section of the config: */ + /* Validate the config: */ if(H5C_validate_cache_image_config(config_ptr) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid cache image configuration") - if(H5F_INTENT(f) & H5F_ACC_RDWR) /* file has been opened R/W */ - cache_ptr->image_ctl = *config_ptr; - else { /* file opened R/O -- suppress cache image silently */ - H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL; - - cache_ptr->image_ctl = default_image_ctl; - HDassert(!(cache_ptr->image_ctl.generate_image)); - } /* end else */ - #ifdef H5_HAVE_PARALLEL - /* the collective metadata write code is not currently compatible + /* The collective metadata write code is not currently compatible * with cache image. Until this is fixed, suppress cache image silently * if there is more than one process. * JRM -- 11/8/16 @@ -1513,6 +1631,30 @@ H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr, cache_ptr->image_ctl = default_image_ctl; HDassert(!(cache_ptr->image_ctl.generate_image)); } /* end if */ + else { +#endif /* H5_HAVE_PARALLEL */ + /* A cache image can only be generated if the file is opened read / write + * and the superblock supports superblock extension messages. + * + * However, the superblock version is not available at this point -- + * hence we can only check the former requirement now. Do the latter + * check just before we construct the image.. + * + * If the file is opened read / write, apply the supplied configuration. + * + * If it is not, set the image configuration to the default, which has + * the effect of silently disabling the cache image if it was requested. + */ + if(H5F_INTENT(f) & H5F_ACC_RDWR) + cache_ptr->image_ctl = *config_ptr; + else { + H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL; + + cache_ptr->image_ctl = default_image_ctl; + HDassert(!(cache_ptr->image_ctl.generate_image)); + } /* end else */ +#ifdef H5_HAVE_PARALLEL + } /* end else */ #endif /* H5_HAVE_PARALLEL */ done: @@ -3186,7 +3328,13 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, * this as otherwise the cache will attempt to write them on file * close. Since the file is R/O, the metadata cache image superblock * extension message and the cache image block will not be removed. - * Hence no danger in this. + * Hence no danger in this for subsequent opens. + * + * However, if the dirty entry (marked clean for purposes of the R/O + * file open) is evicted and then referred to, the cache will read + * either invalid or obsolete data from the file. Handle this by + * setting the prefetched_dirty field, and hiding such entries from + * the eviction candidate selection algorithm. */ pf_entry_ptr->is_dirty = (is_dirty && file_is_rw); @@ -3204,6 +3352,8 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, /* Decode dirty dependency child count */ UINT16DECODE(p, pf_entry_ptr->fd_dirty_child_count); + if(!file_is_rw) + pf_entry_ptr->fd_dirty_child_count = 0; if(pf_entry_ptr->fd_dirty_child_count > pf_entry_ptr->fd_child_count) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "invalid dirty flush dependency child count") @@ -3263,6 +3413,7 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, pf_entry_ptr->image_up_to_date = TRUE; pf_entry_ptr->type = H5AC_PREFETCHED_ENTRY; pf_entry_ptr->prefetched = TRUE; + pf_entry_ptr->prefetched_dirty = is_dirty && (!file_is_rw); /* Sanity checks */ HDassert(pf_entry_ptr->size > 0 && pf_entry_ptr->size < H5C_MAX_ENTRY_SIZE); diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index ebb98b3..06ce714 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -65,6 +65,11 @@ /* Local Prototypes */ /********************/ static herr_t H5C__collective_write(H5F_t *f, hid_t dxpl_id); +static herr_t H5C__flush_candidate_entries(H5F_t *f, hid_t dxpl_id, + unsigned entries_to_flush[H5C_RING_NTYPES], + unsigned entries_to_clear[H5C_RING_NTYPES]); +static herr_t H5C__flush_candidates_in_ring(H5F_t *f, hid_t dxpl_id, + H5C_ring_t ring, unsigned entries_to_flush, unsigned entries_to_clear); /*********************/ @@ -175,31 +180,18 @@ H5C_apply_candidate_list(H5F_t * f, int mpi_rank, int mpi_size) { - hbool_t restart_scan; - hbool_t prev_is_dirty; int i; int m; int n; unsigned first_entry_to_flush; unsigned last_entry_to_flush; - unsigned entries_to_clear = 0; - unsigned entries_to_flush = 0; - unsigned entries_to_flush_or_clear_last = 0; - unsigned entries_to_flush_collectively = 0; - unsigned entries_cleared = 0; - unsigned entries_flushed = 0; - unsigned entries_delayed = 0; - unsigned entries_flushed_or_cleared_last = 0; - unsigned entries_flushed_collectively = 0; - unsigned entries_examined = 0; - unsigned initial_list_len; + unsigned total_entries_to_clear = 0; + unsigned total_entries_to_flush = 0; int * candidate_assignment_table = NULL; + unsigned entries_to_flush[H5C_RING_NTYPES]; + unsigned entries_to_clear[H5C_RING_NTYPES]; haddr_t addr; - H5C_cache_entry_t * clear_ptr = NULL; - H5C_cache_entry_t * next_ptr = NULL; H5C_cache_entry_t * entry_ptr = NULL; - H5C_cache_entry_t * flush_ptr = NULL; - H5C_cache_entry_t * delayed_ptr = NULL; #if H5C_DO_SANITY_CHECKS haddr_t last_addr; #endif /* H5C_DO_SANITY_CHECKS */ @@ -220,6 +212,10 @@ H5C_apply_candidate_list(H5F_t * f, HDassert(0 <= mpi_rank); HDassert(mpi_rank < mpi_size); + /* Initialize the entries_to_flush and entries_to_clear arrays */ + HDmemset(entries_to_flush, 0, sizeof(entries_to_flush)); + HDmemset(entries_to_clear, 0, sizeof(entries_to_clear)); + #if H5C_APPLY_CANDIDATE_LIST__DEBUG HDfprintf(stdout, "%s:%d: setting up candidate assignment table.\n", FUNC, mpi_rank); @@ -330,17 +326,26 @@ H5C_apply_candidate_list(H5F_t * f, */ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry is protected?!?!?") + /* Sanity checks */ + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(entry_ptr->ring >= H5C_RING_USER); + HDassert(entry_ptr->ring <= H5C_RING_SB); + HDassert(!entry_ptr->flush_immediately); + HDassert(!entry_ptr->clear_on_unprotect); + /* Determine whether the entry is to be cleared or flushed, * and mark it accordingly. We will scan the protected and * pinned list shortly, and clear or flush according to these * markings. */ if(u >= first_entry_to_flush && u <= last_entry_to_flush) { - entries_to_flush++; + total_entries_to_flush++; + entries_to_flush[entry_ptr->ring]++; entry_ptr->flush_immediately = TRUE; } /* end if */ else { - entries_to_clear++; + total_entries_to_clear++; + entries_to_clear[entry_ptr->ring]++; entry_ptr->clear_on_unprotect = TRUE; } /* end else */ @@ -356,372 +361,36 @@ H5C_apply_candidate_list(H5F_t * f, } /* end if */ } /* end for */ +#if H5C_DO_SANITY_CHECKS + m = 0; + n = 0; + for(i = 0; i < H5C_RING_NTYPES; i++) { + m += (int)entries_to_flush[i]; + n += (int)entries_to_clear[i]; + } /* end if */ + + HDassert((unsigned)m == total_entries_to_flush); + HDassert((unsigned)n == total_entries_to_clear); +#endif /* H5C_DO_SANITY_CHECKS */ + #if H5C_APPLY_CANDIDATE_LIST__DEBUG HDfprintf(stdout, "%s:%d: num candidates/to clear/to flush = %u/%u/%u.\n", - FUNC, mpi_rank, num_candidates, entries_to_clear, - entries_to_flush); + FUNC, mpi_rank, num_candidates, total_entries_to_clear, + total_entries_to_flush); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ /* We have now marked all the entries on the candidate list for * either flush or clear -- now scan the LRU and the pinned list - * for these entries and do the deed. + * for these entries and do the deed. Do this via a call to + * H5C__flush_candidate_entries(). * * Note that we are doing things in this round about manner so as * to preserve the order of the LRU list to the best of our ability. * If we don't do this, my experiments indicate that we will have a * noticably poorer hit ratio as a result. */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: scanning LRU list. len = %d.\n", FUNC, mpi_rank, - (int)(cache_ptr->LRU_list_len)); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* ===================================================================== * - * Now scan the LRU and PEL lists, flushing or clearing entries as - * needed. - * - * The flush_me_last flag may dictate how or - * when some entries can be flushed, and should be addressed here. - * However, in their initial implementation, these flags only apply to the - * superblock, so there's only a relatively small change to this function - * to account for this one case where they come into play. If these flags - * are ever expanded upon, this function and the following flushing steps - * should be reworked to account for additional cases. - * ===================================================================== */ - - HDassert(entries_to_flush >= 0); - - restart_scan = FALSE; - entries_examined = 0; - initial_list_len = cache_ptr->LRU_list_len; - entry_ptr = cache_ptr->LRU_tail_ptr; - - /* Examine each entry in the LRU list */ - while ( ( entry_ptr != NULL ) - && - ( entries_examined <= (entries_to_flush + 1) * initial_list_len ) - && - ( (entries_cleared + entries_flushed) < num_candidates ) ) { - - if ( entry_ptr->prev != NULL ) - prev_is_dirty = entry_ptr->prev->is_dirty; - - /* If this process needs to clear this entry. */ - if(entry_ptr->clear_on_unprotect) { - - HDassert(entry_ptr->is_dirty); - - next_ptr = entry_ptr->next; - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->prev; - entries_cleared++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, - (long long)clear_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* reset entries_removed_counter and - * last_entry_removed_ptr prior to the call to - * H5C__flush_single_entry() so that we can spot - * unexpected removals of entries from the cache, - * and set the restart_scan flag if proceeding - * would be likely to cause us to scan an entry - * that is no longer in the cache. - * - * Note that as of this writing (April 2015) this - * case cannot occur in the parallel case. However - * Quincey is making noises about changing this, hence - * the insertion of this test. - * - * Note also that there is no test code to verify - * that this code actually works (although similar code - * in the serial version exists and is tested). - * - * Implementing a test will likely require implementing - * flush op like facilities in the parallel tests. At - * a guess this will not be terribly painful, but it - * will take a bit of time. - */ - cache_ptr->entries_removed_counter = 0; - cache_ptr->last_entry_removed_ptr = NULL; - - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") - - if((cache_ptr->entries_removed_counter > 1) || - (cache_ptr->last_entry_removed_ptr == entry_ptr)) - restart_scan = TRUE; - } /* end if */ - - /* Else, if this process needs to flush this entry. */ - else if (entry_ptr->flush_immediately) { - - HDassert(entry_ptr->is_dirty); - - next_ptr = entry_ptr->next; - entry_ptr->flush_immediately = FALSE; - flush_ptr = entry_ptr; - entry_ptr = entry_ptr->prev; - entries_flushed++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, - (long long)flush_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* reset entries_removed_counter and - * last_entry_removed_ptr prior to the call to - * H5C__flush_single_entry() so that we can spot - * unexpected removals of entries from the cache, - * and set the restart_scan flag if proceeding - * would be likely to cause us to scan an entry - * that is no longer in the cache. - * - * Note that as of this writing (April 2015) this - * case cannot occur in the parallel case. However - * Quincey is making noises about changing this, hence - * the insertion of this test. - * - * Note also that there is no test code to verify - * that this code actually works (although similar code - * in the serial version exists and is tested). - * - * Implementing a test will likely require implementing - * flush op like facilities in the parallel tests. At - * a guess this will not be terribly painful, but it - * will take a bit of time. - */ - cache_ptr->entries_removed_counter = 0; - cache_ptr->last_entry_removed_ptr = NULL; - - /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") - - if((cache_ptr->entries_removed_counter > 1) || - (cache_ptr->last_entry_removed_ptr == entry_ptr)) - restart_scan = TRUE; - } /* end else-if */ - - /* Otherwise, no action to be taken on this entry. Grab the next. */ - else { - entry_ptr = entry_ptr->prev; - - if ( entry_ptr != NULL ) - next_ptr = entry_ptr->next; - - } /* end else */ - - if ( ( entry_ptr != NULL ) - && - ( ( restart_scan ) - || - ( entry_ptr->is_dirty != prev_is_dirty ) - || - ( entry_ptr->next != next_ptr ) - || - ( entry_ptr->is_protected ) - || - ( entry_ptr->is_pinned ) - ) - ) { - - /* something has happened to the LRU -- start over - * from the tail. - * - * Recall that this code should be un-reachable at present, - * as all the operations by entries on flush that could cause - * it to be reachable are disallowed in the parallel case at - * present. Hence the following assertion which should be - * removed if the above changes. - */ - - HDassert( ! restart_scan ); - HDassert( entry_ptr->is_dirty == prev_is_dirty ); - HDassert( entry_ptr->next == next_ptr ); - HDassert( ! entry_ptr->is_protected ); - HDassert( ! entry_ptr->is_pinned ); - - HDassert(FALSE); /* see comment above */ - - restart_scan = FALSE; - entry_ptr = cache_ptr->LRU_tail_ptr; -/* - H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) -*/ - } - - entries_examined++; - } /* end while */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %u/%u/%u.\n", - FUNC, mpi_rank, entries_examined, - entries_cleared, entries_flushed); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* It is also possible that some of the cleared entries are on the - * pinned list. Must scan that also. - * - * WARNING: - * - * As we now allow unpinning, and removal of other entries as a side - * effect of flushing an entry, it is possible that the next entry - * in a PEL scan could either be no longer pinned, or no longer in - * the cache by the time we get to it. - * - * At present, this is not possible in this case, as we disallow such - * operations in the parallel version of the library. However, Quincey - * has been making noises about relaxing this. If and when he does, - * we have a potential problem here. - * - * The same issue exists in the serial cache, and there are tests - * to detect this problem when it occurs, and adjust to it. As seen - * above in the LRU scan, I have ported such tests to the parallel - * code where a close cognate exists in the serial code. - * - * I haven't done so here, as there are no PEL scans where the problem - * can occur in the serial code. Needless to say, this will have to - * be repaired if the constraints on pre_serialize and serialize - * callbacks are relaxed in the parallel version of the metadata cache. - * - * JRM -- 4/1/15 - */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n", - FUNC, mpi_rank, (int)(cache_ptr->pel_len)); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - entry_ptr = cache_ptr->pel_head_ptr; - while((entry_ptr != NULL) && - ((entries_cleared + entries_flushed + entries_delayed) - < num_candidates)) { - - /* If entry is marked for flush or for clear */ - if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) { - - /* If this entry needs to be flushed last */ - if (entry_ptr->flush_me_last) { - - /* At this time, only the superblock supports being - flushed last. Conveniently, it also happens to be the only - entry that supports being flushed collectively, as well. Also - conveniently, it's always pinned, so we only need to check - for it while scanning the PEL here. Finally, it's never - included in a candidate list that excludes other dirty - entries in a cache, so we can handle this relatively simple - case here. - - For now, this function asserts this and saves the entry - to flush it after scanning the rest of the PEL list. - - If there are ever more entries that either need to be - flushed last and/or flushed collectively, this whole routine - will need to be reworked to handle all additional cases. As - it is the simple case of a single pinned entry needing - flushed last and collectively is just a minor addition to - this routine, but signficantly buffing up the usage of - flush_me_last will require a more - intense rework of this function and potentially the function - of candidate lists as a whole. */ - - entries_to_flush_or_clear_last++; - entries_to_flush_collectively++; - HDassert(entries_to_flush_or_clear_last == 1); - HDassert(entries_to_flush_collectively == 1); - - /* Delay the entry. It will be flushed later. */ - delayed_ptr = entry_ptr; - entries_delayed++; - HDassert(entries_delayed == 1); - - } /* end if */ - - /* Else, this process needs to clear this entry. */ - else if (entry_ptr->clear_on_unprotect) { - HDassert(!entry_ptr->flush_immediately); - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_cleared++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, - (long long)clear_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") - } /* end else-if */ - - /* Else, if this process needs to independently flush this entry. */ - else if (entry_ptr->flush_immediately) { - entry_ptr->flush_immediately = FALSE; - flush_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_flushed++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, - (long long)flush_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") - } /* end else-if */ - } /* end if */ - - /* Otherwise, this entry is not marked for flush or clear. Grab the next. */ - else { - entry_ptr = entry_ptr->next; - } /* end else */ - - } /* end while */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, - "%s:%d: pel entries examined/cleared/flushed = %u/%u/%u.\n", - FUNC, mpi_rank, entries_examined, - entries_cleared, entries_flushed); - HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank); - - HDfsync(stdout); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* ====================================================================== * - * Now, handle all delayed entries. * - * * - * This can *only* be the superblock at this time, so it's relatively * - * easy to deal with. We're collectively flushing the entry saved from * - * above. This will need to be handled differently if there are ever more * - * than one entry needing this special treatment.) * - * ====================================================================== */ - - if (delayed_ptr) { - - if (delayed_ptr->clear_on_unprotect) { - if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") - - entry_ptr->clear_on_unprotect = FALSE; - entries_cleared++; - } else if (delayed_ptr->flush_immediately) { - /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry collectively.") - - entry_ptr->flush_immediately = FALSE; - entries_flushed++; - } /* end if */ - - entries_flushed_collectively++; - entries_flushed_or_cleared_last++; - } /* end if */ + if(H5C__flush_candidate_entries(f, dxpl_id, entries_to_flush, entries_to_clear) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates failed") /* If we've deferred writing to do it collectively, take care of that now */ if(f->coll_md_write) { @@ -733,21 +402,6 @@ H5C_apply_candidate_list(H5F_t * f, HGOTO_ERROR(H5E_CACHE, H5E_WRITEERROR, FAIL, "can't write metadata collectively") } /* end if */ - /* ====================================================================== * - * Finished flushing everything. * - * ====================================================================== */ - - HDassert((entries_flushed == entries_to_flush)); - HDassert((entries_cleared == entries_to_clear)); - HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last)); - HDassert((entries_flushed_collectively == entries_to_flush_collectively)); - - if((entries_flushed != entries_to_flush) || - (entries_cleared != entries_to_clear) || - (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) || - (entries_flushed_collectively != entries_to_flush_collectively)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch") - done: if(candidate_assignment_table != NULL) candidate_assignment_table = (int *)H5MM_xfree((void *)candidate_assignment_table); @@ -998,11 +652,13 @@ H5C_mark_entries_as_clean(H5F_t * f, { H5C_t * cache_ptr; unsigned entries_cleared; + unsigned pinned_entries_cleared; + hbool_t progress; unsigned entries_examined; unsigned initial_list_len; haddr_t addr; -#if H5C_DO_SANITY_CHECKS unsigned pinned_entries_marked = 0; +#if H5C_DO_SANITY_CHECKS unsigned protected_entries_marked = 0; unsigned other_entries_marked = 0; haddr_t last_addr; @@ -1086,19 +742,13 @@ H5C_mark_entries_as_clean(H5F_t * f, } /* end if */ entry_ptr->clear_on_unprotect = TRUE; + if(entry_ptr->is_pinned) + pinned_entries_marked++; #if H5C_DO_SANITY_CHECKS - if ( entry_ptr->is_protected ) { - + else if(entry_ptr->is_protected) protected_entries_marked++; - - } else if ( entry_ptr->is_pinned ) { - - pinned_entries_marked++; - - } else { - + else other_entries_marked++; - } #endif /* H5C_DO_SANITY_CHECKS */ } } @@ -1140,7 +790,8 @@ H5C_mark_entries_as_clean(H5F_t * f, entry_ptr = entry_ptr->prev; entries_cleared++; - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry") } /* end if */ else @@ -1155,19 +806,27 @@ H5C_mark_entries_as_clean(H5F_t * f, /* It is also possible that some of the cleared entries are on the * pinned list. Must scan that also. */ - entry_ptr = cache_ptr->pel_head_ptr; - while(entry_ptr != NULL) { - if(entry_ptr->clear_on_unprotect) { - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_cleared++; + pinned_entries_cleared = 0; + progress = TRUE; + while((pinned_entries_cleared < pinned_entries_marked) && progress) { + progress = FALSE; + entry_ptr = cache_ptr->pel_head_ptr; + while(entry_ptr != NULL) { + if(entry_ptr->clear_on_unprotect && entry_ptr->flush_dep_ndirty_children == 0) { + entry_ptr->clear_on_unprotect = FALSE; + clear_ptr = entry_ptr; + entry_ptr = entry_ptr->next; + entries_cleared++; + pinned_entries_cleared++; + progress = TRUE; - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry") - } /* end if */ - else - entry_ptr = entry_ptr->next; + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry") + } /* end if */ + else + entry_ptr = entry_ptr->next; + } /* end while */ } /* end while */ #if H5C_DO_SANITY_CHECKS @@ -1297,7 +956,6 @@ H5C__collective_write(H5F_t *f, hid_t dxpl_id) /* Get number of entries in collective write list */ count = (int)H5SL_count(cache_ptr->coll_write_list); - if(count > 0) { H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_COLLECTIVE; H5SL_node_t *node; @@ -1410,4 +1068,511 @@ done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5C__collective_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5C__flush_candidate_entries + * + * Purpose: Flush or clear (as indicated) the candidate entries that + * have been marked in the metadata cache. In so doing, + * observe rings and flush dependencies. + * + * Note that this function presumes that: + * + * 1) no candidate entries are protected, + * + * 2) all candidate entries are dirty, and + * + * 3) if a candidate entry has a dirty flush dependency + * child, that child is also a candidate entry. + * + * The function will fail if any of these preconditions are + * not met. + * + * Candidate entries are marked by setting either the + * flush_immediately or the clear_on_unprotect flags in the + * cache entry (but not both). Entries marked flush_immediately + * will be flushed, those marked clear_on_unprotect will be + * cleared. + * + * Note that this function is a modified version of + * H5C_flush_cache() -- any changes there may need to be + * reflected here and vise versa. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer + * 2/10/17 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5C__flush_candidate_entries(H5F_t *f, hid_t dxpl_id, + unsigned entries_to_flush[H5C_RING_NTYPES], + unsigned entries_to_clear[H5C_RING_NTYPES]) +{ +#if H5C_DO_SANITY_CHECKS + int i; + uint32_t index_len = 0; + size_t index_size = (size_t)0; + size_t clean_index_size = (size_t)0; + size_t dirty_index_size = (size_t)0; + size_t slist_size = (size_t)0; + uint32_t slist_len = 0; +#endif /* H5C_DO_SANITY_CHECKS */ + H5C_ring_t ring; + H5C_t * cache_ptr; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(f); + HDassert(f->shared); + + cache_ptr = f->shared->cache; + + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_ptr); + + HDassert(entries_to_flush[H5C_RING_UNDEFINED] == 0); + HDassert(entries_to_clear[H5C_RING_UNDEFINED] == 0); + +#if H5C_DO_SANITY_CHECKS + HDassert(cache_ptr->index_ring_len[H5C_RING_UNDEFINED] == 0); + HDassert(cache_ptr->index_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + HDassert(cache_ptr->clean_index_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + HDassert(cache_ptr->dirty_index_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + HDassert(cache_ptr->slist_ring_len[H5C_RING_UNDEFINED] == 0); + HDassert(cache_ptr->slist_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + + for(i = H5C_RING_USER; i < H5C_RING_NTYPES; i++) { + index_len += cache_ptr->index_ring_len[i]; + index_size += cache_ptr->index_ring_size[i]; + clean_index_size += cache_ptr->clean_index_ring_size[i]; + dirty_index_size += cache_ptr->dirty_index_ring_size[i]; + + slist_len += cache_ptr->slist_ring_len[i]; + slist_size += cache_ptr->slist_ring_size[i]; + } /* end for */ + + HDassert(cache_ptr->index_len == index_len); + HDassert(cache_ptr->index_size == index_size); + HDassert(cache_ptr->clean_index_size == clean_index_size); + HDassert(cache_ptr->dirty_index_size == dirty_index_size); + HDassert(cache_ptr->slist_len == slist_len); + HDassert(cache_ptr->slist_size == slist_size); +#endif /* H5C_DO_SANITY_CHECKS */ + +#if H5C_DO_EXTREME_SANITY_CHECKS + if(H5C_validate_protected_entry_list(cache_ptr) < 0 + || H5C_validate_pinned_entry_list(cache_ptr) < 0 + || H5C_validate_lru_list(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + + cache_ptr->flush_in_progress = TRUE; + + /* flush each ring, starting from the outermost ring and + * working inward. + */ + ring = H5C_RING_USER; + while(ring < H5C_RING_NTYPES) { + if(H5C__flush_candidates_in_ring(f, dxpl_id, ring, entries_to_flush[ring], entries_to_clear[ring]) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates in ring failed") + + ring++; + } /* end while */ + +done: + cache_ptr->flush_in_progress = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__flush_candidate_entries() */ + + +/*------------------------------------------------------------------------- + * Function: H5C__flush_candidates_in_ring + * + * Purpose: Flush or clear (as indicated) the candidate entries + * contained in the specified cache and ring. All candidate + * entries in rings outside the specified ring must have been + * flushed (or cleared) on entry. + * + * Note that this function presumes that: + * + * 1) no candidate entries are protected, + * + * 2) all candidate entries are dirty, and + * + * 3) if a candidate entry has a dirty flush dependency + * child, that child is also a candidate entry. + * + * The function will fail if any of these preconditions are + * not met. + * + * Candidate entries are marked by setting either the + * flush_immediately or the clear_on_unprotect flags in the + * cache entry (but not both). Entries marked flush_immediately + * will be flushed, those marked clear_on_unprotect will be + * cleared. + * + * Candidate entries residing in the LRU must be flushed + * (or cleared) in LRU order to avoid performance issues. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer + * 2/10/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5C__flush_candidates_in_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, + unsigned entries_to_flush, unsigned entries_to_clear) +{ + H5C_t * cache_ptr; + hbool_t progress; + hbool_t restart_scan = FALSE; + unsigned entries_flushed = 0; + unsigned entries_cleared = 0; +#if H5C_DO_SANITY_CHECKS + unsigned init_index_len; +#endif /* H5C_DO_SANITY_CHECKS */ + unsigned clear_flags = H5C__FLUSH_CLEAR_ONLY_FLAG | + H5C__GENERATE_IMAGE_FLAG | + H5C__UPDATE_PAGE_BUFFER_FLAG; + unsigned flush_flags = H5C__NO_FLAGS_SET; + unsigned op_flags; + H5C_cache_entry_t *op_ptr; + H5C_cache_entry_t *entry_ptr; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_ptr); + HDassert(ring > H5C_RING_UNDEFINED); + HDassert(ring < H5C_RING_NTYPES); + +#if H5C_DO_EXTREME_SANITY_CHECKS + if((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || + (H5C_validate_lru_list(cache_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + +#if H5C_DO_SANITY_CHECKS + /* index len should not change */ + init_index_len = cache_ptr->index_len; +#endif /* H5C_DO_SANITY_CHECKS */ + + /* Examine entries in the LRU list, and flush or clear all entries + * so marked in the target ring. + * + * With the current implementation of flush dependencies, no entry + * in the LRU can have flush dependency children -- thus one pass + * through the LRU will be sufficient. + * + * It is possible that this will change -- hence the assertion. + */ + restart_scan = FALSE; + entry_ptr = cache_ptr->LRU_tail_ptr; + while(((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear)) + && (entry_ptr != NULL)) { + hbool_t prev_is_dirty = FALSE; + H5C_cache_entry_t *next_ptr; + + /* Entries in the LRU must not have flush dependency children */ + HDassert(entry_ptr->flush_dep_nchildren == 0); + + /* Remember dirty state of entry to advance to */ + if(entry_ptr->prev != NULL) + prev_is_dirty = entry_ptr->prev->is_dirty; + + /* If the entry is in the ring */ + if(entry_ptr->ring == ring) { + /* If this process needs to clear this entry. */ + if(entry_ptr->clear_on_unprotect) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = clear_flags; + + /* Set next entry appropriately */ + next_ptr = entry_ptr->next; + + /* Reset entry flag */ + entry_ptr->clear_on_unprotect = FALSE; + entries_cleared++; + } /* end if */ + else if(entry_ptr->flush_immediately) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = flush_flags; + + /* Set next entry appropriately */ + next_ptr = entry_ptr->next; + + /* Reset entry flag */ + entry_ptr->flush_immediately = FALSE; + entries_flushed++; + } /* end else-if */ + else { + /* No operation for this entry */ + op_ptr = NULL; + + /* Set next entry appropriately */ + next_ptr = entry_ptr; + } /* end else */ + + /* Advance to next entry */ + entry_ptr = entry_ptr->prev; + + /* Check for operation */ + if(op_ptr) { + /* reset entries_removed_counter and + * last_entry_removed_ptr prior to the call to + * H5C__flush_single_entry() so that we can spot + * unexpected removals of entries from the cache, + * and set the restart_scan flag if proceeding + * would be likely to cause us to scan an entry + * that is no longer in the cache. + * + * Note that as of this writing, this + * case cannot occur in the parallel case. + * + * Note also that there is no test code to verify + * that this code actually works (although similar code + * in the serial version exists and is tested). + */ + cache_ptr->entries_removed_counter = 0; + cache_ptr->last_entry_removed_ptr = NULL; + + if(H5C__flush_single_entry(f, dxpl_id, op_ptr, op_flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush entry") + + if(cache_ptr->entries_removed_counter != 0 + || cache_ptr->last_entry_removed_ptr != NULL) + restart_scan = TRUE; + } /* end if */ + } /* end if */ + else { + /* Remember "next" pointer (after advancing entries) */ + next_ptr = entry_ptr; + + /* Advance to next entry */ + entry_ptr = entry_ptr->prev; + } /* end else */ + + /* Check for restarts, etc. */ + if((entry_ptr != NULL) && + (restart_scan || (entry_ptr->is_dirty != prev_is_dirty) + || (entry_ptr->next != next_ptr) || entry_ptr->is_protected + || entry_ptr->is_pinned)) { + + /* Something has happened to the LRU -- start over + * from the tail. + * + * Recall that this code should be un-reachable at present, + * as all the operations by entries on flush that could cause + * it to be reachable are disallowed in the parallel case at + * present. Hence the following assertion which should be + * removed if the above changes. + */ + HDassert(!restart_scan); + HDassert(entry_ptr->is_dirty == prev_is_dirty); + HDassert(entry_ptr->next == next_ptr); + HDassert(!entry_ptr->is_protected); + HDassert(!entry_ptr->is_pinned); + + HDassert(FALSE); /* see comment above */ + + restart_scan = FALSE; + entry_ptr = cache_ptr->LRU_tail_ptr; + + H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) + } /* end if */ + } /* end while */ + + /* It is also possible that some of the cleared entries are on the + * pinned list. Must scan that also. + * + * Observe that in the case of the pinned entry list, most of the + * entries will have flush dependency children. As entries with + * flush dependency children may not be flushed until all of their + * children are clean, multiple passes throguh the pinned entry list + * may be required. + * + * WARNING: + * + * As we now allow unpinning, and removal of other entries as a side + * effect of flushing an entry, it is possible that the next entry + * in a PEL scan could either be no longer pinned, or no longer in + * the cache by the time we get to it. + * + * At present, this should not be possible in this case, as we disallow + * such operations in the parallel version of the library. However, + * this may change, and to that end, I have included code to detect + * such changes and cause this function to fail if they are detected. + */ + progress = TRUE; + while(progress && ((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear))) { + progress = FALSE; + entry_ptr = cache_ptr->pel_head_ptr; + while((entry_ptr != NULL) && + ((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear))) { + H5C_cache_entry_t *prev_ptr; + hbool_t next_is_dirty = FALSE; + + HDassert(entry_ptr->is_pinned); + + /* Remember dirty state of entry to advance to */ + if(entry_ptr->next != NULL) + next_is_dirty = entry_ptr->next->is_dirty; + + if(entry_ptr->ring == ring && entry_ptr->flush_dep_ndirty_children == 0) { + if(entry_ptr->clear_on_unprotect) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = clear_flags; + + /* Reset entry flag */ + entry_ptr->clear_on_unprotect = FALSE; + entries_cleared++; + progress = TRUE; + } /* end if */ + else if(entry_ptr->flush_immediately) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = flush_flags; + + /* Reset entry flag */ + entry_ptr->flush_immediately = FALSE; + entries_flushed++; + progress = TRUE; + } /* end else-if */ + else + /* No operation for this entry */ + op_ptr = NULL; + + /* Check for operation */ + if(op_ptr) { + /* reset entries_removed_counter and + * last_entry_removed_ptr prior to the call to + * H5C__flush_single_entry() so that we can spot + * unexpected removals of entries from the cache, + * and set the restart_scan flag if proceeding + * would be likely to cause us to scan an entry + * that is no longer in the cache. + * + * Note that as of this writing, this + * case cannot occur in the parallel case. + * + * Note also that there is no test code to verify + * that this code actually works (although similar code + * in the serial version exists and is tested). + */ + cache_ptr->entries_removed_counter = 0; + cache_ptr->last_entry_removed_ptr = NULL; + + /* Add this entry to the list of entries to collectively write + * + * This comment is misleading -- the entry will be added to the + * collective write list only if said list exists. + * + * JRM -- 2/9/17 + */ + if(H5C__flush_single_entry(f, dxpl_id, op_ptr, op_flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush entry") + + if(cache_ptr->entries_removed_counter != 0 + || cache_ptr->last_entry_removed_ptr != NULL) + restart_scan = TRUE; + } /* end if */ + } /* end if */ + + /* Remember "previous" pointer (after advancing entries) */ + prev_ptr = entry_ptr; + + /* Advance to next entry */ + entry_ptr = entry_ptr->next; + + /* Check for restarts, etc. */ + if((entry_ptr != NULL) && + (restart_scan || (entry_ptr->is_dirty != next_is_dirty) + || (entry_ptr->prev != prev_ptr) || entry_ptr->is_protected + || !entry_ptr->is_pinned)) { + /* Something has happened to the pinned entry list -- start + * over from the head. + * + * Recall that this code should be un-reachable at present, + * as all the operations by entries on flush that could cause + * it to be reachable are disallowed in the parallel case at + * present. Hence the following assertion which should be + * removed if the above changes. + */ + + HDassert(!restart_scan); + HDassert(entry_ptr->is_dirty == next_is_dirty); + HDassert(entry_ptr->prev == prev_ptr); + HDassert(!entry_ptr->is_protected); + HDassert(entry_ptr->is_pinned); + + HDassert(FALSE); /* see comment above */ + + restart_scan = FALSE; + + entry_ptr = cache_ptr->pel_head_ptr; + + /* we don't keeps stats for pinned entry list scan + * restarts. If this code ever becomes reachable, + * define the necessary field, and implement the + * the following macro: + * + * H5C__UPDATE_STATS_FOR_PEL_SCAN_RESTART(cache_ptr) + */ + } /* end if */ + } /* end while ( ( entry_ptr != NULL ) && + * ( ( entries_flushed > entries_to_flush ) || + * ( entries_cleared > entries_to_clear ) ) ) + */ + } /* end while ( ( ( entries_flushed > entries_to_flush ) || + * ( entries_cleared > entries_to_clear ) ) && + * ( progress ) ) + */ + +#if H5C_DO_SANITY_CHECKS + HDassert(init_index_len == cache_ptr->index_len); +#endif /* H5C_DO_SANITY_CHECKS */ + + if(entries_flushed != entries_to_flush || entries_cleared != entries_to_clear) { + entry_ptr = cache_ptr->il_head; + while(entry_ptr != NULL) { + HDassert(!entry_ptr->clear_on_unprotect || (entry_ptr->ring > ring)); + HDassert(!entry_ptr->flush_immediately || (entry_ptr->ring > ring)); + entry_ptr = entry_ptr->il_next; + } /* end while */ + + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush/clear all entries") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__flush_candidates_in_ring() */ #endif /* H5_HAVE_PARALLEL */ + diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 5b923e9..69e8145 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -4104,6 +4104,23 @@ typedef struct H5C_tag_info_t { * of changes to the file driver info superblock extension * management code needed to support rings. * + * msic_in_progress: As the metadata cache has become re-entrant, and as + * the free space manager code has become more tightly + * integrated with the metadata cache, it is possible that + * a call to H5C_insert_entry() may trigger a call to + * H5C_make_space_in_cache(), which, via H5C__flush_single_entry() + * and client callbacks, may trigger an infinite regression + * of calls to H5C_make_space_in_cache(). + * + * The msic_in_progress boolean flag is used to detect this, + * and prevent the infinite regression that would otherwise + * occur. + * + * Note that this is issue is not hypothetical -- this field + * was added 2/16/17 to address this issue when it was + * exposed by modifications to test/fheap.c to cause it to + * use paged allocation. + * * resize_ctl: Instance of H5C_auto_size_ctl_t containing configuration * data for automatic cache resizing. * @@ -4500,12 +4517,22 @@ typedef struct H5C_tag_info_t { * total_entries_skipped_in_msic: Number of clean entries skipped while * enforcing the min_clean_fraction in H5C__make_space_in_cache(). * + * total_dirty_pf_entries_skipped_in_msic: Number of dirty prefetched entries + * skipped in H5C__make_space_in_cache(). Note that this can + * only occur when a file is opened R/O with a cache image + * containing dirty entries. + * * total_entries_scanned_in_msic: Number of clean entries skipped while * enforcing the min_clean_fraction in H5C__make_space_in_cache(). * * max_entries_skipped_in_msic: Maximum number of clean entries skipped * in any one call to H5C__make_space_in_cache(). * + * max_dirty_pf_entries_skipped_in_msic: Maximum number of dirty prefetched + * entries skipped in any one call to H5C__make_space_in_cache(). + * Note that this can only occur when the file is opened + * R/O with a cache image containing dirty entries. + * * max_entries_scanned_in_msic: Maximum number of entries scanned over * in any one call to H5C__make_space_in_cache(). * @@ -4733,6 +4760,7 @@ struct H5C_t { hbool_t cache_full; hbool_t size_decreased; hbool_t resize_in_progress; + hbool_t msic_in_progress; H5C_auto_size_ctl_t resize_ctl; /* Fields for epoch markers used in automatic cache size adjustment */ @@ -4822,8 +4850,10 @@ struct H5C_t { /* Fields for tracking 'make space in cache' (msic) operations */ int64_t calls_to_msic; int64_t total_entries_skipped_in_msic; + int64_t total_dirty_pf_entries_skipped_in_msic; int64_t total_entries_scanned_in_msic; int32_t max_entries_skipped_in_msic; + int32_t max_dirty_pf_entries_skipped_in_msic; int32_t max_entries_scanned_in_msic; int64_t entries_scanned_to_make_space; @@ -4871,7 +4901,8 @@ typedef int (*H5C_tag_iter_cb_t)(H5C_cache_entry_t *entry, void *ctx); /******************************/ /* Package Private Prototypes */ /******************************/ -H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id, + hbool_t *image_generated); H5_DLL herr_t H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t * cache_ptr, H5C_cache_entry_t** entry_ptr_ptr, const H5C_class_t * type, haddr_t addr, void * udata); diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 28eacf2..539dece 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -185,6 +185,7 @@ * H5C__TAKE_OWNERSHIP_FLAG * H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG * H5C__GENERATE_IMAGE_FLAG + * H5C__UPDATE_PAGE_BUFFER_FLAG */ #define H5C__NO_FLAGS_SET 0x00000 #define H5C__SET_FLUSH_MARKER_FLAG 0x00001 @@ -205,6 +206,7 @@ #define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x08000 #define H5C__DURING_FLUSH_FLAG 0x10000 /* Set when the entire cache is being flushed */ #define H5C__GENERATE_IMAGE_FLAG 0x20000 /* Set during parallel I/O */ +#define H5C__UPDATE_PAGE_BUFFER_FLAG 0x40000 /* Set during parallel I/O */ /* Debugging/sanity checking/statistics settings */ #ifndef NDEBUG @@ -1066,9 +1068,9 @@ typedef int H5C_ring_t; * * is_read_only: Boolean flag that is only meaningful if is_protected is * TRUE. In this circumstance, it indicates whether the - * entry has been protected read only, or read/write. + * entry has been protected read-only, or read/write. * - * If the entry has been protected read only (i.e. is_protected + * If the entry has been protected read-only (i.e. is_protected * and is_read_only are both TRUE), we allow the entry to be * protected more than once. * @@ -1078,7 +1080,7 @@ typedef int H5C_ring_t; * the entry is actually unprotected. * * ro_ref_count: Integer field used to maintain a count of the number of - * outstanding read only protects on this entry. This field + * outstanding read-only protects on this entry. This field * must be zero whenever either is_protected or is_read_only * are TRUE. * @@ -1458,13 +1460,13 @@ typedef int H5C_ring_t; * the load of a cache image block, although other scenarios * are contemplated for the use of this feature. Note that * unlike the usual prefetch situation, this means that a - * pre fetched entry can be dirty, and/or can be a party to + * prefetched entry can be dirty, and/or can be a party to * flush dependency relationship(s). This complicates matters * somewhat. * - * The essential feature of a pre-fetched entry is that it + * The essential feature of a prefetched entry is that it * consists only of a buffer containing the on disk image of - * the entry. Thus it must be deserialized before it can + * the entry. Thus it must be deserialized before it can * be passed back to the library on a protect call. This * task is handled by H5C_deserialized_prefetched_entry(). * In essence, this routine calls the deserialize callback @@ -1475,7 +1477,7 @@ typedef int H5C_ring_t; * * Further, if the prefetched entry is a flush dependency parent, * all its flush dependency children (which must also be - * pre-fetched entries), must be tranfered to the new cache + * prefetched entries), must be tranfered to the new cache * entry returned by the deserailization callback. * * Finally, if the prefetched entry is a flush dependency child, @@ -1511,6 +1513,46 @@ typedef int H5C_ring_t; * * This field must be zero if prefetched is FALSE. * + * prefetched_dirty: Boolean field that must be set to FALSE unless the + * following conditions hold: + * + * 1) The file has been opened R/O. + * + * 2) The entry is either a prefetched entry, or was + * re-constructed from a prefetched entry. + * + * 3) The base prefetched entry was marked dirty. + * + * This field exists to solve the following problem with + * files containing cache images that are opened R/O. + * + * If the cache image contains a dirty entry, that entry + * must be marked clean when it is inserted into the cache + * in the read-only case, as otherwise the metadata cache + * will attempt to flush it on file close -- which is poor + * form in the read-only case. + * + * However, since the entry is marked clean, it is possible + * that the metadata cache will evict it if the size of the + * metadata in the file exceeds the size of the metadata cache, + * and the application visits much of this data. + * + * If this happens, and the metadata cache is then asked for + * this entry, it will attempt to read it from file, and will + * obtain either obsolete or invalid data depending on whether + * the entry has ever been written to it assigned location in + * the file. + * + * With this background, the purpose of this field should be + * obvious -- when set, it allows the eviction candidate + * selection code to skip over the entry, thus avoiding the + * issue. + * + * Since the issue only arises in the R/O case, there is + * no possible interaction with SWMR. There are also + * potential interactions with Evict On Close -- at present, + * we deal with this by disabling EOC in the R/O case. + * * serialization_count: Integer field used to maintain a count of the * number of times each entry is serialized during cache * serialization. While no entry should be serialized more than @@ -1627,6 +1669,7 @@ typedef struct H5C_cache_entry_t { hbool_t prefetched; int prefetch_type_id; int32_t age; + hbool_t prefetched_dirty; #ifndef NDEBUG /* debugging field */ int serialization_count; @@ -2208,6 +2251,7 @@ H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, unsigned flags); H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); +H5_DLL herr_t H5C_force_cache_image_load(H5F_t * f, hid_t dxpl_id); H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global); H5_DLL herr_t H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); H5_DLL herr_t H5C_get_tag(const void *thing, /*OUT*/ haddr_t *tag); @@ -2259,7 +2303,6 @@ H5_DLL herr_t H5C_set_trace_file_ptr(H5C_t *cache_ptr, FILE *trace_file_ptr); H5_DLL herr_t H5C_stats(H5C_t *cache_ptr, const char *cache_name, hbool_t display_detailed_stats); H5_DLL void H5C_stats__reset(H5C_t *cache_ptr); -H5_DLL herr_t H5C_dump_cache(H5C_t *cache_ptr, const char *cache_name); H5_DLL herr_t H5C_unpin_entry(void *thing); H5_DLL herr_t H5C_destroy_flush_dependency(void *parent_thing, void *child_thing); H5_DLL herr_t H5C_unprotect(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *thing, @@ -2273,9 +2316,11 @@ H5_DLL herr_t H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked); H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring); H5_DLL herr_t H5C_unsettle_entry_ring(void *thing); +H5_DLL herr_t H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring); H5_DLL herr_t H5C_remove_entry(void *thing); H5_DLL herr_t H5C_cache_image_status(H5F_t * f, hbool_t *load_ci_ptr, hbool_t *write_ci_ptr); +H5_DLL hbool_t H5C_cache_image_pending(const H5C_t *cache_ptr); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id, @@ -2289,6 +2334,8 @@ H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, unsigned ce_arr #endif /* H5_HAVE_PARALLEL */ #ifndef NDEBUG /* debugging functions */ +H5_DLL herr_t H5C_dump_cache(H5C_t *cache_ptr, const char *cache_name); +H5_DLL herr_t H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name); H5_DLL hbool_t H5C_get_serialization_in_progress(const H5C_t *cache_ptr); H5_DLL hbool_t H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring); H5_DLL herr_t H5C_dump_cache_skip_list(H5C_t *cache_ptr, char *calling_fcn); diff --git a/src/H5Ctag.c b/src/H5Ctag.c index 157a838..a9bcca1 100644 --- a/src/H5Ctag.c +++ b/src/H5Ctag.c @@ -465,7 +465,7 @@ H5C__evict_tagged_entries_cb(H5C_cache_entry_t *entry, void *_ctx) entry and we'll loop back around again (as evicting other entries will hopefully unpin this entry) */ ctx->pinned_entries_need_evicted = TRUE; - else { + else if(!entry->prefetched_dirty) { /* Evict the Entry */ if(H5C__flush_single_entry(ctx->f, ctx->dxpl_id, entry, H5C__FLUSH_INVALIDATE_FLAG | H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, H5_ITER_ERROR, "Entry eviction failed.") @@ -406,7 +406,7 @@ H5Fis_hdf5(const char *name) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "no file name specified") /* call the private is_HDF5 function */ - if((ret_value = H5F_is_hdf5(name, H5AC_ind_read_dxpl_id)) < 0) + if((ret_value = H5F__is_hdf5(name, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable open file") done: @@ -535,7 +535,7 @@ done: * Modifications: * Robb Matzke, 1997-07-18 * File struct creation and destruction is through H5F_new() and - * H5F_dest(). Reading the root symbol table entry is done with + * H5F__dest(). Reading the root symbol table entry is done with * H5G_decode(). * * Robb Matzke, 1997-09-23 @@ -713,12 +713,12 @@ H5Fflush(hid_t object_id, H5F_scope_t scope) /* Flush other files, depending on scope */ if(H5F_SCOPE_GLOBAL == scope) { /* Call the flush routine for mounted file hierarchies */ - if(H5F_flush_mounts(f, H5AC_ind_read_dxpl_id) < 0) + if(H5F_flush_mounts(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush mounted file hierarchy") } /* end if */ else { /* Call the flush routine, for this file */ - if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") } /* end else */ } /* end if */ @@ -773,7 +773,7 @@ H5Fclose(hid_t file_id) if((nref = H5I_get_ref(file_id, FALSE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTGET, FAIL, "can't get ID ref count") if(nref == 1) - if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache") } /* end if */ @@ -838,7 +838,7 @@ H5Freopen(hid_t file_id) done: if(ret_value < 0 && new_file) - if(H5F_dest(new_file, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__dest(new_file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file") FUNC_LEAVE_API(ret_value) @@ -1042,7 +1042,7 @@ H5Fget_file_image(hid_t file_id, void *buf_ptr, size_t buf_len) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") /* call private get_file_image function */ - if((ret_value = H5F_get_file_image(file, buf_ptr, buf_len, H5AC_ind_read_dxpl_id)) < 0) + if((ret_value = H5F_get_file_image(file, buf_ptr, buf_len, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file image") done: @@ -1619,7 +1619,7 @@ H5Fstart_swmr_write(hid_t file_id) H5G_name_t *obj_paths=NULL; /* Group hierarchy path */ size_t u; /* Local index variable */ hbool_t setup = FALSE; /* Boolean flag to indicate whether SWMR setting is enabled */ - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1650,7 +1650,7 @@ H5Fstart_swmr_write(hid_t file_id) HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and MDC cache image") /* Flush data buffers */ - if(H5F_flush(file, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") /* Get the # of opened named datatypes and attributes */ @@ -1704,7 +1704,9 @@ H5Fstart_swmr_write(hid_t file_id) /* Set up I/O info for operation */ fio_info.f = file; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") /* Flush and reset the accumulator */ @@ -1923,7 +1925,7 @@ H5Fset_latest_format(hid_t file_id, hbool_t latest_format) latest_flags = H5F_USE_LATEST_FLAGS(f, H5F_LATEST_ALL_FLAGS); if(latest_format != (H5F_LATEST_ALL_FLAGS == latest_flags)) { /* Call the flush routine, for this file */ - if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") /* Toggle the 'latest format' flag */ @@ -1973,7 +1975,9 @@ H5Fformat_convert(hid_t fid) /* Check for persistent freespace manager, which needs to be downgraded */ if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF && - f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF)) { + f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF && + f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF && + f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF)) { /* Check to remove free-space manager info message from superblock extension */ if(H5F_addr_defined(f->shared->sblock->ext_addr)) if(H5F_super_ext_remove_msg(f, H5AC_ind_read_dxpl_id, H5O_FSINFO_ID) < 0) @@ -1985,7 +1989,9 @@ H5Fformat_convert(hid_t fid) /* Set non-persistent freespace manager */ f->shared->fs_strategy = H5F_FILE_SPACE_STRATEGY_DEF; + f->shared->fs_persist = H5F_FREE_SPACE_PERSIST_DEF; f->shared->fs_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; + f->shared->fs_page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF; /* Indicate that the superblock should be marked dirty */ mark_dirty = TRUE; @@ -2004,3 +2010,79 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Fformat_convert() */ + +/*------------------------------------------------------------------------- + * Function: H5Freset_page_buffering_stats + * + * Purpose: Resets statistics for the page buffer layer. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5Freset_page_buffering_stats(hid_t file_id) +{ + H5F_t *file; /* File to reset stats on */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", file_id); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object(file_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier") + if(NULL == file->shared->page_buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file") + + /* Reset the statistics */ + if(H5PB_reset_stats(file->shared->page_buf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't reset stats for page buffering") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Freset_page_buffering_stats() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fget_page_buffering_stats + * + * Purpose: Retrieves statistics for the page buffer layer. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], unsigned hits[2], + unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]) +{ + H5F_t *file; /* File object for file ID */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "i*Iu*Iu*Iu*Iu*Iu", file_id, accesses, hits, misses, evictions, + bypasses); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") + if(NULL == file->shared->page_buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file") + if(NULL == accesses || NULL == hits || NULL == misses || NULL == evictions || NULL == bypasses) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL input parameters for stats") + + /* Get the statistics */ + if(H5PB_get_stats(file->shared->page_buf, accesses, hits, misses, evictions, bypasses) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve stats for page buffering") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fget_page_buffering_stats() */ + @@ -1530,7 +1530,7 @@ herr_t H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, void *buf/*out*/) { - H5P_genplist_t *dxpl; /* DXPL object */ + H5FD_io_info_t fdio_info; /* File driver I/O object */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1549,13 +1549,24 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size if(!buf) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer") - /* Get the DXPL plist object for DXPL ID */ - if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + /* Set up the file driver I/O info object */ + fdio_info.file = file; + if(H5FD_MEM_DRAW == type) { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ /* Do the real work */ /* (Note compensating for base address addition in internal routine) */ - if(H5FD_read(file, dxpl, type, addr - file->base_addr, size, buf) < 0) + if(H5FD_read(&fdio_info, type, addr - file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed") done: @@ -1584,7 +1595,7 @@ herr_t H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf) { - H5P_genplist_t *dxpl; /* DXPL object */ + H5FD_io_info_t fdio_info; /* File driver I/O object */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1602,13 +1613,24 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz if(!buf) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer") - /* Get the DXPL plist object for DXPL ID */ - if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + /* Set up the file driver I/O info object */ + fdio_info.file = file; + if(H5FD_MEM_DRAW == type) { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ /* The real work */ /* (Note compensating for base address addition in internal routine) */ - if(H5FD_write(file, dxpl, type, addr - file->base_addr, size, buf) < 0) + if(H5FD_write(&fdio_info, type, addr - file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed") done: @@ -2032,3 +2054,27 @@ H5FD_get_base_addr(const H5FD_t *file) FUNC_LEAVE_NOAPI(file->base_addr) } /* end H5FD_get_base_addr() */ + +/*-------------------------------------------------------------------------- + * Function: H5FD_set_paged_aggr + * + * Purpose: Set "paged_aggr" for the file. + * + * Return: Non-negative if succeed; negative if fails. + * + * Programmer: Vailin Choi; April 2013 + * + *-------------------------------------------------------------------------- + */ +herr_t +H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(file); + + /* Indicate whether paged aggregation for handling file space is enabled or not */ + file->paged_aggr = paged; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_set_paged_aggr() */ diff --git a/src/H5FDint.c b/src/H5FDint.c index 744c3d1..0809ac8 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -93,12 +93,9 @@ *------------------------------------------------------------------------- */ herr_t -H5FD_locate_signature(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, haddr_t *sig_addr) +H5FD_locate_signature(H5FD_io_info_t *fdio_info, haddr_t *sig_addr) { + H5FD_t *file; haddr_t addr, eoa, eof; uint8_t buf[H5F_SIGNATURE_LEN]; unsigned n, maxpow; @@ -106,6 +103,10 @@ H5P_genplist_t *dxpl, haddr_t *sig_addr) FUNC_ENTER_NOAPI_NOINIT + HDassert(fdio_info); + file = fdio_info->file; + HDassert(file); + /* Find the least N such that 2^N is larger than the file size */ eof = H5FD_get_eof(file, H5FD_MEM_SUPER); eoa = H5FD_get_eoa(file, H5FD_MEM_SUPER); @@ -124,7 +125,7 @@ H5P_genplist_t *dxpl, haddr_t *sig_addr) addr = (8 == n) ? 0 : (haddr_t)1 << n; if(H5FD_set_eoa(file, H5FD_MEM_SUPER, addr + H5F_SIGNATURE_LEN) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to set EOA value for file signature") - if(H5FD_read(file, dxpl, H5FD_MEM_SUPER, addr, (size_t)H5F_SIGNATURE_LEN, buf) < 0) + if(H5FD_read(fdio_info, H5FD_MEM_SUPER, addr, (size_t)H5F_SIGNATURE_LEN, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to read file signature") if(!HDmemcmp(buf, H5F_SIGNATURE, (size_t)H5F_SIGNATURE_LEN)) break; @@ -162,29 +163,36 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FD_read(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, +H5FD_read(H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) { + H5FD_t *file; + H5P_genplist_t *io_dxpl; haddr_t eoa = HADDR_UNDEF; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) + HDassert(fdio_info); + file = fdio_info->file; HDassert(file && file->cls); - HDassert(TRUE == H5P_class_isa(H5P_CLASS(dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->meta_dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->raw_dxpl), H5P_CLS_DATASET_XFER_g)); HDassert(buf); + /* Set up proper DXPL for I/O */ + if(H5FD_MEM_DRAW == type) + io_dxpl = fdio_info->raw_dxpl; + else + io_dxpl = fdio_info->meta_dxpl; + /* Sanity check the dxpl type against the mem type */ #ifdef H5_DEBUG_BUILD { H5FD_dxpl_type_t dxpl_type; /* Property indicating the type of the internal dxpl */ /* get the dxpl type */ - if(H5P_get(dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) + if(H5P_get(io_dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't retrieve dxpl type") /* we shouldn't be here if the dxpl is labeled with NO I/O */ @@ -219,7 +227,7 @@ H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size = %llu, eoa = %llu", (unsigned long long)(addr + file->base_addr), (unsigned long long)size, (unsigned long long)eoa) /* Dispatch to driver */ - if((file->cls->read)(file, type, H5P_PLIST_ID(dxpl), addr + file->base_addr, size, buf) < 0) + if((file->cls->read)(file, type, H5P_PLIST_ID(io_dxpl), addr + file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed") done: @@ -241,29 +249,36 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FD_write(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, +H5FD_write(const H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf) { + H5FD_t *file; + H5P_genplist_t *io_dxpl; haddr_t eoa = HADDR_UNDEF; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) + HDassert(fdio_info); + file = fdio_info->file; HDassert(file && file->cls); - HDassert(TRUE == H5P_class_isa(H5P_CLASS(dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->meta_dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->raw_dxpl), H5P_CLS_DATASET_XFER_g)); HDassert(buf); + /* Set up proper DXPL for I/O */ + if(H5FD_MEM_DRAW == type) + io_dxpl = fdio_info->raw_dxpl; + else + io_dxpl = fdio_info->meta_dxpl; + /* Sanity check the dxpl type against the mem type */ #ifdef H5_DEBUG_BUILD { H5FD_dxpl_type_t dxpl_type; /* Property indicating the type of the internal dxpl */ /* get the dxpl type */ - if(H5P_get(dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) + if(H5P_get(io_dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't retrieve dxpl type") /* we shouldn't be here if the dxpl is labeled with NO I/O */ @@ -291,7 +306,7 @@ H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, (unsigned long long)(addr+ file->base_addr), (unsigned long long)size, (unsigned long long)eoa) /* Dispatch to driver */ - if((file->cls->write)(file, type, H5P_PLIST_ID(dxpl), addr + file->base_addr, size, buf) < 0) + if((file->cls->write)(file, type, H5P_PLIST_ID(io_dxpl), addr + file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write request failed") done: diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 03228d2..f320946 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -934,13 +934,7 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsi /* Compute the address for the block to allocate */ addr = file->eoa; - /* Check if we need to align this block */ - if(size >= file->pub.threshold) { - /* Check for an already aligned block */ - if(addr % file->pub.alignment != 0) - addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment; - } /* end if */ - + /* Extend the end-of-allocated space address */ file->eoa = addr + size; /* Retain the (first) flavor of the information written to the file */ diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 7e12869..0ab7fc5 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1194,6 +1194,8 @@ H5FD_multi_query(const H5FD_t *_f, unsigned long *flags /* out */) *flags = 0; *flags |= H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */ *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ + *flags |= H5FD_FEAT_USE_ALLOC_SIZE; /* OK just pass the allocation size to the alloc callback */ + *flags |= H5FD_FEAT_PAGED_AGGR; /* OK special file space mapping for paged aggregation */ } /* end if */ return(0); @@ -1538,6 +1540,14 @@ H5FD_multi_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) mmt = file->fa.memb_map[type]; if (H5FD_MEM_DEFAULT==mmt) mmt = type; + /* XXX: NEED to work on this again */ + if(file->pub.paged_aggr) { + ALL_MEMBERS(mt) { + if(file->memb[mt]) + file->memb[mt]->paged_aggr = file->pub.paged_aggr; + } END_MEMBERS; + } + if (HADDR_UNDEF==(addr=H5FDalloc(file->memb[mmt], mmt, dxpl_id, size))) H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "member file can't alloc", HADDR_UNDEF) addr += file->fa.memb_addr[mmt]; diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 45f0187..c64ec30 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -115,6 +115,19 @@ typedef enum { #define H5FD_DXPL_TYPE_NAME "H5P_dxpl_type" #endif /* H5_DEBUG_BUILD */ +/* I/O Info for an operation */ +typedef struct H5FD_io_info_t { + H5FD_t *file; /* File driver object */ +#ifndef H5_DEBUG_BUILD + const +#endif /* H5_DEBUG_BUILD */ + H5P_genplist_t *meta_dxpl; /* Metadata DXPL object */ +#ifndef H5_DEBUG_BUILD + const +#endif /* H5_DEBUG_BUILD */ + H5P_genplist_t *raw_dxpl; /* Raw data DXPL object */ +} H5FD_io_info_t; + /*****************************/ /* Library Private Variables */ @@ -126,15 +139,10 @@ typedef enum { /******************************/ /* Forward declarations for prototype arguments */ -struct H5P_genplist_t; struct H5F_t; H5_DLL int H5FD_term_interface(void); -H5_DLL herr_t H5FD_locate_signature(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, haddr_t *sig_addr); +H5_DLL herr_t H5FD_locate_signature(H5FD_io_info_t *fdio_info, haddr_t *sig_addr); H5_DLL H5FD_class_t *H5FD_get_class(hid_t id); H5_DLL hsize_t H5FD_sb_size(H5FD_t *file); H5_DLL herr_t H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf); @@ -146,8 +154,8 @@ H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); H5_DLL herr_t H5FD_close(H5FD_t *file); H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2); -H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f, - hsize_t size, haddr_t *align_addr, hsize_t *align_size); +H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, + struct H5F_t *f, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size); H5_DLL herr_t H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f, haddr_t addr, hsize_t size); H5_DLL htri_t H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f, @@ -159,17 +167,9 @@ H5_DLL haddr_t H5FD_get_maxaddr(const H5FD_t *file); H5_DLL herr_t H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags); H5_DLL herr_t H5FD_set_feature_flags(H5FD_t *file, unsigned long feature_flags); H5_DLL herr_t H5FD_get_fs_type_map(const H5FD_t *file, H5FD_mem_t *type_map); -H5_DLL herr_t H5FD_read(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, +H5_DLL herr_t H5FD_read(H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/); -H5_DLL herr_t H5FD_write(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, +H5_DLL herr_t H5FD_write(const H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, hbool_t closing); H5_DLL herr_t H5FD_truncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); @@ -179,6 +179,7 @@ H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum); H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle); H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); +H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); /* Function prototypes for MPI based VFDs*/ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 436be10..883b28d 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -239,6 +239,19 @@ typedef enum H5F_mem_t H5FD_mem_t; * driver supports the single-writer/multiple-readers I/O pattern. */ #define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000 + /* + * Defining H5FD_FEAT_USE_ALLOC_SIZE for a VFL driver + * means that the library will just pass the allocation size to the + * the driver's allocation callback which will eventually handle alignment. + * This is specifically used for the multi/split driver. + */ +#define H5FD_FEAT_USE_ALLOC_SIZE 0x00002000 + /* + * Defining H5FD_FEAT_PAGED_AGGR for a VFL driver + * means that the driver needs special file space mapping for paged aggregation. + * This is specifically used for the multi/split driver. + */ +#define H5FD_FEAT_PAGED_AGGR 0x00004000 /* Forward declaration */ typedef struct H5FD_t H5FD_t; @@ -307,6 +320,7 @@ struct H5FD_t { /* Space allocation management fields */ hsize_t threshold; /* Threshold for alignment */ hsize_t alignment; /* Allocation alignment */ + hbool_t paged_aggr; /* Paged aggregation for file space is enabled or not */ }; /* Define enum for the source of file image callbacks */ diff --git a/src/H5FDspace.c b/src/H5FDspace.c index fcddecf..0ad3cf0 100644 --- a/src/H5FDspace.c +++ b/src/H5FDspace.c @@ -99,7 +99,7 @@ H5FL_DEFINE(H5FD_free_t); *------------------------------------------------------------------------- */ static haddr_t -H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +H5FD_extend(H5FD_t *file, H5FD_mem_t type, hsize_t size) { hsize_t orig_size = size; /* Original allocation size */ haddr_t eoa; /* Address of end-of-allocated space */ @@ -117,40 +117,18 @@ H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, hadd /* Get current end-of-allocated space address */ eoa = file->cls->get_eoa(file, type); - /* Compute extra space to allocate, if this is a new block and should be aligned */ - extra = 0; - if(new_block && file->alignment > 1 && orig_size >= file->threshold) { - hsize_t mis_align; /* Amount EOA is misaligned */ - - /* Check for EOA already aligned */ - if((mis_align = (eoa % file->alignment)) > 0) { - extra = file->alignment - mis_align; - if(frag_addr) - *frag_addr = eoa - file->base_addr; /* adjust for file's base address */ - if(frag_size) - *frag_size = extra; - } /* end if */ - } /* end if */ - - /* Add in extra allocation amount */ - size += extra; - /* Check for overflow when extending */ if(H5F_addr_overflow(eoa, size) || (eoa + size) > file->maxaddr) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed") - /* Set the [possibly aligned] address to return */ - ret_value = eoa + extra; + /* Set the [NOT aligned] address to return */ + ret_value = eoa; /* Extend the end-of-allocated space address */ eoa += size; if(file->cls->set_eoa(file, type, eoa) < 0) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed") - /* Post-condition sanity check */ - if(new_block && file->alignment && orig_size >= file->threshold) - HDassert(!(ret_value % file->alignment)); - done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_extend() */ @@ -160,6 +138,8 @@ done: * Function: H5FD_alloc_real * * Purpose: Allocate space in the file with the VFD + * Note: the handling of alignment is moved up from each driver to + * this routine. * * Return: Success: The format address of the new file memory. * Failure: The undefined address HADDR_UNDEF @@ -170,8 +150,14 @@ done: *------------------------------------------------------------------------- */ haddr_t -H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size, + haddr_t *frag_addr, hsize_t *frag_size) { + hsize_t orig_size = size; /* Original allocation size */ + haddr_t eoa; /* Address of end-of-allocated space */ + hsize_t extra; /* Extra space to allocate, to align request */ + unsigned long flags = 0; /* Driver feature flags */ + hbool_t use_alloc_size; /* Just pass alloc size to the driver */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ FUNC_ENTER_NOAPI(HADDR_UNDEF) @@ -185,16 +171,53 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); HDassert(size > 0); + /* Check for query driver and call it */ + if(file->cls->query) + (file->cls->query)(file, &flags); + + /* Check for the driver feature flag */ + use_alloc_size = flags & H5FD_FEAT_USE_ALLOC_SIZE; + + /* Get current end-of-allocated space address */ + eoa = file->cls->get_eoa(file, type); + + /* Compute extra space to allocate, if this should be aligned */ + extra = 0; + if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold) { + hsize_t mis_align; /* Amount EOA is misaligned */ + + /* Check for EOA already aligned */ + if((mis_align = (eoa % file->alignment)) > 0) { + extra = file->alignment - mis_align; + if(frag_addr) + *frag_addr = eoa - file->base_addr; /* adjust for file's base address */ + if(frag_size) + *frag_size = extra; + } /* end if */ + } /* end if */ + /* Dispatch to driver `alloc' callback or extend the end-of-address marker */ + /* For the multi/split driver: the size passed down to the alloc callback is the original size from H5FD_alloc() */ + /* For all other drivers: the size passed down to the alloc callback is the size + [possibly] alignment size */ if(file->cls->alloc) { - if((ret_value = (file->cls->alloc)(file, type, dxpl_id, size)) == HADDR_UNDEF) + ret_value = (file->cls->alloc)(file, type, dxpl_id, use_alloc_size ? size : size + extra); + if(!H5F_addr_defined(ret_value)) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver allocation request failed") } /* end if */ else { - if((ret_value = H5FD_extend(file, type, TRUE, size, frag_addr, frag_size)) == HADDR_UNDEF) + ret_value = H5FD_extend(file, type, size + extra); + if(!H5F_addr_defined(ret_value)) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver eoa update request failed") } /* end else */ + /* Set the [possibly aligned] address to return */ + if(!use_alloc_size) + ret_value += extra; + + /* Post-condition sanity check */ + if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold) + HDassert(!(ret_value % file->alignment)); + /* Convert absolute file offset to relative address */ ret_value -= file->base_addr; @@ -420,7 +443,7 @@ H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, hid_t dxpl_id, /* Check if the block is exactly at the end of the file */ if(H5F_addr_eq(blk_end, eoa)) { /* Extend the object by extending the underlying file */ - if(HADDR_UNDEF == H5FD_extend(file, type, FALSE, extra_requested, NULL, NULL)) + if(HADDR_UNDEF == H5FD_extend(file, type, extra_requested)) HGOTO_ERROR(H5E_VFL, H5E_CANTEXTEND, FAIL, "driver extend request failed") /* Mark EOA info dirty in cache, so change will get encoded */ diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 4c62bcc..13f728e 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -601,13 +601,6 @@ H5FD_stdio_alloc(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxp /* Compute the address for the block to allocate */ addr = file->eoa; - /* Check if we need to align this block */ - if(size >= file->pub.threshold) { - /* Check for an already aligned block */ - if((addr % file->pub.alignment) != 0) - addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment; - } /* end if */ - file->eoa = addr + size; return addr; @@ -135,7 +135,7 @@ HDfprintf(stderr, "%s: Creating free space manager, nclasses = %Zu\n", FUNC, ncl fspace->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0; fspace->alignment = alignment; - fspace->threshold = threshold; + fspace->align_thres = threshold; /* Check if the free space tracker is supposed to be persistant */ if(fs_addr) { @@ -227,7 +227,7 @@ HDfprintf(stderr, "%s: fspace->rc = %u\n", FUNC, fspace->rc); HGOTO_ERROR(H5E_FSPACE, H5E_CANTINC, NULL, "unable to increment ref. count on free space header") fspace->alignment = alignment; - fspace->threshold = threshold; + fspace->align_thres = threshold; /* Unlock free space header */ if(H5AC_unprotect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, fspace, H5AC__NO_FLAGS_SET) < 0) @@ -873,10 +873,7 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) HDassert(fspace); if(!H5F_addr_defined(fspace->sect_addr) && fspace->sinfo && fspace->serial_sect_count > 0) { - /* Allocate space for section info from aggregator/vfd (or temp. address space) */ - /* (The original version called H5MF_alloc(), but that may cause sect_size to change again) */ - /* (This routine is only called during file close operations, so don't allocate from temp. address space) */ - if(HADDR_UNDEF == (fspace->sect_addr = H5MF_aggr_vfd_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size))) + if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size))) HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info") fspace->alloc_sect_size = fspace->sect_size; @@ -901,169 +898,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5FS_alloc_vfd_alloc_hdr_and_section_info - * - * Purpose: This function is part of a hack to patch over a design - * flaw in the free space managers for file space allocation. - * Specifically, if a free space manager allocates space for - * its own section info, it is possible for it to - * go into an infinite loop as it: - * - * 1) computes the size of the section info - * - * 2) allocates file space for the section info - * - * 3) notices that the size of the section info - * has changed - * - * 4) deallocates the section info file space and - * returns to 1) above. - * - * Similarly, if it allocates space for its own header, it - * can go into an infinte loop as it: - * - * 1) allocates space for the header - * - * 2) notices that the free space manager is empty - * and thus should not have file space allocated - * to its header - * - * 3) frees the space allocated to the header - * - * 4) notices that the free space manager is not - * empty and thus must have file space allocated - * to it, and thus returns to 1) above. - * - * In a nutshell, the solution applied in this hack is to - * deallocate file space for the free space manager(s) that - * handle FSM header and/or section info file space allocations, - * wait until all other allocation/deallocation requests have - * been handled, and then test to see if the free space manager(s) - * in question are empty. If they are, do nothing. If they - * are not, allocate space for them at end of file bypassing the - * usual file space allocation calls, and thus avoiding the - * potential infinite loops. - * - * The purpose of this function is to allocate file space for - * the header and section info of the target free space manager - * directly from the VFD if needed. In this case the function - * also re-inserts the header and section info in the metadata - * cache with this allocation. - * - * Note that if f->shared->alignment > 1, and EOA is not a - * multiple of the alignment, it is possible that performing - * this allocation may generate a fragment of file space in - * addition to the space allocated for the section info. - * - * At present we deal with this by screaming and dying. - * Obviously, this is not acceptatable, but it should do - * for now. - * - * - * Return: Success: non-negative - * Failure: negative - * - * Programmer: John Mainzer - * 6/6/16 - * - *------------------------------------------------------------------------- - */ -herr_t -H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, - H5FS_t *fspace, haddr_t *fs_addr_ptr) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) - - /* Check arguments */ - HDassert(f); - HDassert(fspace); - HDassert(fs_addr_ptr); - - /* The section info should be unlocked */ - HDassert(fspace->sinfo_lock_count == 0); - - /* No space should be allocated */ - HDassert(*fs_addr_ptr == HADDR_UNDEF); - HDassert(fspace->addr == HADDR_UNDEF); - HDassert(fspace->sect_addr == HADDR_UNDEF); - HDassert(fspace->alloc_sect_size == 0); - - /* Check if any space will be needed */ - if(fspace->serial_sect_count > 0) { - haddr_t sect_addr; /* Address of sinfo */ - - /* The section info is floating, so fspace->sinfo should be defined */ - HDassert(fspace->sinfo); - - /* Start by allocating file space for the header */ - if(HADDR_UNDEF == (fspace->addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_HDR, - (hsize_t)H5FS_HEADER_SIZE(f), FALSE))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr") - - /* Cache the new free space header (pinned) */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space header to cache") - - - /* Now allocate file space for the section info */ - if(HADDR_UNDEF == (sect_addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_SINFO, - fspace->sect_size, FALSE))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space") - - /* Update fspace->alloc_sect_size and fspace->sect_addr to reflect - * the allocation - */ - fspace->alloc_sect_size = fspace->sect_size; - fspace->sect_addr = sect_addr; - - /* We have changed the sinfo address -- Mark free space header dirty */ - if(H5AC_mark_entry_dirty(fspace) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") - - /* Insert the new section info into the metadata cache. */ - - /* Question: Do we need to worry about this insertion causing an - * eviction from the metadata cache? Think on this. If so, add a - * flag to H5AC_insert() to force it to skip the call to make space in - * cache. - * - * On reflection, no. - * - * On a regular file close, any eviction will not change the - * the contents of the free space manger(s), as all entries - * should have correct file space allocated by the time this - * function is called. - * - * In the cache image case, the selection of entries for inclusion - * in the cache image will not take place until after this call. - * (Recall that this call is made during the metadata fsm settle - * routine, which is called during the serialization routine in - * the cache image case. Entries are not selected for inclusion - * in the image until after the cache is serialized.) - * - * JRM -- 11/4/16 - */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space sinfo to cache") - - /* Since space has been allocated for the section info and the sinfo - * has been inserted into the cache, relinquish owership (i.e. float) - * the section info. - */ - fspace->sinfo = NULL; - - /* Set the address of the free space header, on success */ - *fs_addr_ptr = fspace->addr; - } /* end if */ - -done: - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5FS_alloc_vfd_alloc_hdr_and_section_info() */ - - -/*------------------------------------------------------------------------- * Function: H5FS_free() * * Purpose: Free space for free-space manager header and section info header @@ -1076,7 +910,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) +H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id, hbool_t free_file_space) { haddr_t saved_addr; /* Previous address of item */ unsigned cache_flags; /* Flags for unprotecting cache entries */ @@ -1123,7 +957,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) /* Free space for the free-space manager section info */ if(!H5F_IS_TMP_ADDR(f, saved_addr)) { - if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0) + if(free_file_space && + H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections") } /* end if */ @@ -1165,7 +1000,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) fspace->addr = HADDR_UNDEF; /* Free space for the free-space manager header */ - if(H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0) + if(free_file_space && + H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space header") } /* end if */ @@ -1339,6 +1175,23 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FS_sinfo_dest() */ +herr_t +H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(frsp); + HDassert(tot_sect_count); + + /* Report statistics for free space */ + *tot_sect_count = frsp->serial_sect_count; + + FUNC_LEAVE_NOAPI(ret_value) +} + #ifdef H5FS_DEBUG_ASSERT /*------------------------------------------------------------------------- diff --git a/src/H5FScache.c b/src/H5FScache.c index dcc8122..ddc66fd 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -37,6 +37,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5ACprivate.h" /* Metadata cache */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -296,7 +297,7 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* # of section classes */ /* (only check if we actually have some classes) */ UINT16DECODE(image, nclasses); - if(fspace->nclasses > 0 && fspace->nclasses != nclasses) + if(fspace->nclasses > 0 && nclasses > fspace->nclasses) HGOTO_ERROR(H5E_FSPACE, H5E_CANTLOAD, NULL, "section class count mismatch") /* Shrink percent */ @@ -596,9 +597,11 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, * real file space lest the header be written to file with * a nonsense section info address. */ - HDassert(fspace->serial_sect_count > 0); - HDassert(fspace->sect_size > 0); - HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size); + if(!H5F_POINT_OF_NO_RETURN(f)) { + HDassert(fspace->serial_sect_count > 0); + HDassert(fspace->sect_size > 0); + HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size); + } /* end if */ if(H5F_IS_TMP_ADDR(f, fspace->sect_addr)) { unsigned sect_status = 0; @@ -702,10 +705,12 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len, * The following asserts are a cursory check on this. */ HDassert((! H5F_addr_defined(fspace->sect_addr)) || (! H5F_IS_TMP_ADDR(f, fspace->sect_addr))); - HDassert((! H5F_addr_defined(fspace->sect_addr)) || - ((fspace->serial_sect_count > 0) && - (fspace->sect_size > 0) && - (fspace->alloc_sect_size == (size_t)fspace->sect_size))); + + if(!H5F_POINT_OF_NO_RETURN(f)) + HDassert((! H5F_addr_defined(fspace->sect_addr)) || + ((fspace->serial_sect_count > 0) && + (fspace->sect_size > 0) && + (fspace->alloc_sect_size == (size_t)fspace->sect_size))); /* Magic number */ HDmemcpy(image, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC); @@ -1061,7 +1066,7 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, /* Insert section in free space manager, unless requested not to */ if(!(des_flags & H5FS_DESERIALIZE_NO_ADD)) - if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, NULL) < 0) + if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, udata) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, NULL, "can't add section to free space manager") } /* end for */ } while(image < (((const uint8_t *)_image + old_sect_size) - H5FS_SIZEOF_CHKSUM)); @@ -1177,8 +1182,9 @@ H5FS__cache_sinfo_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, HDassert(new_len); HDassert(flags); - /* we shouldn't be called if the section info is empty */ - HDassert(fspace->serial_sect_count > 0); + /* we shouldn't be called if the section info is empty, unless we hit the point of no return. */ + if(!H5F_POINT_OF_NO_RETURN(f)) + HDassert(fspace->serial_sect_count > 0); sinfo_addr = addr; /* this will change if we relocate the section data */ diff --git a/src/H5FSpkg.h b/src/H5FSpkg.h index 4411236..ab97955 100644 --- a/src/H5FSpkg.h +++ b/src/H5FSpkg.h @@ -187,8 +187,9 @@ struct H5FS_t { /* must be either H5C__NO_FLAGS_SET (i.e r/w) */ /* or H5AC__READ_ONLY_FLAG (i.e. r/o). */ size_t max_cls_serial_size; /* Max. additional size of serialized form of section */ - hsize_t threshold; /* Threshold for alignment */ hsize_t alignment; /* Alignment */ + hsize_t align_thres; /* Threshold for alignment */ + /* Memory data structures (not stored directly) */ H5FS_section_class_t *sect_cls; /* Array of section classes for this free list */ diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h index 1816a6e..74f9dcb 100644 --- a/src/H5FSprivate.h +++ b/src/H5FSprivate.h @@ -67,6 +67,12 @@ * managed sections is in flux) */ +#define H5FS_PAGE_END_NO_ADD 0x08 /* For "small" page fs: + * Don't add section to free space: + * when the section is at page end and + * when the section size is <= "small" + */ + /* Flags for deserialize callback */ #define H5FS_DESERIALIZE_NO_ADD 0x01 /* Don't add section to free space * manager after it's deserialized @@ -98,11 +104,11 @@ typedef struct H5FS_section_class_t { herr_t (*term_cls)(struct H5FS_section_class_t *); /* Routine to terminate class-specific settings */ /* Object methods */ - herr_t (*add)(H5FS_section_info_t *, unsigned *, void *); /* Routine called when section is about to be added to manager */ + herr_t (*add)(H5FS_section_info_t **, unsigned *, void *); /* Routine called when section is about to be added to manager */ herr_t (*serialize)(const struct H5FS_section_class_t *, const H5FS_section_info_t *, uint8_t *); /* Routine to serialize a "live" section into a buffer */ H5FS_section_info_t *(*deserialize)(const struct H5FS_section_class_t *, hid_t dxpl_id, const uint8_t *, haddr_t, hsize_t, unsigned *); /* Routine to deserialize a buffer into a "live" section */ htri_t (*can_merge)(const H5FS_section_info_t *, const H5FS_section_info_t *, void *); /* Routine to determine if two nodes are mergable */ - herr_t (*merge)(H5FS_section_info_t *, H5FS_section_info_t *, void *); /* Routine to merge two nodes */ + herr_t (*merge)(H5FS_section_info_t **, H5FS_section_info_t *, void *); /* Routine to merge two nodes */ htri_t (*can_shrink)(const H5FS_section_info_t *, void *); /* Routine to determine if node can shrink container */ herr_t (*shrink)(H5FS_section_info_t **, void *); /* Routine to shrink container */ herr_t (*free)(H5FS_section_info_t *); /* Routine to free node */ @@ -182,9 +188,8 @@ H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr); H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace); H5_DLL herr_t H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id); H5_DLL herr_t H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id); -H5_DLL herr_t H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, - H5FS_t *fspace, haddr_t *fs_addr_ptr); -H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id); +H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id, + hbool_t free_file_space); /* Free space section routines */ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, @@ -192,7 +197,7 @@ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5_DLL htri_t H5FS_sect_try_merge(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *sect, unsigned flags, void *op_data); H5_DLL htri_t H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, - haddr_t addr, hsize_t size, hsize_t extra_requested); + haddr_t addr, hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data); H5_DLL herr_t H5FS_sect_remove(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *node); H5_DLL htri_t H5FS_sect_find(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, @@ -202,11 +207,18 @@ H5_DLL herr_t H5FS_sect_stats(const H5FS_t *fspace, hsize_t *tot_space, hsize_t *nsects); H5_DLL herr_t H5FS_sect_change_class(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *sect, uint16_t new_class); -H5_DLL htri_t H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data); -H5_DLL herr_t H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size); +H5_DLL htri_t H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, + void *op_data); /* Statistics routine */ H5_DLL herr_t H5FS_stat_info(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *stats); +H5_DLL herr_t H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count); + +/* free space manager settling routines */ +H5_DLL herr_t H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, + hid_t dxpl_id, + H5FS_t *fspace, + haddr_t *fs_addr_ptr); /* Debugging routines for dumping file structures */ H5_DLL herr_t H5FS_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, diff --git a/src/H5FSsection.c b/src/H5FSsection.c index efe0031..49ff003 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -25,6 +25,8 @@ /* Module Setup */ /****************/ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ + #include "H5FSmodule.h" /* This source code file is part of the H5FS module */ @@ -33,6 +35,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -1210,11 +1213,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data) HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures") /* Merge the two sections together */ - if((*tmp_sect_cls->merge)(tmp_sect, *sect, op_data) < 0) + if((*tmp_sect_cls->merge)(&tmp_sect, *sect, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections") /* Retarget section pointer to 'less than' node that was merged into */ *sect = tmp_sect; + if(*sect == NULL) + HGOTO_DONE(ret_value); /* Indicate successful merge occurred */ modified = TRUE; @@ -1254,9 +1259,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data) HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures") /* Merge the two sections together */ - if((*sect_cls->merge)(*sect, tmp_sect, op_data) < 0) + if((*sect_cls->merge)(sect, tmp_sect, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections") + /* It's possible that the merge caused the section to be deleted (particularly in the paged allocation case) */ + if(*sect == NULL) + HGOTO_DONE(ret_value); + /* Indicate successful merge occurred */ modified = TRUE; } /* end if */ @@ -1383,7 +1392,7 @@ HDfprintf(stderr, "%s: *sect = {%a, %Hu, %u, %s}\n", FUNC, sect->addr, sect->siz /* Call "add" section class callback, if there is one */ cls = &fspace->sect_cls[sect->type]; if(cls->add) { - if((*cls->add)(sect, &flags, op_data) < 0) + if((*cls->add)(§, &flags, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed") } /* end if */ @@ -1411,7 +1420,7 @@ HDfprintf(stderr, "%s: fspace->tot_space = %Hu\n", FUNC, fspace->tot_space); #endif /* H5FS_SINFO_DEBUG */ /* Mark free space sections as changed */ /* (if adding sections while deserializing sections, don't set the flag) */ - if(!(flags & H5FS_ADD_DESERIALIZING)) + if(!(flags & (H5FS_ADD_DESERIALIZING | H5FS_PAGE_END_NO_ADD))) sinfo_modified = TRUE; done: @@ -1444,7 +1453,7 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); */ htri_t H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, haddr_t addr, - hsize_t size, hsize_t extra_requested) + hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data) { hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */ hbool_t sinfo_modified = FALSE; /* Whether the section info was modified */ @@ -1528,10 +1537,16 @@ if(_section_) /* Adjust section by amount requested */ sect->addr += extra_requested; sect->size -= extra_requested; - - /* Re-add adjusted section to free sections data structures */ - if(H5FS_sect_link(fspace, sect, 0) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list") + if(cls->add) + if((*cls->add)(§, &flags, op_data) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed") + + /* Re-adding the section could cause it to disappear (particularly when paging) */ + if(sect) { + /* Re-add adjusted section to free sections data structures */ + if(H5FS_sect_link(fspace, sect, 0) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list") + } /* end if */ } /* end if */ else { /* Sanity check */ @@ -1667,7 +1682,7 @@ HDfprintf(stderr, "%s: fspace->sinfo->nbins = %u\n", FUNC, fspace->sinfo->nbins) HDfprintf(stderr, "%s: bin = %u\n", FUNC, bin); #endif /* QAK */ alignment = fspace->alignment; - if(!((alignment > 1) && (request >= fspace->threshold))) + if(!((alignment > 1) && (request >= fspace->align_thres))) alignment = 0; /* no alignment */ do { @@ -2349,7 +2364,7 @@ HDfprintf(stderr, "%s: sect->size = %Hu, sect->addr = %a, sect->type = %u\n", "H *------------------------------------------------------------------------- */ htri_t -H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data) +H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, void *op_data) { hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */ hbool_t section_removed = FALSE; /* Whether a section was removed */ @@ -2406,42 +2421,264 @@ done: /*------------------------------------------------------------------------- - * Function: H5FS_sect_query_last_sect - * - * Purpose: Retrieve info about the last section on the merge list - * - * Return: Success: non-negative - * Failure: negative + * Function: H5FS_vfd_alloc_hdr_and_section_info_if_needed + * + * Purpose: This function is part of a hack to patch over a design + * flaw in the free space managers for file space allocation. + * Specifically, if a free space manager allocates space for + * its own section info, it is possible for it to + * go into an infinite loop as it: + * + * 1) computes the size of the section info + * + * 2) allocates file space for the section info + * + * 3) notices that the size of the section info + * has changed + * + * 4) deallocates the section info file space and + * returns to 1) above. + * + * Similarly, if it allocates space for its own header, it + * can go into an infinte loop as it: + * + * 1) allocates space for the header + * + * 2) notices that the free space manager is empty + * and thus should not have file space allocated + * to its header + * + * 3) frees the space allocated to the header + * + * 4) notices that the free space manager is not + * empty and thus must have file space allocated + * to it, and thus returns to 1) above. + * + * In a nutshell, the solution applied in this hack is to + * deallocate file space for the free space manager(s) that + * handle FSM header and/or section info file space allocations, + * wait until all other allocation/deallocation requests have + * been handled, and then test to see if the free space manager(s) + * in question are empty. If they are, do nothing. If they + * are not, allocate space for them at end of file bypassing the + * usual file space allocation calls, and thus avoiding the + * potential infinite loops. + * + * The purpose of this function is to allocate file space for + * the header and section info of the target free space manager + * directly from the VFD if needed. In this case the function + * also re-inserts the header and section info in the metadata + * cache with this allocation. + * + * When paged allocation is not enabled, allocation of space + * for the free space manager header and section info is + * straight forward -- we simply allocate the space directly + * from file driver. + * + * Note that if f->shared->alignment > 1, and EOA is not a + * multiple of the alignment, it is possible that performing + * these allocations may generate a fragment of file space in + * addition to the space allocated for the section info. This + * excess space is dropped on the floor. As shall be seen, + * it will usually be reclaimed later. + * + * When page allocation is enabled, things are more difficult, + * as there is the possibility that page buffering will be + * enabled when the free space managers are read. To allow + * for this, we must ensure that space allocated for the + * free space manager header and section info is either larger + * than a page, or resides completely withing a page. + * + * Do this by allocating space for the free space header and + * section info starting at page boundaries, and extending + * allocation to the next page boundary. This of course wastes + * space, but see below. + * + * On the first free space allocation / deallocation after the + * next file open, we will read the self referential free space + * managers, float them and reduce the EOA to its value prior + * to allocation of file space for the self referential free + * space managers on the preceeding file close. This EOA value + * is stored in the free space manager superblock extension + * message. + * + * Return: Success: non-negative + * Failure: negative * - * Programmer: Vailin Choi + * Programmer: John Mainzer + * 6/6/16 * *------------------------------------------------------------------------- */ herr_t -H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size) +H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, hid_t dxpl_id, + H5FS_t *fspace, haddr_t *fs_addr_ptr) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + hsize_t hdr_alloc_size; + hsize_t sinfo_alloc_size; + haddr_t sect_addr = HADDR_UNDEF; /* address of sinfo */ + haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ + hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */ + haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT /* Check arguments. */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); HDassert(fspace); + HDassert(fs_addr_ptr); - if(fspace->sinfo && fspace->sinfo->merge_list) { - H5SL_node_t *last_node; /* Last node in merge list */ + /* the section info should be unlocked */ + HDassert(fspace->sinfo_lock_count == 0); - /* Check for last node in the merge list */ - if(NULL != (last_node = H5SL_last(fspace->sinfo->merge_list))) { - H5FS_section_info_t *tmp_sect; /* Temporary free space section */ + /* no space should be allocated */ + HDassert(*fs_addr_ptr == HADDR_UNDEF); + HDassert(fspace->addr == HADDR_UNDEF); + HDassert(fspace->sect_addr == HADDR_UNDEF); + HDassert(fspace->alloc_sect_size == 0); - /* Get the pointer to the last section, from the last node */ - tmp_sect = (H5FS_section_info_t *)H5SL_item(last_node); - HDassert(tmp_sect); - if(sect_addr) - *sect_addr = tmp_sect->addr; - if(sect_size) - *sect_size = tmp_sect->size; - } /* end if */ + /* persistant free space managers must be enabled */ + HDassert(f->shared->fs_persist); + + /* At present, all free space strategies enable the free space managers. + * This will probably change -- at which point this assertion should + * be revisited. + */ + /* Updated: Only the following two strategies enable the free-space managers */ + HDassert((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) || + (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE)); + + if(fspace->serial_sect_count > 0) { + /* the section info is floating, so space->sinfo should be defined */ + HDassert(fspace->sinfo); + + /* start by allocating file space for the header */ + + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") + + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space") + + hdr_alloc_size = H5FS_HEADER_SIZE(f); + + /* if page allocation is enabled, extend the hdr_alloc_size to the + * next page boundary. + */ + if(H5F_PAGED_AGGR(f)) { + HDassert(0 == (eoa % f->shared->fs_page_size)); + + hdr_alloc_size = ((hdr_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + + HDassert(hdr_alloc_size >= H5FS_HEADER_SIZE(f)); + HDassert((hdr_alloc_size % f->shared->fs_page_size) == 0); + } /* end if */ + + /* allocate space for the hdr */ + if(HADDR_UNDEF == (fspace->addr = H5FD_alloc(f->shared->lf, dxpl_id, + H5FD_MEM_FSPACE_HDR, f, + hdr_alloc_size, + &eoa_frag_addr, + &eoa_frag_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr") + + /* if the file alignement is 1, there should be no + * eoa fragment. Otherwise, drop any fragment on the floor. + */ + HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + + /* Cache the new free space header (pinned) */ + if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache") + + *fs_addr_ptr = fspace->addr; + + /* now allocate file space for the section info */ + + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa") + + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space") + + sinfo_alloc_size = fspace->sect_size; + + /* if paged allocation is enabled, extend the sinfo_alloc_size to the + * next page boundary. + */ + if(H5F_PAGED_AGGR(f)) { + HDassert(0 == (eoa % f->shared->fs_page_size)); + + sinfo_alloc_size = ((sinfo_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + + HDassert(sinfo_alloc_size >= fspace->sect_size); + HDassert((sinfo_alloc_size % f->shared->fs_page_size) == 0); + } /* end if */ + + /* allocate space for the section info */ + if(HADDR_UNDEF == (sect_addr = H5FD_alloc(f->shared->lf, dxpl_id, + H5FD_MEM_FSPACE_SINFO, f, + sinfo_alloc_size, + &eoa_frag_addr, + &eoa_frag_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space") + + /* if the file alignement is 1, there should be no + * eoa fragment. Otherwise, drop the fragment on the floor. + */ + HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + + /* update fspace->alloc_sect_size and fspace->sect_addr to reflect + * the allocation + */ + fspace->alloc_sect_size = fspace->sect_size; + fspace->sect_addr = sect_addr; + + /* insert the new section info into the metadata cache. */ + + /* Question: Do we need to worry about this insertion causing an + * eviction from the metadata cache? Think on this. If so, add a + * flag to H5AC_insert() to force it to skip the call to make space in + * cache. + * + * On reflection, no. + * + * On a regular file close, any eviction will not change the + * the contents of the free space manger(s), as all entries + * should have correct file space allocated by the time this + * function is called. + * + * In the cache image case, the selection of entries for inclusion + * in the cache image will not take place until after this call. + * (Recall that this call is made during the metadata fsm settle + * routine, which is called during the serialization routine in + * the cache image case. Entries are not selected for inclusion + * in the image until after the cache is serialized.) + * + * JRM -- 11/4/16 + */ + if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache") + + /* We have changed the sinfo address -- Mark free space header dirty */ + if(H5AC_mark_entry_dirty(fspace) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") + + /* since space has been allocated for the section info and the sinfo + * has been inserted into the cache, relinquish owership (i.e. float) + * the section info. + */ + fspace->sinfo = NULL; } /* end if */ - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FS_sect_query_last_sect() */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FS_vfd_alloc_hdr_and_section_info_if_needed() */ diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 48f9bdd..7f3bb39 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -112,18 +112,25 @@ H5FL_BLK_DEFINE_STATIC(meta_accum); *------------------------------------------------------------------------- */ herr_t -H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr, +H5F__accum_read(const H5F_io_info2_t *fio_info, H5FD_mem_t map_type, haddr_t addr, size_t size, void *buf/*out*/) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); HDassert(buf); + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Check if this information is in the metadata accumulator */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ @@ -178,7 +185,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr accum->dirty_off += amount_before; /* Dispatch to driver */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, amount_before, accum->buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, amount_before, accum->buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end if */ else @@ -192,7 +199,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr H5_CHECKED_ASSIGN(amount_after, size_t, ((addr + size) - (accum->loc + accum->size)), hsize_t); /* Dispatch to driver */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, (accum->loc + accum->size), amount_after, (accum->buf + accum->size + amount_before)) < 0) + if(H5FD_read(&fdio_info, map_type, (accum->loc + accum->size), amount_after, (accum->buf + accum->size + amount_before)) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end if */ @@ -206,13 +213,13 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr /* Current read doesn't overlap with metadata accumulator, read it from file */ else { /* Dispatch to driver */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end else */ } /* end if */ else { /* Read the data */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") /* Check for overlap w/dirty accumulator */ @@ -255,7 +262,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr } /* end if */ else { /* Read the data */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end else */ @@ -278,7 +285,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, +H5F__accum_adjust(H5F_meta_accum_t *accum, const H5FD_io_info_t *fdio_info, H5F_accum_adjust_t adjust, size_t size) { herr_t ret_value = SUCCEED; /* Return value */ @@ -286,7 +293,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, FUNC_ENTER_STATIC HDassert(accum); - HDassert(fio_info); + HDassert(fdio_info); HDassert(H5F_ACCUM_APPEND == adjust || H5F_ACCUM_PREPEND == adjust); HDassert(size > 0); HDassert(size <= H5F_ACCUM_MAX_SIZE); @@ -344,7 +351,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, /* Check if the dirty region overlaps the region to eliminate from the accumulator */ if((accum->size - shrink_size) < (accum->dirty_off + accum->dirty_len)) { /* Write out the dirty region from the metadata accumulator, with dispatch to driver */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) + if(H5FD_write(fdio_info, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "file write failed") /* Reset accumulator dirty flag */ @@ -355,7 +362,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, /* Check if the dirty region overlaps the region to eliminate from the accumulator */ if(shrink_size > accum->dirty_off) { /* Write out the dirty region from the metadata accumulator, with dispatch to driver */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) + if(H5FD_write(fdio_info, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "file write failed") /* Reset accumulator dirty flag */ @@ -417,9 +424,10 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr, +H5F__accum_write(const H5F_io_info2_t *fio_info, H5FD_mem_t map_type, haddr_t addr, size_t size, const void *buf) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -427,9 +435,15 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add HDassert(fio_info); HDassert(fio_info->f); HDassert(H5F_INTENT(fio_info->f) & H5F_ACC_RDWR); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); HDassert(buf); + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Check for accumulating metadata */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ @@ -446,7 +460,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add /* Check if the new metadata adjoins the beginning of the current accumulator */ if((addr + size) == accum->loc) { /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_PREPEND, size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_PREPEND, size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Move the existing metadata to the proper location */ @@ -471,7 +485,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add /* Check if the new metadata adjoins the end of the current accumulator */ else if(addr == (accum->loc + accum->size)) { /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_APPEND, size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_APPEND, size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Copy the new metadata to the end */ @@ -531,7 +545,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add H5_CHECKED_ASSIGN(add_size, size_t, (accum->loc - addr), hsize_t); /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_PREPEND, add_size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_PREPEND, add_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Calculate the proper offset of the existing metadata */ @@ -571,7 +585,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add H5_CHECKED_ASSIGN(add_size, size_t, (addr + size) - (accum->loc + accum->size), hsize_t); /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_APPEND, add_size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_APPEND, add_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Compute offset of dirty region (after adjusting accumulator) */ @@ -637,7 +651,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add else { /* Write out the existing metadata accumulator, with dispatch to driver */ if(accum->dirty) { - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, accum->loc + accum->dirty_off, accum->dirty_len, accum->buf + accum->dirty_off) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, accum->loc + accum->dirty_off, accum->dirty_len, accum->buf + accum->dirty_off) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Reset accumulator dirty flag */ @@ -733,7 +747,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") /* Write the data */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_write(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Check for overlap w/accumulator */ @@ -818,7 +832,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add } /* end if */ else { /* Write the data */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_write(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end else */ @@ -842,10 +856,11 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr, +H5F__accum_free(const H5F_io_info2_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr, hsize_t size) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -853,11 +868,17 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h /* check arguments */ HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); /* Set up alias for file's metadata accumulator info */ accum = &fio_info->f->shared->accum; + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Adjust the metadata accumulator to remove the freed block, if it overlaps */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && H5F_addr_overlap(addr, size, accum->loc, accum->size)) { @@ -930,7 +951,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h /* Check if block to free is entirely before dirty region */ if(H5F_addr_le(tail_addr, dirty_start)) { /* Write out the entire dirty region of the accumulator */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start, accum->dirty_len, accum->buf + accum->dirty_off) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start, accum->dirty_len, accum->buf + accum->dirty_off) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ /* Block to free overlaps with some/all of dirty region */ @@ -945,7 +966,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h HDassert(write_size > 0); /* Write out the unfreed dirty region of the accumulator */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ @@ -965,7 +986,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h HDassert(write_size > 0); /* Write out the unfreed end of the dirty region of the accumulator */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ @@ -1006,7 +1027,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_flush(const H5F_io_info_t *fio_info) +H5F__accum_flush(const H5F_io_info2_t *fio_info) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1014,12 +1035,20 @@ H5F__accum_flush(const H5F_io_info_t *fio_info) HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); /* Check if we need to flush out the metadata accumulator */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && fio_info->f->shared->accum.dirty) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Flush the metadata contents */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, fio_info->f->shared->accum.loc + fio_info->f->shared->accum.dirty_off, fio_info->f->shared->accum.dirty_len, fio_info->f->shared->accum.buf + fio_info->f->shared->accum.dirty_off) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, fio_info->f->shared->accum.loc + fio_info->f->shared->accum.dirty_off, fio_info->f->shared->accum.dirty_len, fio_info->f->shared->accum.buf + fio_info->f->shared->accum.dirty_off) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Reset the dirty flag */ @@ -1045,7 +1074,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush) +H5F__accum_reset(const H5F_io_info2_t *fio_info, hbool_t flush) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1053,7 +1082,6 @@ H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush) HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); /* Flush any dirty data in accumulator, if requested */ if(flush) diff --git a/src/H5Fint.c b/src/H5Fint.c index d122357..794be50 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -76,8 +76,8 @@ typedef struct H5F_olist_t { static int H5F_get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F_build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name);/* Declare a free list to manage the H5F_t struct */ -static herr_t H5F__flush_phase1(H5F_t *f, hid_t dxpl_id); -static herr_t H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing); +static herr_t H5F__flush_phase1(H5F_t *f, hid_t meta_dxpl_id); +static herr_t H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing); /*********************/ @@ -182,6 +182,14 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) efc_size = H5F_efc_max_nfiles(f->shared->efc); if(H5P_set(new_plist, H5F_ACS_EFC_SIZE_NAME, &efc_size) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set elink file cache size") + if(f->shared->page_buf != NULL) { + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &(f->shared->page_buf->max_size)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set page buffer size") + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &(f->shared->page_buf->min_meta_perc)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum metadata fraction of page buffer") + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &(f->shared->page_buf->min_raw_perc)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum raw data fraction of page buffer") + } /* end if */ #ifdef H5_HAVE_PARALLEL if(H5P_set(new_plist, H5_COLL_MD_READ_FLAG_NAME, &(f->coll_md_read)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set collective metadata read flag") @@ -495,7 +503,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_is_hdf5 + * Function: H5F__is_hdf5 * * Purpose: Check the file signature to detect an HDF5 file. * @@ -509,17 +517,14 @@ done: * * Programmer: Unknown * - * Modifications: - * Robb Matzke, 1999-08-02 - * Rewritten to use the virtual file layer. *------------------------------------------------------------------------- */ htri_t -H5F_is_hdf5(const char *name, hid_t dxpl_id) +H5F__is_hdf5(const char *name, hid_t meta_dxpl_id, hid_t raw_dxpl_id) { H5FD_t *file = NULL; /* Low-level file struct */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ haddr_t sig_addr; /* Addess of hdf5 file signature */ - H5P_genplist_t *xfer_plist= NULL; /* Dataset transfer property list object */ htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -528,12 +533,15 @@ H5F_is_hdf5(const char *name, hid_t dxpl_id) if(NULL == (file = H5FD_open(name, H5F_ACC_RDONLY, H5P_FILE_ACCESS_DEFAULT, HADDR_UNDEF))) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to open file") - /* Get the property list object */ - if(NULL == (xfer_plist = (H5P_genplist_t *)H5I_object(dxpl_id))) + /* Set up the file driver info */ + fdio_info.file = file; + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object") /* The file is an hdf5 file if the hdf5 file signature can be found */ - if(H5FD_locate_signature(file, xfer_plist, &sig_addr) < 0) + if(H5FD_locate_signature(&fdio_info, &sig_addr) < 0) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") ret_value = (HADDR_UNDEF != sig_addr); @@ -544,7 +552,7 @@ done: HDONE_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close file") FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_is_hdf5() */ +} /* end H5F__is_hdf5() */ /*------------------------------------------------------------------------- @@ -597,11 +605,26 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t f->shared->flags = flags; f->shared->sohm_addr = HADDR_UNDEF; f->shared->sohm_vers = HDF5_SHAREDHEADER_VERSION; - for(u = 0; u < NELMTS(f->shared->fs_addr); u++) - f->shared->fs_addr[u] = HADDR_UNDEF; f->shared->accum.loc = HADDR_UNDEF; f->shared->lf = lf; + /* Initialization for handling file space */ + for(u = 0; u < NELMTS(f->shared->fs_addr); u++) { + f->shared->fs_state[u] = H5F_FS_STATE_CLOSED; + f->shared->fs_addr[u] = HADDR_UNDEF; + f->shared->fs_man[u] = NULL; + } /* end for */ + f->shared->first_alloc_dealloc = FALSE; + f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + f->shared->eoa_post_fsm_fsalloc = HADDR_UNDEF; + f->shared->eoa_post_mdci_fsalloc = HADDR_UNDEF; + + /* Initialization for handling file space (for paged aggregation) */ + f->shared->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; + + /* intialize point of no return */ + f->shared->point_of_no_return = FALSE; + /* * Copy the file creation and file access property lists into the * new file handle. We do this early because some values might need @@ -621,8 +644,19 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t HDassert(f->shared->sohm_nindexes < 255); if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &f->shared->fs_strategy) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space strategy") + if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &f->shared->fs_persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space persisting status") if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &f->shared->fs_threshold) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get free-space section threshold") + if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &f->shared->fs_page_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space page size") + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + + /* Temporary for multi/split drivers: fail file creation + when persisting free-space or using paged aggregation strategy */ + if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE || f->shared->fs_persist) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't open with this strategy or persistent fs") /* Get the FAPL values to cache */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) @@ -796,7 +830,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_dest + * Function: H5F__dest * * Purpose: Destroys a file structure. This function flushes the cache * but doesn't do any other cleanup other than freeing memory @@ -812,11 +846,11 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) +H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_PACKAGE /* Sanity check */ HDassert(f); @@ -824,14 +858,14 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) if(1 == f->shared->nrefs) { int actype; /* metadata cache type (enum value) */ - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ /* Flush at this point since the file will be closed (phase 1). * Only try to flush the file if it was opened with write access, and if * the caller requested a flush. */ if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush) - if(H5F__flush_phase1(f, dxpl_id) < 0) + if(H5F__flush_phase1(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cached data (phase 1)") @@ -839,7 +873,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * This allows the cache to set up for creating a metadata cache * image if this has been requested. */ - if(H5AC_prep_for_file_close(f, dxpl_id) < 0) + if(H5AC_prep_for_file_close(f, meta_dxpl_id) < 0) /* Push error, but keep going */ HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "metadata cache prep for close failed") @@ -848,7 +882,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * the caller requested a flush. */ if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush) - if(H5F__flush_phase2(f, dxpl_id, TRUE) < 0) + if(H5F__flush_phase2(f, meta_dxpl_id, raw_dxpl_id, TRUE) < 0) /* Push error, but keep going */ HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cached data (phase 2)") @@ -895,7 +929,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * -- JRM */ if(H5F_ACC_RDWR & H5F_INTENT(f)) { - if(H5MF_close(f, dxpl_id) < 0) + if(H5MF_close(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file free space info") @@ -913,7 +947,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_SWMR_WRITE_ACCESS); /* Mark EOA info dirty in cache, so change will get encoded */ - if(H5F_eoa_dirty(f, dxpl_id) < 0) + if(H5F_eoa_dirty(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") @@ -924,12 +958,12 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * At most, this should change the superblock or the * superblock extension messages. */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + if(H5MF_free_aggrs(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space") /* Truncate the file to the current allocated size */ - if(H5FD_truncate(f->shared->lf, dxpl_id, TRUE) < 0) + if(H5FD_truncate(f->shared->lf, meta_dxpl_id, TRUE) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed") @@ -968,10 +1002,15 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") /* Shutdown the metadata cache */ - if(H5AC_dest(f, dxpl_id)) + if(H5AC_dest(f, meta_dxpl_id)) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") + /* Shutdown the page buffer cache */ + if(H5PB_dest(f) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing page buffer cache") + /* Clean up the metadata cache log location string */ if(f->shared->mdc_log_location) f->shared->mdc_log_location = (char *)H5MM_xfree(f->shared->mdc_log_location); @@ -990,7 +1029,9 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) /* Set up I/O info for operation */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") /* Destroy other components of the file */ @@ -1130,7 +1171,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) */ H5F_t * H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, - hid_t dxpl_id) + hid_t meta_dxpl_id) { H5F_t *file = NULL; /*the success return value */ H5F_file_t *shared = NULL; /*shared part of `file' */ @@ -1139,6 +1180,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_class_t *drvr; /*file driver class info */ H5P_genplist_t *a_plist; /*file access property list */ H5F_close_degree_t fc_degree; /*file close degree */ + hid_t raw_dxpl_id = H5AC_rawdata_dxpl_id; /* Raw data dxpl used by library */ + size_t page_buf_size; + unsigned page_buf_min_meta_perc; + unsigned page_buf_min_raw_perc; hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ @@ -1289,6 +1334,25 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + /* Check if page buffering is enabled */ + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size") + if(page_buf_size) { +#ifdef H5_HAVE_PARALLEL + /* Collective metadata writes are not supported with page buffering */ + if(file->coll_md_write) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "collective metadata writes are not supported with page buffering") + + /* Temporary: fail file create when page buffering feature is enabled for parallel */ + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "page buffering is disabled for parallel") +#endif /* H5_HAVE_PARALLEL */ + /* Query for other page buffer cache properties */ + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &page_buf_min_meta_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum metadata fraction of page buffer") + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum raw data fraction of page buffer") + } /* end if */ + /* * Read or write the file superblock, depending on whether the file is * empty or not. @@ -1299,26 +1363,35 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, * to create & write the superblock. */ + /* Create the page buffer before initializing the superblock */ + if(page_buf_size) + if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") + /* Initialize information about the superblock and allocate space for it */ /* (Writes superblock extension messages, if there are any) */ - if(H5F__super_init(file, dxpl_id) < 0) + if(H5F__super_init(file, meta_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to allocate file superblock") /* Create and open the root group */ /* (This must be after the space for the superblock is allocated in * the file, since the superblock must be at offset 0) */ - if(H5G_mkroot(file, dxpl_id, TRUE) < 0) + if(H5G_mkroot(file, meta_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create/open root group") } /* end if */ else if (1 == shared->nrefs) { - /* Read the superblock if it hasn't been read before. */ - if(H5F__super_read(file, dxpl_id, TRUE) < 0) + if(H5F__super_read(file, meta_dxpl_id, raw_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") + /* Create the page buffer before initializing the superblock */ + if(page_buf_size) + if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") + /* Open the root group */ - if(H5G_mkroot(file, dxpl_id, FALSE) < 0) + if(H5G_mkroot(file, meta_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group") } /* end if */ @@ -1360,12 +1433,11 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, */ if(H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value") - - if(shared->nrefs == 1) { + if(shared->nrefs == 1) shared->evict_on_close = evict_on_close; - } else if(shared->nrefs > 1) { + else if(shared->nrefs > 1) { if(shared->evict_on_close != evict_on_close) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file evict-on-close value doesn't match") + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "file evict-on-close value doesn't match") } /* end if */ /* Formulate the absolute path for later search of target file for external links */ @@ -1393,7 +1465,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, /* Flush the superblock */ if(H5F_super_dirty(file) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, NULL, "unable to mark superblock as dirty") - if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, H5AC_ind_read_dxpl_id) < 0) + if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, meta_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock") /* Remove the file lock for SWMR_WRITE */ @@ -1405,7 +1477,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, else { /* H5F_ACC_RDONLY: check consistency of status_flags */ /* Skip check of status_flags for file with < superblock version 3 */ if(file->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) { - if(H5F_INTENT(file) & H5F_ACC_SWMR_READ) { if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS && !(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) @@ -1413,12 +1484,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, (!(file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) && file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is not already open for SWMR writing") - } /* end if */ else if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) || (file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for write (may use <h5clear file> to clear file consistency flags)") - } /* version 3 superblock */ } /* end else */ } /* end if set_flag */ @@ -1428,7 +1497,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, done: if((NULL == ret_value) && file) - if(H5F_dest(file, dxpl_id, FALSE) < 0) + if(H5F__dest(file, meta_dxpl_id, raw_dxpl_id, FALSE) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file") FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_open() */ @@ -1448,7 +1517,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) +H5F__flush_phase1(H5F_t *f, hid_t meta_dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1458,7 +1527,7 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) HDassert(f); /* Flush any cached dataset storage raw data */ - if(H5D_flush(f, dxpl_id) < 0) + if(H5D_flush(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush dataset cache") @@ -1468,7 +1537,7 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) /* (needs to happen before cache flush, with superblock write, since the * 'eoa' value is written in superblock -QAK) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + if(H5MF_free_aggrs(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space") @@ -1490,9 +1559,9 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) *------------------------------------------------------------------------- */ static herr_t -H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) +H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -1501,23 +1570,26 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) HDassert(f); /* Flush the entire metadata cache */ - if(H5AC_flush(f, dxpl_id) < 0) + if(H5AC_flush(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache") /* Truncate the file to the current allocated size */ - if(H5FD_truncate(f->shared->lf, dxpl_id, closing) < 0) + if(H5FD_truncate(f->shared->lf, meta_dxpl_id, closing) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed") /* Flush the entire metadata cache again since the EOA could have changed in the truncate call. */ - if(H5AC_flush(f, dxpl_id) < 0) + if(H5AC_flush(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache") /* Set up I/O info for operation */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) /* Push error, but keep going*/ HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") @@ -1526,8 +1598,13 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) /* Push error, but keep going*/ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush metadata accumulator") + /* Flush the page buffer */ + if(H5PB_flush(&fio_info) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "page buffer flush failed") + /* Flush file buffers to disk. */ - if(H5FD_flush(f->shared->lf, dxpl_id, closing) < 0) + if(H5FD_flush(f->shared->lf, meta_dxpl_id, closing) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "low level flush failed") @@ -1536,7 +1613,7 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) /*------------------------------------------------------------------------- - * Function: H5F_flush + * Function: H5F__flush * * Purpose: Flushes cached data. * @@ -1549,28 +1626,27 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) *------------------------------------------------------------------------- */ herr_t -H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing) +H5F__flush(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_PACKAGE /* Sanity check arguments */ HDassert(f); /* First phase of flushing data */ - if(H5F__flush_phase1(f, dxpl_id) < 0) + if(H5F__flush_phase1(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data") /* Second phase of flushing data */ - if(H5F__flush_phase2(f, dxpl_id, closing) < 0) + if(H5F__flush_phase2(f, meta_dxpl_id, raw_dxpl_id, closing) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data") -done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_flush() */ +} /* end H5F__flush() */ /*------------------------------------------------------------------------- @@ -1795,7 +1871,7 @@ H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/) if(H5F_efc_try_close(f) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't attempt to close EFC") - /* Delay flush until the shared file struct is closed, in H5F_dest. If the + /* Delay flush until the shared file struct is closed, in H5F__dest. If the * application called H5Fclose, it would have been flushed in that function * (unless it will have been flushed in H5F_dest anyways). */ @@ -1804,7 +1880,7 @@ H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/) * shared H5F_file_t struct. If the reference count for the H5F_file_t * struct reaches zero then destroy it also. */ - if(H5F_dest(f, H5AC_ind_read_dxpl_id, TRUE) < 0) + if(H5F__dest(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problems closing file") /* Since we closed the file, this should be set to TRUE */ @@ -2362,7 +2438,8 @@ H5F_set_store_msg_crt_idx(H5F_t *f, hbool_t flag) *------------------------------------------------------------------------- */ ssize_t -H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) +H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t meta_dxpl_id, + hid_t raw_dxpl_id) { H5FD_t *fd_ptr; /* file driver */ haddr_t eoa; /* End of file address */ @@ -2429,10 +2506,10 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) /* test to see if a buffer was provided -- if not, we are done */ if(buf_ptr != NULL) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ size_t space_needed; /* size of file image */ hsize_t tmp; size_t tmp_size; - H5P_genplist_t *xfer_plist= NULL; /* Dataset transfer property list object */ /* Check for buffer too small */ if((haddr_t)buf_len < eoa) @@ -2440,13 +2517,16 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) space_needed = (size_t)eoa; - /* Get the property list object */ - if(NULL == (xfer_plist = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object") + /* Set up file driver I/O info object */ + fdio_info.file = fd_ptr; + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get property list object") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) + HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get property list object") /* read in the file image */ /* (Note compensation for base address addition in internal routine) */ - if(H5FD_read(fd_ptr, xfer_plist, H5FD_MEM_DEFAULT, 0, space_needed, buf_ptr) < 0) + if(H5FD_read(&fdio_info, H5FD_MEM_DEFAULT, 0, space_needed, buf_ptr) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "file image read request failed") /* Offset to "status_flags" in the superblock */ @@ -2648,6 +2728,38 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__set_eoa() */ + +/*------------------------------------------------------------------------- + * Function: H5F__set_paged_aggr + * + * Purpose: Quick and dirty routine to set the file's paged_aggr mode + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * June 19, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__set_paged_aggr(const H5F_t *f, hbool_t paged) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(f); + HDassert(f->shared); + + /* Dispatch to driver */ + if(H5FD_set_paged_aggr(f->shared->lf, paged) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "driver set paged aggr mode failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__set_paged_aggr() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Fio.c b/src/H5Fio.c index afe1278..6d23995 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -39,6 +39,7 @@ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ #include "H5Iprivate.h" /* IDs */ +#include "H5PBprivate.h" /* Page Buffer */ /****************/ @@ -96,15 +97,11 @@ herr_t H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, hid_t dxpl_id, void *buf/*out*/) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ H5FD_mem_t map_type; /* Mapped memory type */ - hid_t my_dxpl_id = dxpl_id; /* transfer property to use for I/O */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) -#ifdef QAK -HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size); -#endif /* QAK */ HDassert(f); HDassert(f->shared); @@ -118,20 +115,24 @@ HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size); /* Treat global heap as raw data */ map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; -#ifdef H5_DEBUG_BUILD - /* GHEAP type is treated as RAW, so update the dxpl type property too */ - if(H5FD_MEM_GHEAP == type) - my_dxpl_id = H5AC_rawdata_dxpl_id; -#endif /* H5_DEBUG_BUILD */ - - /* Set up I/O info for operation */ + /* Set up the I/O info object */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(my_dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(H5FD_MEM_DRAW == type) { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ - /* Pass through metadata accumulator layer */ - if(H5F__accum_read(&fio_info, map_type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through metadata accumulator failed") + /* Pass through page buffer layer */ + if(H5PB_read(&fio_info, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through page buffer failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -157,15 +158,11 @@ herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, hid_t dxpl_id, const void *buf) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ H5FD_mem_t map_type; /* Mapped memory type */ - hid_t my_dxpl_id = dxpl_id; /* transfer property to use for I/O */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) -#ifdef QAK -HDfprintf(stderr, "%s: write to addr = %a, size = %Zu\n", FUNC, addr, size); -#endif /* QAK */ HDassert(f); HDassert(f->shared); @@ -180,20 +177,24 @@ HDfprintf(stderr, "%s: write to addr = %a, size = %Zu\n", FUNC, addr, size); /* Treat global heap as raw data */ map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; -#ifdef H5_DEBUG_BUILD - /* GHEAP type is treated as RAW, so update the dxpl type property too */ - if(H5FD_MEM_GHEAP == type) - my_dxpl_id = H5AC_rawdata_dxpl_id; -#endif /* H5_DEBUG_BUILD */ - - /* Set up I/O info for operation */ + /* Set up the I/O info object */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(my_dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(H5FD_MEM_DRAW == type) { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ - /* Pass through metadata accumulator layer */ - if(H5F__accum_write(&fio_info, map_type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") + /* Pass through page buffer layer */ + if(H5PB_write(&fio_info, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through page buffer failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -216,7 +217,7 @@ done: herr_t H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) @@ -227,10 +228,10 @@ H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id) /* Set up I/O info for operation */ fio_info.f = f; - - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") - /* Flush and reset the accumulator */ if(H5F__accum_reset(&fio_info, TRUE) < 0) diff --git a/src/H5Fmount.c b/src/H5Fmount.c index e3d4952..859b9d6 100644 --- a/src/H5Fmount.c +++ b/src/H5Fmount.c @@ -614,7 +614,7 @@ H5F_mount_count_ids(H5F_t *f, unsigned *nopen_files, unsigned *nopen_objs) *------------------------------------------------------------------------- */ static herr_t -H5F_flush_mounts_recurse(H5F_t *f, hid_t dxpl_id) +H5F_flush_mounts_recurse(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id) { unsigned nerrors = 0; /* Errors from recursive flushes */ unsigned u; /* Index variable */ @@ -627,11 +627,11 @@ H5F_flush_mounts_recurse(H5F_t *f, hid_t dxpl_id) /* Flush all child files, not stopping for errors */ for(u = 0; u < f->shared->mtab.nmounts; u++) - if(H5F_flush_mounts_recurse(f->shared->mtab.child[u].file, dxpl_id) < 0) + if(H5F_flush_mounts_recurse(f->shared->mtab.child[u].file, meta_dxpl_id, raw_dxpl_id) < 0) nerrors++; /* Call the "real" flush routine, for this file */ - if(H5F_flush(f, dxpl_id, FALSE) < 0) + if(H5F__flush(f, meta_dxpl_id, raw_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") /* Check flush errors for children - errors are already on the stack */ @@ -656,7 +656,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_flush_mounts(H5F_t *f, hid_t dxpl_id) +H5F_flush_mounts(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ @@ -670,7 +670,7 @@ H5F_flush_mounts(H5F_t *f, hid_t dxpl_id) f = f->parent; /* Flush the mounted file hierarchy */ - if(H5F_flush_mounts_recurse(f, dxpl_id) < 0) + if(H5F_flush_mounts_recurse(f, meta_dxpl_id, raw_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush mounted file hierarchy") done: diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 93a3978..4dd9e20 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -42,6 +42,7 @@ #include "H5FSprivate.h" /* File free space */ #include "H5Gprivate.h" /* Groups */ #include "H5Oprivate.h" /* Object header messages */ +#include "H5PBprivate.h" /* Page buffer */ #include "H5UCprivate.h" /* Reference counted object functions */ @@ -68,8 +69,8 @@ /* Macro to abstract checking whether file is using a free space manager */ #define H5F_HAVE_FREE_SPACE_MANAGER(F) \ - ((F)->shared->fs_strategy == H5F_FILE_SPACE_ALL || \ - (F)->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) + ((F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || \ + (F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) /* Macros for encoding/decoding superblock */ #define H5F_MAX_DRVINFOBLOCK_SIZE 1024 /* Maximum size of superblock driver info buffer */ @@ -200,13 +201,6 @@ typedef struct H5F_meta_accum_t { hbool_t dirty; /* Flag to indicate that the accumulated metadata is dirty */ } H5F_meta_accum_t; -/* Enum for free space manager state */ -typedef enum H5F_fs_state_t { - H5F_FS_STATE_CLOSED, /* Free space manager is closed */ - H5F_FS_STATE_OPEN, /* Free space manager has been opened */ - H5F_FS_STATE_DELETING /* Free space manager is being deleted */ -} H5F_fs_state_t; - /* A record of the mount table */ typedef struct H5F_mount_t { struct H5G_t *group; /* Mount point group held open */ @@ -281,6 +275,7 @@ struct H5F_file_t { unsigned long feature_flags; /* VFL Driver feature Flags */ haddr_t maxaddr; /* Maximum address for file */ + H5PB_t *page_buf; /* The page buffer cache */ H5AC_t *cache; /* The object cache */ H5AC_cache_config_t mdc_initCacheCfg; /* initial configuration for the */ @@ -316,19 +311,38 @@ struct H5F_file_t { H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */ /* File space allocation information */ - H5F_file_space_type_t fs_strategy; /* File space handling strategy */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ hsize_t fs_threshold; /* Free space section threshold */ + hbool_t fs_persist; /* Free-space persist or not */ hbool_t use_tmp_space; /* Whether temp. file space allocation is allowed */ haddr_t tmp_addr; /* Next address to use for temp. space in the file */ + hbool_t point_of_no_return; /* flag to indicate that we can't go back and delete a freespace header when it's used up */ + + H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */ + haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */ + H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */ + hbool_t first_alloc_dealloc; /* TRUE iff free space managers */ + /* are persistant and have not */ + /* been used accessed for either */ + /* allocation or deallocation */ + /* since file open. */ + haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_mdci_fsalloc; /* eoa past file space allocation */ + /* for metadata cache image, or */ + /* HADDR_UNDEF if no cache image. */ + + /* Free-space aggregation info */ unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */ - H5F_fs_state_t fs_state[H5FD_MEM_NTYPES]; /* State of free space manager for each type */ - haddr_t fs_addr[H5FD_MEM_NTYPES]; /* Address of free space manager info for each type */ - H5FS_t *fs_man[H5FD_MEM_NTYPES]; /* Free space manager for each file space type */ - H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ - H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info */ - /* (if aggregating metadata allocations) */ - H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info */ - /* (if aggregating "small data" allocations) */ + H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ + H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */ + H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */ + + /* Paged aggregation info */ + hsize_t fs_page_size; /* File space page size */ + size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */ /* Metadata accumulator information */ H5F_meta_accum_t accum; /* Metadata accumulator info */ @@ -383,11 +397,12 @@ H5FL_EXTERN(H5F_file_t); /* General routines */ H5F_t *H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); -herr_t H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush); -H5_DLL herr_t H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing); -H5_DLL htri_t H5F_is_hdf5(const char *name, hid_t dxpl_id); +H5_DLL herr_t H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush); +H5_DLL herr_t H5F__flush(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing); +H5_DLL htri_t H5F__is_hdf5(const char *name, hid_t meta_dxpl_id, hid_t raw_dxpl_id); H5_DLL herr_t H5F_get_objects(const H5F_t *f, unsigned types, size_t max_index, hid_t *obj_id_list, hbool_t app_ref, size_t *obj_id_count_ptr); -H5_DLL ssize_t H5F_get_file_image(H5F_t *f, void *buf_ptr, size_t buf_len, hid_t dxpl_id); +H5_DLL ssize_t H5F_get_file_image(H5F_t *f, void *buf_ptr, size_t buf_len, + hid_t meta_dxpl_id, hid_t raw_dxpl_id); H5_DLL herr_t H5F_close(H5F_t *f); /* File mount related routines */ @@ -397,7 +412,8 @@ H5_DLL herr_t H5F_mount_count_ids(H5F_t *f, unsigned *nopen_files, unsigned *nop /* Superblock related routines */ H5_DLL herr_t H5F__super_init(H5F_t *f, hid_t dxpl_id); -H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read); +H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, + hbool_t initial_read); H5_DLL herr_t H5F__super_size(H5F_t *f, hid_t dxpl_id, hsize_t *super_size, hsize_t *super_ext_size); H5_DLL herr_t H5F__super_free(H5F_super_t *sblock); @@ -410,14 +426,14 @@ H5_DLL herr_t H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hid_t dxpl_id, hbool_t was_created); /* Metadata accumulator routines */ -H5_DLL herr_t H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t type, +H5_DLL herr_t H5F__accum_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, size_t size, void *buf); -H5_DLL herr_t H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t type, +H5_DLL herr_t H5F__accum_write(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); -H5_DLL herr_t H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t type, +H5_DLL herr_t H5F__accum_free(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, hsize_t size); -H5_DLL herr_t H5F__accum_flush(const H5F_io_info_t *fio_info); -H5_DLL herr_t H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush); +H5_DLL herr_t H5F__accum_flush(const H5F_io_info2_t *fio_info); +H5_DLL herr_t H5F__accum_reset(const H5F_io_info2_t *fio_info, hbool_t flush); /* Shared file list related routines */ H5_DLL herr_t H5F_sfile_add(H5F_file_t *shared); @@ -431,9 +447,16 @@ H5_DLL herr_t H5F_efc_release(H5F_efc_t *efc); H5_DLL herr_t H5F_efc_destroy(H5F_efc_t *efc); H5_DLL herr_t H5F_efc_try_close(H5F_t *f); +/* Space allocation routines */ +H5_DLL haddr_t H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size); +H5_DLL herr_t H5F_free(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, haddr_t addr, hsize_t size); +H5_DLL htri_t H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, + haddr_t blk_end, hsize_t extra_requested); + /* Functions that get/retrieve values from VFD layer */ H5_DLL herr_t H5F__set_eoa(const H5F_t *f, H5F_mem_t type, haddr_t addr); H5_DLL herr_t H5F__set_base_addr(const H5F_t *f, haddr_t addr); +H5_DLL herr_t H5F__set_paged_aggr(const H5F_t *f, hbool_t paged); /* Functions that flush or evict */ H5_DLL herr_t H5F__evict_cache_entries(H5F_t *f, hid_t dxpl_id); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 7d288fa..886063a 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -315,8 +315,8 @@ #define H5F_SET_STORE_MSG_CRT_IDX(F, FL) ((F)->shared->store_msg_crt_idx = (FL)) #define H5F_GRP_BTREE_SHARED(F) ((F)->shared->grp_btree_shared) #define H5F_SET_GRP_BTREE_SHARED(F, RC) (((F)->shared->grp_btree_shared = (RC)) ? SUCCEED : FAIL) -#define H5F_USE_TMP_SPACE(F) ((F)->shared->use_tmp_space) -#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->tmp_addr, (ADDR))) +#define H5F_USE_TMP_SPACE(F) ((F)->shared->fs.use_tmp_space) +#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->fs.tmp_addr, (ADDR))) #define H5F_SET_LATEST_FLAGS(F, FL) ((F)->shared->latest_flags = (FL)) #ifdef H5_HAVE_PARALLEL #define H5F_COLL_MD_READ(F) ((F)->coll_md_read) @@ -324,6 +324,12 @@ #define H5F_USE_MDC_LOGGING(F) ((F)->shared->use_mdc_logging) #define H5F_START_MDC_LOG_ON_ACCESS(F) ((F)->shared->start_mdc_log_on_access) #define H5F_MDC_LOG_LOCATION(F) ((F)->shared->mdc_log_location) +#define H5F_ALIGNMENT(F) ((F)->shared->alignment) +#define H5F_THRESHOLD(F) ((F)->shared->threshold) +#define H5F_PGEND_META_THRES(F) ((F)->shared->fs.pgend_meta_thres) +#define H5F_POINT_OF_NO_RETURN(F) ((F)->shared->fs.point_of_no_return) +#define H5F_FIRST_ALLOC_DEALLOC(F) ((F)->shared->first_alloc_dealloc) +#define H5F_EOA_PRE_FSM_FSALLOC(F) ((F)->shared->eoa_pre_fsm_fsalloc) #else /* H5F_MODULE */ #define H5F_INTENT(F) (H5F_get_intent(F)) #define H5F_OPEN_NAME(F) (H5F_get_open_name(F)) @@ -375,6 +381,12 @@ #define H5F_USE_MDC_LOGGING(F) (H5F_use_mdc_logging(F)) #define H5F_START_MDC_LOG_ON_ACCESS(F) (H5F_start_mdc_log_on_access(F)) #define H5F_MDC_LOG_LOCATION(F) (H5F_mdc_log_location(F)) +#define H5F_ALIGNMENT(F) (H5F_get_alignment(F)) +#define H5F_THRESHOLD(F) (H5F_get_threshold(F)) +#define H5F_PGEND_META_THRES(F) (H5F_get_pgend_meta_thres(F)) +#define H5F_POINT_OF_NO_RETURN(F) (H5F_get_point_of_no_return(F)) +#define H5F_FIRST_ALLOC_DEALLOC(F) (H5F_get_first_alloc_dealloc(F)) +#define H5F_EOA_PRE_FSM_FSALLOC(F) (H5F_get_eoa_pre_fsm_fsalloc(F)) #endif /* H5F_MODULE */ @@ -448,7 +460,9 @@ #define H5F_CRT_SHMSG_LIST_MAX_NAME "shmsg_list_max" /* Shared message list maximum size */ #define H5F_CRT_SHMSG_BTREE_MIN_NAME "shmsg_btree_min" /* Shared message B-tree minimum size */ #define H5F_CRT_FILE_SPACE_STRATEGY_NAME "file_space_strategy" /* File space handling strategy */ +#define H5F_CRT_FREE_SPACE_PERSIST_NAME "free_space_persist" /* Free-space persisting status */ #define H5F_CRT_FREE_SPACE_THRESHOLD_NAME "free_space_threshold" /* Free space section threshold */ +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME "file_space_page_size" /* File space page size */ @@ -484,6 +498,9 @@ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ #define H5F_ACS_COLL_MD_WRITE_FLAG_NAME "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */ #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME "mdc_initCacheImageCfg" /* Initial metadata cache image creation configuration */ +#define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */ +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -522,10 +539,34 @@ /* See format specification on version 1 B-trees */ /* Default file space handling strategy */ -#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_ALL +#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR + +/* Default free space section threshold used by free-space managers */ +#define H5F_FREE_SPACE_PERSIST_DEF FALSE + /* Default free space section threshold used by free-space managers */ #define H5F_FREE_SPACE_THRESHOLD_DEF 1 +/* For paged aggregation: default file space page size when not set */ +#define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096 +/* For paged aggregation: minimum value for file space page size */ +#define H5F_FILE_SPACE_PAGE_SIZE_MIN 512 + +/* For paged aggregation: drop free-space with size <= this threshold for small meta section */ +#define H5F_FILE_SPACE_PGEND_META_THRES 10 + +/* Default for threshold for alignment (can be set via H5Pset_alignment()) */ +#define H5F_ALIGN_DEF 1 +/* Default for alignment (can be set via H5Pset_alignment()) */ +#define H5F_ALIGN_THRHD_DEF 1 +/* Default size for meta data aggregation block (can be set via H5Pset_meta_block_size()) */ +#define H5F_META_BLOCK_SIZE_DEF 2048 +/* Default size for small data aggregation block (can be set via H5Pset_small_data_block_size()) */ +#define H5F_SDATA_BLOCK_SIZE_DEF 2048 + +/* Check for file using paged aggregation */ +#define H5F_PAGED_AGGR(F) (F->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE && F->shared->fs_page_size) + /* Metadata read attempt values */ #define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */ #define H5F_SWMR_METADATA_READ_ATTEMPTS 100 /* Default # of read attempts for SWMR access */ @@ -628,18 +669,54 @@ typedef struct H5F_object_flush_t { void *udata; /* User data */ } H5F_object_flush_t; -/* I/O Info for an operation */ +/* I/O Info for an operation (old) */ typedef struct H5F_io_info_t { const H5F_t *f; /* File object */ const struct H5P_genplist_t *dxpl; /* DXPL object */ } H5F_io_info_t; +/* I/O Info for an operation */ +/* (Migrate toward this one, so that both raw data & metadata DXPLs are available) */ +typedef struct H5F_io_info2_t { + const H5F_t *f; /* File object */ + const struct H5P_genplist_t *meta_dxpl; /* Metadata DXPL object */ + const struct H5P_genplist_t *raw_dxpl; /* Raw data DXPL object */ +} H5F_io_info2_t; + /* Concise info about a block of bytes in a file */ typedef struct H5F_block_t { haddr_t offset; /* Offset of the block in the file */ hsize_t length; /* Length of the block in the file */ } H5F_block_t; +/* Enum for free space manager state */ +typedef enum H5F_fs_state_t { + H5F_FS_STATE_CLOSED = 0, /* Free space manager is closed */ + H5F_FS_STATE_OPEN = 1, /* Free space manager has been opened */ + H5F_FS_STATE_DELETING = 2 /* Free space manager is being deleted */ +} H5F_fs_state_t; + +/* For paged aggregation */ +/* The values 0 to 6 is the same as H5F_mem_t */ +typedef enum H5F_mem_page_t { + H5F_MEM_PAGE_DEFAULT = 0, /* Not used */ + H5F_MEM_PAGE_SUPER = 1, + H5F_MEM_PAGE_BTREE = 2, + H5F_MEM_PAGE_DRAW = 3, + H5F_MEM_PAGE_GHEAP = 4, + H5F_MEM_PAGE_LHEAP = 5, + H5F_MEM_PAGE_OHDR = 6, + H5F_MEM_PAGE_LARGE_SUPER = 7, + H5F_MEM_PAGE_LARGE_BTREE = 8, + H5F_MEM_PAGE_LARGE_DRAW = 9, + H5F_MEM_PAGE_LARGE_GHEAP = 10, + H5F_MEM_PAGE_LARGE_LHEAP = 11, + H5F_MEM_PAGE_LARGE_OHDR = 12, + H5F_MEM_PAGE_NTYPES = 13 /* Sentinel value - must be last */ +} H5F_mem_page_t; + +#define H5F_MEM_PAGE_META H5F_MEM_PAGE_SUPER /* Small-sized meta data */ +#define H5F_MEM_PAGE_GENERIC H5F_MEM_PAGE_LARGE_SUPER /* Large-sized generic: meta and raw */ /*****************************/ /* Library-private Variables */ @@ -674,6 +751,10 @@ H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref); H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref); H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, size_t *obj_id_count_ptr); H5_DLL herr_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *oid_list, hbool_t app_ref, size_t *obj_id_count_ptr); +H5_DLL hsize_t H5F_get_pgend_meta_thres(const H5F_t *f); +H5_DLL hbool_t H5F_get_point_of_no_return(const H5F_t *f); +H5_DLL hbool_t H5F_get_first_alloc_dealloc(const H5F_t *f); +H5_DLL hbool_t H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); @@ -704,6 +785,8 @@ H5_DLL herr_t H5F_set_grp_btree_shared(H5F_t *f, struct H5UC_t *rc); H5_DLL hbool_t H5F_use_tmp_space(const H5F_t *f); H5_DLL hbool_t H5F_is_tmp_addr(const H5F_t *f, haddr_t addr); H5_DLL herr_t H5F_set_latest_flags(H5F_t *f, unsigned flags); +H5_DLL hsize_t H5F_get_alignment(const H5F_t *f); +H5_DLL hsize_t H5F_get_threshold(const H5F_t *f); #ifdef H5_HAVE_PARALLEL H5_DLL H5P_coll_md_read_flag_t H5F_coll_md_read(const H5F_t *f); H5_DLL void H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t flag); @@ -723,7 +806,7 @@ H5_DLL herr_t H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_hand H5_DLL hbool_t H5F_is_mount(const H5F_t *file); H5_DLL hbool_t H5F_has_mount(const H5F_t *file); H5_DLL herr_t H5F_traverse_mount(struct H5O_loc_t *oloc/*in,out*/); -H5_DLL herr_t H5F_flush_mounts(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5F_flush_mounts(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id); /* Functions that operate on blocks of bytes wrt super block */ H5_DLL herr_t H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index c57a821..f87aaad 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -182,6 +182,17 @@ typedef enum H5F_libver_t { } H5F_libver_t; /* File space handling strategy */ +typedef enum H5F_fspace_strategy_t { + H5F_FSPACE_STRATEGY_FSM_AGGR = 0, /* Mechanisms: free-space managers, aggregators, and virtual file drivers */ + /* This is the library default when not set */ + H5F_FSPACE_STRATEGY_PAGE = 1, /* Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */ + H5F_FSPACE_STRATEGY_AGGR = 2, /* Mechanisms: aggregators and virtual file drivers */ + H5F_FSPACE_STRATEGY_NONE = 3, /* Mechanisms: virtual file drivers */ + H5F_FSPACE_STRATEGY_NTYPES /* must be last */ +} H5F_fspace_strategy_t; + +/* Deprecated: File space handling strategy for release 1.10.0 */ +/* They are mapped to H5F_fspace_strategy_t as defined above from release 1.10.1 onwards */ typedef enum H5F_file_space_type_t { H5F_FILE_SPACE_DEFAULT = 0, /* Default (or current) free space strategy setting */ H5F_FILE_SPACE_ALL_PERSIST = 1, /* Persistent free space managers, aggregators, virtual file driver */ @@ -253,6 +264,10 @@ H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, /*OUT*/ hbool_t *is_enabled, /*OUT*/ hbool_t *is_currently_logging); H5_DLL herr_t H5Fformat_convert(hid_t fid); +H5_DLL herr_t H5Freset_page_buffering_stats(hid_t file_id); +H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], + unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); + #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag); H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 14dd655..34fe8f9 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1235,3 +1235,150 @@ H5F_mdc_log_location(const H5F_t *f) FUNC_LEAVE_NOAPI(f->shared->mdc_log_location) } /* end H5F_mdc_log_location() */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_alignment + * + * Purpose: Retrieve the 'alignment' for the file. + * + * Return: Success: Non-negative, the 'alignment' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_alignment(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->alignment) +} /* end H5F_get_alignment() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_threshold + * + * Purpose: Retrieve the 'threshold' for alignment in the file. + * + * Return: Success: Non-negative, the 'threshold' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_threshold(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->threshold) +} /* end H5F_get_threshold() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_pgend_meta_thres + * + * Purpose: Retrieve the 'page end meta threshold size' for the file. + * + * Return: Success: Non-negative, the 'pgend_meta_thres' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_pgend_meta_thres(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->pgend_meta_thres) +} /* end H5F_get_pgend_meta_thres() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_point_of_no_return + * + * Purpose: Retrieve the 'point of no return' value for the file. + * + * Return: Success: Non-negative, the 'point_of_no_return' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_point_of_no_return(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->point_of_no_return) +} /* end H5F_get_point_of_no_return() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_first_alloc_dealloc + * + * Purpose: Retrieve the 'first alloc / dealloc' value for the file. + * + * Return: Success: Non-negative, the 'first_alloc_dealloc' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_first_alloc_dealloc(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->first_alloc_dealloc) +} /* end H5F_get_first_alloc_dealloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_eoa_pre_fsm_fsalloc + * + * Purpose: Retrieve the 'EOA pre-FSM fsalloc' value for the file. + * + * Return: Success: Non-negative, the 'EOA pre-FSM fsalloc' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->eoa_pre_fsm_fsalloc) +} /* end H5F_get_eoa_pre_fsm_fsalloc() */ + diff --git a/src/H5Fspace.c b/src/H5Fspace.c new file mode 100644 index 0000000..53570f6 --- /dev/null +++ b/src/H5Fspace.c @@ -0,0 +1,226 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Fspace.c + * Dec 30 2013 + * Quincey Koziol <koziol@hdfgroup.org> + * + * Purpose: Space allocation routines for the file. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Fmodule.h" /* This source code file is part of the H5F module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5F_alloc + * + * Purpose: Wrapper for H5FD_alloc, to make certain EOA changes are + * reflected in superblock. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: The format address of the new file memory. + * Failure: The undefined address HADDR_UNDEF + * + * Programmer: Quincey Koziol + * Monday, December 30, 2013 + * + *------------------------------------------------------------------------- + */ +haddr_t +H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +{ + haddr_t ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI(HADDR_UNDEF) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(size > 0); + + /* Check whether the file can use temporary addresses */ + if(f->shared->use_tmp_space) { + haddr_t eoa; /* Current EOA for the file */ + + /* Get the EOA for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") + + /* Check for overlapping into file's temporary allocation space */ + if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) + HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") + } /* end if */ + + /* Call the file driver 'alloc' routine */ + ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, frag_addr, frag_size); + if(!H5F_addr_defined(ret_value)) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, HADDR_UNDEF, "file driver 'alloc' request failed") + + /* Mark EOA dirty */ + if(H5F_eoa_dirty(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark EOA as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_free + * + * Purpose: Wrapper for H5FD_free, to make certain EOA changes are + * reflected in superblock. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, December 30, 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, hsize_t size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(size > 0); + + /* Call the file driver 'free' routine */ + if(H5FD_free(f->shared->lf, dxpl_id, type, f, addr, size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "file driver 'free' request failed") + + /* Mark EOA dirty */ + if(H5F_eoa_dirty(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_try_extend + * + * Purpose: Extend a block at the end of the file, if possible. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: TRUE(1) - Block was extended + * FALSE(0) - Block could not be extended + * Failure: FAIL + * + * Programmer: Quincey Koziol + * Monday, 30 December, 2013 + * + *------------------------------------------------------------------------- + */ +htri_t +H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t blk_end, hsize_t extra_requested) +{ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(extra_requested > 0); + + /* Extend the object by extending the underlying file */ + if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, blk_end, extra_requested)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTEXTEND, FAIL, "driver try extend request failed") + + /* H5FD_try_extend() updates driver message and marks the superblock + * dirty, so no need to do it again here. + */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_try_extend() */ + diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 3b86dae..58ef9bb 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -29,6 +29,7 @@ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ #include "H5Iprivate.h" /* IDs */ +#include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ #include "H5SMprivate.h" /* Shared Object Header Messages */ @@ -320,13 +321,14 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) +H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial_read) { H5P_genplist_t *dxpl = NULL; /* DXPL object */ H5AC_ring_t ring, orig_ring = H5AC_RING_INV; H5F_super_t * sblock = NULL; /* Superblock structure */ H5F_superblock_cache_ud_t udata; /* User data for cache callbacks */ H5P_genplist_t *c_plist; /* File creation property list */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ unsigned sblock_flags = H5AC__NO_FLAGS_SET; /* flags used in superblock unprotect call */ haddr_t super_addr; /* Absolute address of superblock */ haddr_t eof; /* End of file address */ @@ -334,7 +336,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE_TAG(dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL) + FUNC_ENTER_PACKAGE_TAG(meta_dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL) /* initialize the drvinfo to NULL -- we will overwrite this if there * is a driver information block @@ -342,13 +344,19 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) f->shared->drvinfo = NULL; /* Get the DXPL plist object for DXPL ID */ - if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") if((H5P_get(dxpl, H5AC_RING_NAME, &orig_ring)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get property value"); + /* Set up file driver I/O info */ + fdio_info.file = f->shared->lf; + fdio_info.meta_dxpl = dxpl; + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + /* Find the superblock */ - if(H5FD_locate_signature(f->shared->lf, dxpl, &super_addr) < 0) + if(H5FD_locate_signature(&fdio_info, &super_addr) < 0) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") if(HADDR_UNDEF == super_addr) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "file signature not found") @@ -396,7 +404,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set property value"); /* Look up the superblock */ - if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags))) + if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags))) HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock") if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) @@ -556,7 +564,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") /* Look up the driver info block */ - if(NULL == (drvinfo = (H5O_drvinfo_t *)H5AC_protect(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, &drvrinfo_udata, rw_flags))) + if(NULL == (drvinfo = (H5O_drvinfo_t *)H5AC_protect(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, &drvrinfo_udata, rw_flags))) HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load driver info block") /* Loading the driver info block is enough to set up the right info */ @@ -571,7 +579,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) drvinfo_flags |= H5AC__PIN_ENTRY_FLAG; /* Release the driver info block */ - if(H5AC_unprotect(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, drvinfo, drvinfo_flags) < 0) + if(H5AC_unprotect(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, drvinfo, drvinfo_flags) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTUNPROTECT, FAIL, "unable to release driver info block") /* save a pointer to the driver information cache entry */ @@ -612,14 +620,14 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") /* Check for the extension having a 'driver info' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_DRVINFO_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_DRVINFO_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { /* Check for ignoring the driver info for this file */ if(!udata.ignore_drvrinfo) { /* Retrieve the 'driver info' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_DRVINFO_ID, &drvinfo, dxpl_id)) + if(NULL == H5O_msg_read(&ext_loc, H5O_DRVINFO_ID, &drvinfo, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver info message not present") /* Validate and decode driver information */ @@ -637,15 +645,15 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) } /* end if */ /* Read in the shared OH message information if there is any */ - if(H5SM_get_info(&ext_loc, c_plist, dxpl_id) < 0) + if(H5SM_get_info(&ext_loc, c_plist, meta_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to read SOHM table information") /* Check for the extension having a 'v1 B-tree "K"' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_BTREEK_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_BTREEK_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { /* Retrieve the 'v1 B-tree "K"' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_BTREEK_ID, &btreek, dxpl_id)) + if(NULL == H5O_msg_read(&ext_loc, H5O_BTREEK_ID, &btreek, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "v1 B-tree 'K' info message not present") /* Set non-default v1 B-tree 'K' value info from file */ @@ -661,39 +669,105 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) } /* end if */ /* Check for the extension having a 'free-space manager info' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { - H5O_fsinfo_t fsinfo; /* Free-space manager info message from superblock extension */ + H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */ + uint8_t flags; /* Message flags */ - /* Retrieve the 'free-space manager info' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, dxpl_id)) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") + /* Get message flags */ + if(H5O_msg_get_flags(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id, &flags) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to message flags for free-space manager info message") - /* Check for non-default info */ - if(f->shared->fs_strategy != fsinfo.strategy) { - f->shared->fs_strategy = fsinfo.strategy; + /* If message is NOT marked "unknown"--set up file space info */ + if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) { - /* Set non-default strategy in the property list */ - if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") - } /* end if */ - if(f->shared->fs_threshold != fsinfo.threshold) { - f->shared->fs_threshold = fsinfo.threshold; + /* Retrieve the 'file space info' structure */ + if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, meta_dxpl_id)) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") - /* Set non-default threshold in the property list */ - if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") - } /* end if */ + /* Update changed values */ + if(f->shared->fs_strategy != fsinfo.strategy) { + f->shared->fs_strategy = fsinfo.strategy; + + /* Set non-default strategy in the property list */ + if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + if(f->shared->fs_persist != fsinfo.persist) { + f->shared->fs_persist = fsinfo.persist; + + /* Set non-default strategy in the property list */ + if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &fsinfo.persist) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + if(f->shared->fs_threshold != fsinfo.threshold) { + f->shared->fs_threshold = fsinfo.threshold; + + /* Set non-default threshold in the property list */ + if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + HDassert(fsinfo.page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + if(f->shared->fs_page_size != fsinfo.page_size) { + f->shared->fs_page_size = fsinfo.page_size; + + /* Set file space page size in the property list */ + if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsinfo.page_size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space page size") + } /* end if */ + if(f->shared->pgend_meta_thres != fsinfo.pgend_meta_thres) + /* Initialize page end meta threshold */ + f->shared->pgend_meta_thres = fsinfo.pgend_meta_thres; + + if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc) + f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc; + + /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF + * in the absence of persistant free space managers. + */ + HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF)); + HDassert(!f->shared->first_alloc_dealloc); + + if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF) && + (H5F_INTENT(f) & H5F_ACC_RDWR)) + f->shared->first_alloc_dealloc = TRUE; + + f->shared->fs_addr[0] = HADDR_UNDEF; + for(u = 1; u < NELMTS(f->shared->fs_addr); u++) + f->shared->fs_addr[u] = fsinfo.fs_addr[u - 1]; - /* Set free-space manager addresses */ - f->shared->fs_addr[0] = HADDR_UNDEF; - for(u = 1; u < NELMTS(f->shared->fs_addr); u++) - f->shared->fs_addr[u] = fsinfo.fs_addr[u-1]; + if(fsinfo.mapped && (rw_flags & H5AC__READ_ONLY_FLAG) == 0) { + + /* Do the same kluge until we know for sure. VC */ +#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ + /* KLUGE ALERT!! + * + * H5F_super_ext_write_msg() expects f->shared->sblock to + * be set -- verify that it is NULL, and then set it. + * Set it back to NULL when we are done. + */ + HDassert(f->shared->sblock == NULL); + f->shared->sblock = sblock; +#endif /* JRM */ + + if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_FSINFO_ID) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension") + + if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") +#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ + f->shared->sblock = NULL; +#endif /* JRM */ + + } + } /* end if not marked "unknown" */ } /* end if */ /* Check for the extension having a 'metadata cache image' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0); @@ -712,7 +786,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) */ /* Retrieve the 'metadata cache image message' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, dxpl_id)) + if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get metadata cache image message") /* Indicate to the cache that there's an image to load on first protect call */ @@ -721,7 +795,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) } /* end if */ /* Close superblock extension */ - if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) + if(H5F_super_ext_close(f, &ext_loc, meta_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") } /* end if */ @@ -765,7 +839,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HDassert(f->shared->sblock == NULL); f->shared->sblock = sblock; #endif /* JRM */ - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) + if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") #if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ @@ -777,7 +851,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) /* Check for eliminating the driver info block */ else if(H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) { /* Remove the driver info message from the superblock extension */ - if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_DRVINFO_ID) < 0) + if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_DRVINFO_ID) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension") /* Check if the superblock extension was removed */ @@ -789,13 +863,17 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) /* Set the pointer to the pinned superblock */ f->shared->sblock = sblock; + /* Set the page aggregation mode */ + if(H5F__set_paged_aggr(f, (hbool_t)H5F_PAGED_AGGR(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "failed to set paged_aggr status for file driver") + done: /* Reset the ring in the DXPL */ if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set property value") /* Release the superblock */ - if(sblock && H5AC_unprotect(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, sblock_flags) < 0) + if(sblock && H5AC_unprotect(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, sblock_flags) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTUNPROTECT, FAIL, "unable to close superblock") /* If we have failed, make sure no entries are left in the @@ -808,7 +886,7 @@ done: HDONE_ERROR(H5E_FILE, H5E_CANTUNPIN, FAIL, "unable to unpin driver info") /* Evict the driver info block from the cache */ - if(H5AC_expunge_entry(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_expunge_entry(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTEXPUNGE, FAIL, "unable to expunge driver info block") } /* end if */ @@ -819,7 +897,7 @@ done: HDONE_ERROR(H5E_FILE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock") /* Evict the superblock from the cache */ - if(H5AC_expunge_entry(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_expunge_entry(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, H5AC__NO_FLAGS_SET) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTEXPUNGE, FAIL, "unable to expunge superblock") } /* end if */ } /* end if */ @@ -890,7 +968,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Check for non-default free-space settings */ if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF && - f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF)) + f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF && + f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF && + f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF)) non_default_fs_settings = TRUE; /* Bump superblock version if latest superblock version support is enabled */ @@ -899,8 +979,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Bump superblock version to create superblock extension for SOHM info */ else if(f->shared->sohm_nindexes > 0) super_vers = HDF5_SUPERBLOCK_VERSION_2; - /* Bump superblock version to create superblock extension for - * non-default file space strategy or non-default free-space threshold + /* + * Bump superblock version to create superblock extension for: + * -- non-default file space strategy or + * -- non-default persisting free-space or + * -- non-default free-space threshold or + * -- non-default page_size */ else if(non_default_fs_settings) super_vers = HDF5_SUPERBLOCK_VERSION_2; @@ -921,6 +1005,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set superblock version") } /* end if */ + if(H5FD_set_paged_aggr(f->shared->lf, (hbool_t)H5F_PAGED_AGGR(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "failed to set paged_aggr status for file driver") + /* * The superblock starts immediately after the user-defined * header, which we have already insured is a proper size. The @@ -932,9 +1019,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Sanity check the userblock size vs. the file's allocation alignment */ if(userblock_size > 0) { - if(userblock_size < f->shared->alignment) + /* Set up the alignment to use for page or aggr fs */ + hsize_t alignment = H5F_PAGED_AGGR(f) ? f->shared->fs_page_size : f->shared->alignment; + + if(userblock_size < alignment) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be > file object alignment") - if(0 != (userblock_size % f->shared->alignment)) + if(0 != (userblock_size % alignment)) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be an integral multiple of file object alignment") } /* end if */ @@ -990,10 +1080,6 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) if(super_vers < HDF5_SUPERBLOCK_VERSION_2) superblock_size += driver_size; - /* Reserve space in the file for the superblock, instead of allocating it */ - if(H5F__set_eoa(f, H5FD_MEM_SUPER, superblock_size) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to set EOA value for superblock") - /* Set the ring type in the DXPL */ if(H5AC_set_ring(dxpl_id, H5AC_RING_SB, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set ring value") @@ -1006,6 +1092,10 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Keep a copy of the superblock info */ f->shared->sblock = sblock; + /* Allocate space for the superblock */ + if(HADDR_UNDEF == H5MF_alloc(f, H5FD_MEM_SUPER, dxpl_id, superblock_size)) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for superblock") + /* set the drvinfo filed to NULL -- will overwrite this later if needed */ f->shared->drvinfo = NULL; @@ -1108,17 +1198,23 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) } /* end if */ /* Check for non-default free-space info settings */ - if(non_default_fs_settings) { - H5FD_mem_t type; /* Memory type for iteration */ - H5O_fsinfo_t fsinfo; /* Free space manager info message */ - - /* Write free-space manager info message to superblock extension object header if needed */ - fsinfo.strategy = f->shared->fs_strategy; - fsinfo.threshold = f->shared->fs_threshold; - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type-1] = HADDR_UNDEF; - - if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) + if(non_default_fs_settings) { + H5F_mem_page_t ptype; + H5O_fsinfo_t fsinfo; /* Free space manager info message */ + + /* Write free-space manager info message to superblock extension object header if needed */ + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + fsinfo.mapped = FALSE; + + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + + if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE | H5O_MSG_FLAG_MARK_IF_UNKNOWN, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update free-space info header message") } /* end if */ } /* end if */ diff --git a/src/H5HFsection.c b/src/H5HFsection.c index 37ff8f4..b113ca1 100644 --- a/src/H5HFsection.c +++ b/src/H5HFsection.c @@ -85,14 +85,14 @@ static herr_t H5HF_sect_single_full_dblock(H5HF_hdr_t *hdr, hid_t dxpl_id, H5HF_free_section_t *sect); /* 'single' section callbacks */ -static herr_t H5HF_sect_single_add(H5FS_section_info_t *sect, unsigned *flags, +static herr_t H5HF_sect_single_add(H5FS_section_info_t **sect, unsigned *flags, void *udata); static H5FS_section_info_t *H5HF_sect_single_deserialize(const H5FS_section_class_t *cls, hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size, unsigned *des_flags); static htri_t H5HF_sect_single_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5HF_sect_single_merge(H5FS_section_info_t *sect1, +static herr_t H5HF_sect_single_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); static htri_t H5HF_sect_single_can_shrink(const H5FS_section_info_t *sect, void *udata); @@ -121,7 +121,7 @@ static H5FS_section_info_t *H5HF_sect_row_deserialize(const H5FS_section_class_t unsigned *des_flags); static htri_t H5HF_sect_row_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5HF_sect_row_merge(H5FS_section_info_t *sect1, +static herr_t H5HF_sect_row_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); static htri_t H5HF_sect_row_can_shrink(const H5FS_section_info_t *sect, void *udata); @@ -811,7 +811,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) +H5HF_sect_single_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) { herr_t ret_value = SUCCEED; /* Return value */ @@ -821,7 +821,7 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) * have already been checked when it was first added */ if(!(*flags & H5FS_ADD_DESERIALIZING)) { - H5HF_free_section_t *sect = (H5HF_free_section_t *)_sect; /* Fractal heap free section */ + H5HF_free_section_t **sect = (H5HF_free_section_t **)_sect; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ hid_t dxpl_id = udata->dxpl_id; /* DXPL ID for operation */ @@ -832,14 +832,14 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) /* Check if single section covers entire direct block it's in */ /* (converts to row section possibly) */ - if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect) < 0) + if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section") /* Set the "returned space" flag if the single section was changed * into a row section, so the "merging & shrinking" algorithm * gets executed in the free space manager */ - if(sect->sect_info.type != H5HF_FSPACE_SECT_SINGLE) + if((*sect)->sect_info.type != H5HF_FSPACE_SECT_SINGLE) *flags |= H5FS_ADD_RETURNED_SPACE; } /* end if */ @@ -949,10 +949,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5HF_sect_single_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void *_udata) { - H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */ + H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */ H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ @@ -963,26 +963,26 @@ H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_SINGLE); + HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_SINGLE); HDassert(sect2); HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_SINGLE); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ if(H5HF_sect_single_free((H5FS_section_info_t *)sect2) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTRELEASE, FAIL, "can't free section node") /* Check to see if we should revive first section */ - if(sect1->sect_info.state != H5FS_SECT_LIVE) - if(H5HF_sect_single_revive(hdr, dxpl_id, sect1) < 0) + if((*sect1)->sect_info.state != H5FS_SECT_LIVE) + if(H5HF_sect_single_revive(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Check if single section covers entire direct block it's in */ /* (converts to row section possibly) */ - if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect1) < 0) + if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section") done: @@ -1771,10 +1771,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5HF_sect_row_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void *_udata) { - H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */ + H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */ H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ @@ -1785,7 +1785,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); + HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); HDassert(sect2); HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); @@ -1802,8 +1802,8 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, } /* end if */ else { /* Check to see if we should revive first section */ - if(sect1->sect_info.state != H5FS_SECT_LIVE) - if(H5HF_sect_row_revive(hdr, dxpl_id, sect1) < 0) + if((*sect1)->sect_info.state != H5FS_SECT_LIVE) + if(H5HF_sect_row_revive(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Check to see if we should revive second section */ @@ -1812,7 +1812,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Merge rows' underlying indirect sections together */ - if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, sect1, sect2) < 0) + if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, (*sect1), sect2) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTMERGE, FAIL, "can't merge underlying indirect sections") } /* end else */ @@ -29,6 +29,7 @@ /****************/ #define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5FS_FRIEND /*suppress error about including H5Fpkg */ #include "H5MFmodule.h" /* This source code file is part of the H5MF module */ @@ -38,6 +39,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ +#include "H5FSpkg.h" /* File access */ #include "H5Iprivate.h" /* IDs */ #include "H5MFpkg.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -50,11 +52,6 @@ #define H5MF_FSPACE_SHRINK 80 /* Percent of "normal" size to shrink serialized free space size */ #define H5MF_FSPACE_EXPAND 120 /* Percent of "normal" size to expand serialized free space size */ -/* Map an allocation request type to a free list */ -#define H5MF_ALLOC_TO_FS_TYPE(F, T) ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) \ - ? (T) : (F)->shared->fs_type_map[T]) - - /******************/ /* Local Typedefs */ /******************/ @@ -84,11 +81,24 @@ typedef struct { /********************/ /* Allocator routines */ -static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -static herr_t H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -static herr_t H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl); +static haddr_t H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size); + +/* "File closing" routines */ +static herr_t H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id); +static herr_t H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id); static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id); +/* General routines */ +static herr_t H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums); +static hbool_t H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type); +static hbool_t H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace); + +/* Free-space type manager routines */ +static herr_t H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); + /*********************/ /* Package Variables */ @@ -223,10 +233,54 @@ done: /*------------------------------------------------------------------------- - * Function: H5MF__alloc_open + * Function: H5MF_alloc_to_fs_type + * + * Purpose: Map "alloc_type" to the free-space manager type + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Nov 2016 + * + *------------------------------------------------------------------------- + */ +void +H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(fs_type); + + if(H5F_PAGED_AGGR(f)) { /* paged aggregation */ + if(size >= f->shared->fs_page_size) { + if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) { /* multi or split driver */ + /* For non-contiguous address space, map to large size free-space manager for each alloc_type */ + if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[alloc_type]) + *fs_type = (H5F_mem_page_t) (alloc_type + (H5FD_MEM_NTYPES - 1)); + else + *fs_type = (H5F_mem_page_t) (f->shared->fs_type_map[alloc_type] + (H5FD_MEM_NTYPES - 1)); + } /* end if */ + else + /* For contiguous address space, map to generic large size free-space manager */ + *fs_type = H5F_MEM_PAGE_GENERIC; /* H5F_MEM_PAGE_SUPER */ + } /* end if */ + else + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + } /* end if */ + else /* non-paged aggregation */ + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + + FUNC_LEAVE_NOAPI_VOID +} /* end H5MF_alloc_to_fs_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_open_fstype * * Purpose: Open an existing free space manager of TYPE for file by - * creating a free-space structure + * creating a free-space structure. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -238,31 +292,48 @@ done: *------------------------------------------------------------------------- */ herr_t -H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */ - H5MF_FSPACE_SECT_CLS_SIMPLE}; - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5MF_FSPACE_SECT_CLS_SIMPLE, + H5MF_FSPACE_SECT_CLS_SMALL, + H5MF_FSPACE_SECT_CLS_LARGE }; + hsize_t alignment; /* Alignment to use */ + hsize_t threshold; /* Threshold to use */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(H5F_addr_defined(f->shared->fs_addr[type])); HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED); - HDassert(type == H5MF_ALLOC_TO_FS_TYPE(f, type)); + + /* Set up the aligment and threshold to use depending on the manager type */ + if(H5F_PAGED_AGGR(f)) { + alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF; + threshold = H5F_ALIGN_THRHD_DEF; + } /* end if */ + else { + alignment = f->shared->alignment; + threshold = f->shared->threshold; + } /* end else */ /* Set the ring type in the DXPL */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -272,7 +343,7 @@ H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) /* Open an existing free space structure for the file */ if(NULL == (f->shared->fs_man[type] = H5FS_open(f, dxpl_id, f->shared->fs_addr[type], - NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold))) + NELMTS(classes), classes, f, alignment, threshold))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") /* Set the state for the free space manager to "open", if it is now */ @@ -286,14 +357,15 @@ done: HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF__alloc_open() */ +} /* end H5MF_open_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF_alloc_create + * Function: H5MF__create_fstype * * Purpose: Create free space manager of TYPE for the file by creating - * a free-space structure + * a free-space structure + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -305,21 +377,34 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */ - H5MF_FSPACE_SECT_CLS_SIMPLE}; - herr_t ret_value = SUCCEED; /* Return value */ - H5FS_create_t fs_create; /* Free space creation parameters */ + H5MF_FSPACE_SECT_CLS_SIMPLE, + H5MF_FSPACE_SECT_CLS_SMALL, + H5MF_FSPACE_SECT_CLS_LARGE }; + H5FS_create_t fs_create; /* Free space creation parameters */ + hsize_t alignment; /* Alignment to use */ + hsize_t threshold; /* Threshold to use */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED); @@ -330,24 +415,48 @@ H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) fs_create.max_sect_addr = 1 + H5VM_log2_gen((uint64_t)f->shared->maxaddr); fs_create.max_sect_size = f->shared->maxaddr; - if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL, - &fs_create, NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") + /* Set up alignment and threshold to use depending on TYPE */ + if(H5F_PAGED_AGGR(f)) { + alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF; + threshold = H5F_ALIGN_THRHD_DEF; + } /* end if */ + else { + alignment = f->shared->alignment; + threshold = f->shared->threshold; + } /* end else */ + /* Set the ring type in the DXPL */ + if(H5MF__fsm_type_is_self_referential(f, type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + + if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL, + &fs_create, NELMTS(classes), classes, f, alignment, threshold))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") /* Set the state for the free space manager to "open", if it is now */ if(f->shared->fs_man[type]) f->shared->fs_state[type] = H5F_FS_STATE_OPEN; done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_alloc_create() */ +} /* end H5MF__create_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF__alloc_start + * Function: H5MF_start_fstype * - * Purpose: Open or create a free space manager of a given type + * Purpose: Open or create a free space manager of a given TYPE. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -359,40 +468,123 @@ done: *------------------------------------------------------------------------- */ herr_t -H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ /* Check if the free space manager exists already */ if(H5F_addr_defined(f->shared->fs_addr[type])) { /* Open existing free space manager */ - if(H5MF__alloc_open(f, dxpl_id, type) < 0) + if(H5MF_open_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space") } /* end if */ else { /* Create new free space manager */ - if(H5MF_alloc_create(f, dxpl_id, type) < 0) + if(H5MF__create_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCREATE, FAIL, "can't initialize file free space") } /* end else */ done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF__alloc_start() */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_start_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF__alloc_close + * Function: H5MF__delete_fstype * - * Purpose: Close an existing free space manager of TYPE for file + * Purpose: Delete the free-space manager as specified by TYPE. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + /* check args */ + HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert(H5F_addr_defined(f->shared->fs_addr[type])); + + /* Put address into temporary variable and reset it */ + /* (Avoids loopback in file space freeing routine) */ + tmp_fs_addr = f->shared->fs_addr[type]; + f->shared->fs_addr[type] = HADDR_UNDEF; + + /* Shift to "deleting" state, to make certain we don't track any + * file space freed as a result of deleting the free space manager. + */ + f->shared->fs_state[type] = H5F_FS_STATE_DELETING; + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_type_is_self_referential(f, type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Delete free space manager for this type */ + if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager") + + /* Shift [back] to closed state */ + HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + + /* Sanity check that the free space manager for this type wasn't started up again */ + HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__delete_fstype() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__close_fstype + * + * Purpose: Close the free space manager of TYPE for file + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -402,7 +594,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { herr_t ret_value = SUCCEED; /* Return value */ @@ -412,11 +604,18 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(f->shared->fs_man[type]); HDassert(f->shared->fs_state[type] != H5F_FS_STATE_CLOSED); +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Close an existing free space structure for the file */ if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info") @@ -425,7 +624,158 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) done: FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF__alloc_close() */ +} /* end H5MF__close_fstype() */ + + + +/*------------------------------------------------------------------------- + * Function: H5MF_add_sect + * + * Purpose: To add a section to the specified free-space manager. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + H5MF_sect_ud_t udata; /* User data for callback */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ + + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + HDassert(f); + HDassert(fspace); + HDassert(node); + + H5MF_alloc_to_fs_type(f, alloc_type, node->sect_info.size, &fs_type); + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = TRUE; + udata.allow_eoa_shrink_only = FALSE; + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_is_self_referential(f, fspace)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: adding node, node->sect_info.addr = %a, node->sect_info.size = %Hu\n", FUNC, node->sect_info.addr, node->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Add the section */ + if(H5FS_sect_add(f, dxpl_id, fspace, (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + +done: + /* Reset the ring in the DXPL */ + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_add_sect() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_find_sect + * + * Purpose: To find a section from the specified free-space manager to fulfill the request. + * If found, re-add the left-over space back to the manager. + * + * Return: TRUE if a section is found to fulfill the request + * FALSE if not + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +htri_t +H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + H5MF_free_section_t *node; /* Free space section pointer */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FAIL; /* Whether an existing free list node was found */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + HDassert(f); + HDassert(fspace); + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_is_self_referential(f, fspace)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + + /* Try to get a section from the free space manager */ + if((ret_value = H5FS_sect_find(f, dxpl_id, fspace, size, (H5FS_section_info_t **)&node)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "error locating free space in file") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section found = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Check for actually finding section */ + if(ret_value) { + /* Sanity check */ + HDassert(node); + + /* Retrieve return value */ + if(addr) + *addr = node->sect_info.addr; + + /* Check for eliminating the section */ + if(node->sect_info.size == size) { +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: freeing node\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Free section node */ + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + } /* end if */ + else { + /* Adjust information for section */ + node->sect_info.addr += size; + node->sect_info.size -= size; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Re-add the section to the free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, fspace, node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + } /* end else */ + } /* end if */ + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_find_sect() */ /*------------------------------------------------------------------------- @@ -451,7 +801,7 @@ H5MF_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size) H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* free space manager ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ @@ -466,12 +816,20 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(f->shared->lf); HDassert(size > 0); - /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); + if(f->shared->first_alloc_dealloc) { + HDassert(! H5AC_cache_image_pending(f)); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "tidy of self referential fsm hack failed") + } /* end if */ + + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 1.0\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -481,74 +839,46 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ /* Check if we are using the free space manager for this file */ if(H5F_HAVE_FREE_SPACE_MANAGER(f)) { + /* We are about to change the contents of the free space manager -- + * notify metadata cache that the associated fsm ring is + * unsettled + */ + if(H5AC_unsettle_ring(f, fsm_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, HADDR_UNDEF, "attempt to notify cache that ring is unsettled failed") + /* Check if the free space manager for the file has been initialized */ - if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) - if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0) + if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) { + /* Open the free-space manager */ + if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, HADDR_UNDEF, "can't initialize file free space") + HDassert(f->shared->fs_man[fs_type]); + } /* end if */ /* Search for large enough space in the free space manager */ - if(f->shared->fs_man[fs_type]) { - H5MF_free_section_t *node; /* Free space section pointer */ - htri_t node_found = FALSE; /* Whether an existing free list node was found */ - - /* Try to get a section from the free space manager */ - if((node_found = H5FS_sect_find(f, dxpl_id, f->shared->fs_man[fs_type], size, (H5FS_section_info_t **)&node)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating free space in file") -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.5, node_found = %t\n", FUNC, node_found); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* Check for actually finding section */ - if(node_found) { - /* Sanity check */ - HDassert(node); - - /* Retrieve return value */ - ret_value = node->sect_info.addr; - - /* Check for eliminating the section */ - if(node->sect_info.size == size) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.6, freeing node\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - /* Free section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free simple section node") - } /* end if */ - else { - H5MF_sect_ud_t udata; /* User data for callback */ - - /* Adjust information for section */ - node->sect_info.addr += size; - node->sect_info.size -= size; - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + if(f->shared->fs_man[fs_type]) + if(H5MF_find_sect(f, alloc_type, dxpl_id, size, f->shared->fs_man[fs_type], &ret_value) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating a node") + } /* end if */ + /* If no space is found from the free-space manager, continue further action */ + if(!H5F_addr_defined(ret_value)) { #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.7, re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size); +HDfprintf(stderr, "%s: Check 2.0\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ - /* Re-insert section node into file's free space */ - if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") - } /* end else */ - - /* Leave now */ - HGOTO_DONE(ret_value) - } /* end if */ + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) { + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + if(HADDR_UNDEF == (ret_value = H5MF__alloc_pagefs(f, alloc_type, dxpl_id, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from paged aggregation") } /* end if */ + else { /* For non-paged aggregation, continue further action */ + if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd") + } /* end else */ + } /* end if */ + HDassert(H5F_addr_defined(ret_value)); #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 2.0\n", FUNC); +HDfprintf(stderr, "%s: Check 3.0\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ - } /* end if */ - - /* Allocate from the metadata aggregator (or the VFD) */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd") done: /* Reset the ring in the DXPL */ @@ -568,6 +898,146 @@ H5MF_sects_dump(f, dxpl_id, stderr); /*------------------------------------------------------------------------- + * Function: H5MF__alloc_pagefs + * + * Purpose: Allocate space from either the large or small free-space manager. + * For "large" request: + * Allocate request from VFD + * Determine mis-aligned fragment and return the fragment to the + * appropriate manager + * For "small" request: + * Allocate a page from the large manager + * Determine whether space is available from a mis-aligned fragment + * being returned to the manager + * Return left-over space to the manager after fulfilling request + * + * Return: Success: The file address of new chunk. + * Failure: HADDR_UNDEF + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static haddr_t +H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size) +{ + H5F_mem_page_t ptype; /* Free-space mananger type */ + H5MF_free_section_t *node = NULL; /* Free space section pointer */ + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, HADDR_UNDEF) + +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); +#endif /* H5MF_ALLOC_DEBUG */ + + H5MF_alloc_to_fs_type(f, alloc_type, size, &ptype); + + switch(ptype) { + case H5F_MEM_PAGE_GENERIC: + case H5F_MEM_PAGE_LARGE_BTREE: + case H5F_MEM_PAGE_LARGE_DRAW: + case H5F_MEM_PAGE_LARGE_GHEAP: + case H5F_MEM_PAGE_LARGE_LHEAP: + case H5F_MEM_PAGE_LARGE_OHDR: + { + haddr_t eoa; /* EOA for the file */ + hsize_t frag_size = 0; /* Fragment size */ + + /* Get the EOA for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") + HDassert(!(eoa % f->shared->fs_page_size)); + + H5MF_EOA_MISALIGN(f, (eoa+size), f->shared->fs_page_size, frag_size); + + /* Allocate from VFD */ + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size + frag_size, NULL, NULL))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") + + /* If there is a mis-aligned fragment at EOA */ + if(frag_size) { + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[ptype])) + if(H5MF_start_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space") + + /* Create free space section for the fragment */ + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, ret_value + size, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section") + + /* Add the fragment to the large free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") + + node = NULL; + } /* end if */ + } + break; + + case H5F_MEM_PAGE_META: + case H5F_MEM_PAGE_DRAW: + case H5F_MEM_PAGE_BTREE: + case H5F_MEM_PAGE_GHEAP: + case H5F_MEM_PAGE_LHEAP: + case H5F_MEM_PAGE_OHDR: + { + haddr_t new_page; /* The address for the new file size page */ + + /* Allocate one file space page */ + if(HADDR_UNDEF == (new_page = H5MF_alloc(f, alloc_type, dxpl_id, f->shared->fs_page_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[ptype])) + if(H5MF_start_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space") + HDassert(f->shared->fs_man[ptype]); + + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_SMALL, (new_page + size), (f->shared->fs_page_size - size)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section") + + /* Add the remaining space in the page to the manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") + + node = NULL; + + /* Insert the new page into the Page Buffer list of new pages so + we don't read an empty page from disk */ + if(f->shared->page_buf != NULL && H5PB_add_new_page(f, alloc_type, new_page) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't add new page to Page Buffer new page list") + + ret_value = new_page; + } + break; + + case H5F_MEM_PAGE_NTYPES: + case H5F_MEM_PAGE_DEFAULT: + default: + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space: unrecognized type") + break; + } /* end switch */ + +done: +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); +#endif /* H5MF_ALLOC_DEBUG */ +#ifdef H5MF_ALLOC_DEBUG_DUMP +H5MF_sects_dump(f, dxpl_id, stderr); +#endif /* H5MF_ALLOC_DEBUG_DUMP */ + + /* Release section node, if allocated and not added to section list or merged */ + if(node) + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free section node") + + FUNC_LEAVE_NOAPI_TAG(ret_value, HADDR_UNDEF) +} /* end H5MF__alloc_pagefs() */ + + +/*------------------------------------------------------------------------- * Function: H5MF_alloc_tmp * * Purpose: Allocate temporary space in the file @@ -645,14 +1115,14 @@ herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, hsize_t size) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ H5MF_free_section_t *node = NULL; /* Free space section pointer */ - H5MF_sect_ud_t udata; /* User data for callback */ + unsigned ctype; /* section class type */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ - H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -666,19 +1136,16 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN HGOTO_DONE(SUCCEED) HDassert(addr != 0); /* Can't deallocate the superblock :-) */ - /* Check for attempting to free space that's a 'temporary' file address */ - if(H5F_addr_le(f->shared->tmp_addr, addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space") + if(f->shared->first_alloc_dealloc) { + HDassert(!H5AC_cache_image_pending(f)); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end if */ - /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); -#endif /* H5MF_ALLOC_DEBUG_MORE */ + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -686,10 +1153,33 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; + /* we are about to change the contents of the free space manager -- + * notify metadata cache that the associated fsm ring is + * unsettled + */ + /* Only do so for strategies that use free-space managers */ + if(H5F_HAVE_FREE_SPACE_MANAGER(f)) + if(H5AC_unsettle_ring(f, fsm_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, FAIL, "attempt to notify cache that ring is unsettled failed") + + /* Check for attempting to free space that's a 'temporary' file address */ + if(H5F_addr_le(f->shared->tmp_addr, addr)) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space") + /* Set up I/O info for operation */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(H5FD_MEM_DRAW == alloc_type) { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ /* Check if the space to free intersects with the file's metadata accumulator */ if(H5F__accum_free(&fio_info, alloc_type, addr, size) < 0) @@ -702,7 +1192,7 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); * space is at the end of the file */ #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)fs_type, f->shared->fs_addr[fs_type]); +HDfprintf(stderr, "%s: fs_addr = %a\n", FUNC, f->shared->fs_addr[fs_type]); #endif /* H5MF_ALLOC_DEBUG_MORE */ if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) { htri_t status; /* "can absorb" status for section into */ @@ -726,8 +1216,8 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a /* If we are deleting the free space manager, leave now, to avoid * [re-]starting it. - * or if file space strategy type is not using a free space manager - * (H5F_FILE_SPACE_AGGR_VFD or H5F_FILE_SPACE_VFD), drop free space + * or if file space strategy type is not using a free space manager + * (H5F_FSPACE_STRATEGY_AGGR or H5F_FSPACE_STRATEGY_NONE), drop free space * section on the floor. * * Note: this drops the space to free on the floor... @@ -745,38 +1235,42 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a * space isn't at the end of the file, so start up (or create) * the file space manager */ - if(H5MF__alloc_start(f, dxpl_id, fs_type) < 0) + if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") } /* end if */ - /* Create free space section for block */ - if(NULL == (node = H5MF_sect_simple_new(addr, size))) + /* Create the free-space section for the freed section */ + ctype = H5MF_SECT_CLASS_TYPE(f, size); + if(NULL == (node = H5MF_sect_new(ctype, addr, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; - - /* If size of section freed is larger than threshold, add it to the free space manager */ + /* If size of the freed section is larger than threshold, add it to the free space manager */ if(size >= f->shared->fs_threshold) { HDassert(f->shared->fs_man[fs_type]); #ifdef H5MF_ALLOC_DEBUG_MORE HDfprintf(stderr, "%s: Before H5FS_sect_add()\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Add to the free space for the file */ - if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space") - node = NULL; + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space") + node = NULL; + #ifdef H5MF_ALLOC_DEBUG_MORE HDfprintf(stderr, "%s: After H5FS_sect_add()\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ else { htri_t merged; /* Whether node was merged */ + H5MF_sect_ud_t udata; /* User data for callback */ + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = TRUE; + udata.allow_eoa_shrink_only = FALSE; /* Try to merge the section that is smaller than threshold */ if((merged = H5FS_sect_try_merge(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata)) < 0) @@ -794,7 +1288,7 @@ done: /* Release section node, if allocated and not added to section list or merged */ if(node) - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") #ifdef H5MF_ALLOC_DEBUG @@ -811,6 +1305,14 @@ H5MF_sects_dump(f, dxpl_id, stderr); * Function: H5MF_try_extend * * Purpose: Extend a block in the file if possible. + * For non-paged aggregation: + * --try to extend at EOA + * --try to extend into the aggregators + * --try to extend into a free-space section if adjoined + * For paged aggregation: + * --try to extend at EOA + * --try to extend into a free-space section if adjoined + * --try to extend into the page end threshold if a metadata block * * Return: Success: TRUE(1) - Block was extended * FALSE(0) - Block could not be extended @@ -825,14 +1327,16 @@ htri_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsize_t extra_requested) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - haddr_t end; /* End of block to extend */ - H5FD_mem_t fs_type; /* Memory type of the free space manager */ - H5FD_mem_t map_type; /* Mapped type */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - htri_t ret_value = FAIL; /* Return value */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + haddr_t end; /* End of block to extend */ + H5FD_mem_t map_type; /* Mapped type */ + H5F_mem_page_t fs_type; /* free space type */ + htri_t allow_extend = TRUE; /* Possible to extend the block */ + hsize_t frag_size = 0; /* Size of mis-aligned fragment */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG @@ -849,12 +1353,33 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r /* Compute end of block to extend */ end = addr + size; + /* For paged aggregation: + * To extend a small block: can only extend if not crossing page boundary + * To extend a large block at EOA: calculate in advance mis-aligned fragment so EOA will still end at page boundary + */ + if(H5F_PAGED_AGGR(f)) { + if(size < f->shared->fs_page_size) { + /* To extend a small block: cannot cross page boundary */ + if((addr / f->shared->fs_page_size) != (((end + extra_requested) - 1) / f->shared->fs_page_size)) + allow_extend = FALSE; + } /* end if */ + else { + haddr_t eoa; /* EOA for the file */ + + /* To extend a large block: calculate in advance the mis-aligned fragment so EOA will end at page boundary if extended */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") + HDassert(!(eoa % f->shared->fs_page_size)); + + H5MF_EOA_MISALIGN(f, (eoa+extra_requested), f->shared->fs_page_size, frag_size); + } /* end else */ + } /* end if */ + /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -862,29 +1387,90 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - /* Check if the block is exactly at the end of the file */ - if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, dxpl_id, end, extra_requested)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") - else if(ret_value == FALSE) { - H5F_blk_aggr_t *aggr; /* Aggregator to use */ + if(allow_extend) { + /* Try extending the block at EOA */ + if((ret_value = H5F_try_extend(f, dxpl_id, map_type, end, extra_requested + frag_size)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: extended = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If extending at EOA succeeds: */ + /* for paged aggregation, put the fragment into the large-sized free-space manager */ + if(ret_value == TRUE && H5F_PAGED_AGGR(f) && frag_size) { + H5MF_free_section_t *node = NULL; /* Free space section pointer */ - /* Check for test block able to extend aggregation block */ - aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); - if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") - else if(ret_value == FALSE) { + /* Should be large-sized block */ + HDassert(size >= f->shared->fs_page_size); + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[fs_type])) + if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + + /* Create free space section for the fragment */ + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, end + extra_requested, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") + + /* Add the fragment to the large-sized free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + + node = NULL; + } /* end if */ + + /* For non-paged aggregation: try to extend into the aggregators */ + if(ret_value == FALSE && (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || + f->shared->fs_strategy == H5F_FSPACE_STRATEGY_AGGR) ) { + H5F_blk_aggr_t *aggr; /* Aggregator to use */ + + /* Check if the block is able to extend into aggregation block */ + aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); + if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: H5MF_aggr_try_extend = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + + /* If no extension so far, try to extend into a free-space section */ + if(ret_value == FALSE && ((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) || + (H5F_PAGED_AGGR(f))) ) { + H5MF_sect_ud_t udata; /* User data */ + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; /* Check if the free space for the file has been initialized */ if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) - if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0) + /* Open the free-space manager */ + if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") - /* Check for test block able to block in free space manager */ - if(f->shared->fs_man[fs_type]) - if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested)) < 0) + /* Try to extend the block into a free-space section */ + if(f->shared->fs_man[fs_type]) { + if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested, H5FS_ADD_RETURNED_SPACE, &udata)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending block in free space manager") +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Try to H5FS_sect_try_extend = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + + /* For paged aggregation and a metadata block: try to extend into page end threshold */ + if(ret_value == FALSE && H5F_PAGED_AGGR(f) && map_type != H5FD_MEM_DRAW) { + H5MF_EOA_MISALIGN(f, end, f->shared->fs_page_size, frag_size); + + if(frag_size <= H5F_PGEND_META_THRES(f) && extra_requested <= frag_size) + ret_value = TRUE; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Try to extend into the page end threshold = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ } /* end if */ - } /* end else-if */ + } /* allow_extend */ done: /* Reset the ring in the DXPL */ @@ -904,174 +1490,316 @@ H5MF_sects_dump(f, dxpl_id, stderr); /*------------------------------------------------------------------------- - * Function: H5MF_get_freespace + * Function: H5MF_try_shrink * - * Purpose: Retrieve the amount of free space in a file. + * Purpose: Try to shrink the size of a file with a block or absorb it + * into a block aggregator. * - * Return: Success: Amount of free space in file - * Failure: Negative + * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * Monday, October 6, 2003 + * koziol@hdfgroup.org + * Feb 14 2008 * - * Modifications: - * Vailin Choi; July 2012 - * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY, - * checks are added to account for the last section of each free-space manager - * and the remaining space in the two aggregators are at EOF. *------------------------------------------------------------------------- */ -herr_t -H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size) +htri_t +H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, + hsize_t size) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t needed_ring; /* Ring value needed for loop iteration */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - haddr_t eoa; /* End of allocated space in the file */ - haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ - hsize_t ma_size = 0; /* Size of "metadata aggregator" */ - haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ - hsize_t sda_size = 0; /* Size of "small data aggregator" */ - hsize_t tot_fs_size = 0; /* Amount of all free space managed */ - hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */ - H5FD_mem_t type; /* Memory type for iteration */ - hbool_t fs_started[H5FD_MEM_NTYPES]; /* Indicate whether the free-space manager has been started */ - hbool_t eoa_shrank; /* Whether an EOA shrink occurs */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5MF_free_section_t *node = NULL; /* Free space section pointer */ + H5MF_sect_ud_t udata; /* User data for callback */ + H5FS_section_class_t *sect_cls; /* Section class */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ + H5F_mem_page_t fs_type; /* Free space type */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size); +#endif /* H5MF_ALLOC_DEBUG */ - /* check args */ + /* check arguments */ HDassert(f); HDassert(f->shared); HDassert(f->shared->lf); + HDassert(H5F_addr_defined(addr)); + HDassert(size > 0); - /* Retrieve the 'eoa' for the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_DEFAULT))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + /* Set up free-space section class information */ + sect_cls = H5MF_SECT_CLS_TYPE(f, size); + HDassert(sect_cls); + + /* Get free space type from allocation type */ + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - curr_ring = H5AC_RING_RDFSM; - /* Retrieve metadata aggregator info, if available */ - if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") + /* Create free-space section for block */ + if(NULL == (node = H5MF_sect_new(sect_cls->type, addr, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") - /* Retrieve 'small data' aggregator info, if available */ - if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */ + udata.allow_eoa_shrink_only = FALSE; - /* Iterate over all the free space types that have managers and get each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + /* Check if the block can shrink the container */ + if(sect_cls->can_shrink) { + if((ret_value = (*sect_cls->can_shrink)((const H5FS_section_info_t *)node, &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container") + if(ret_value > 0) { + HDassert(sect_cls->shrink); - fs_started[type] = FALSE; + if((*sect_cls->shrink)((H5FS_section_info_t **)&node, &udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container") + } /* end if */ + } /* end if */ - /* test to see if we need to switch rings -- do so if required */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; - } /* end if */ + /* Free section node allocated */ + if(node && H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") - /* Check if the free space for the file has been initialized */ - if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) { - if(H5MF__alloc_open(f, dxpl_id, type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") - HDassert(f->shared->fs_man[type]); - fs_started[type] = TRUE; - } /* end if */ +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_try_shrink() */ - /* Check if there's free space of this type */ - if(f->shared->fs_man[type]) { - hsize_t type_fs_size = 0; /* Amount of free space managed for each type */ - hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */ + +/*------------------------------------------------------------------------- + * Function: H5MF_close + * + * Purpose: Close the free space tracker(s) for a file: + * paged or non-paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_close(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ - /* Retrieve free space size from free space manager */ - if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") - if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats") + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ - /* Increment total free space for types */ - tot_fs_size += type_fs_size; - tot_meta_size += type_meta_size; - } /* end if */ - } /* end for */ + /* check args */ + HDassert(f); + HDassert(f->shared); - /* Iterate until no more EOA shrink occurs */ - do { - eoa_shrank = FALSE; + if(H5F_PAGED_AGGR(f)) { + if((ret_value = H5MF__close_pagefs(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'page' file space") + } /* end if */ + else { + if((ret_value = H5MF__close_aggrfs(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'aggr' file space") + } /* end else */ - /* Check the last section of each free-space manager */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - haddr_t sect_addr = HADDR_UNDEF; - hsize_t sect_size = 0; - - if(f->shared->fs_man[type]) { - if(H5FS_sect_query_last_sect(f->shared->fs_man[type], §_addr, §_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query last section on merge list") - - /* Deduct space from previous accumulation if the section is at EOA */ - if(H5F_addr_eq(sect_addr + sect_size, eoa)) { - eoa = sect_addr; - eoa_shrank = TRUE; - tot_fs_size -= sect_size; - } /* end if */ - } /* end if */ - } /* end for */ - - /* Check the metadata and raw data aggregators */ - if(ma_size > 0 && H5F_addr_eq(ma_addr + ma_size, eoa)) { - eoa = ma_addr; - eoa_shrank = TRUE; - ma_size = 0; - } /* end if */ - if(sda_size > 0 && H5F_addr_eq(sda_addr + sda_size, eoa)) { - eoa = sda_addr; - eoa_shrank = TRUE; - sda_size = 0; - } /* end if */ - } while(eoa_shrank); +done: - /* Close the free-space managers if they were opened earlier in this routine */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - if(fs_started[type]) { +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__close_delete_fstype + * + * Purpose: Common code for closing and deleting the freespace manager + * of TYPE for file. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi + * Jan 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + HDassert(f->shared); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If the free space manager for this type is open, close it */ + if(f->shared->fs_man[type]) + if(H5MF__close_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If there is free space manager info for this type, delete it */ + if(H5F_addr_defined(f->shared->fs_addr[type])) + if(H5MF__delete_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't delete the free space manager") + +done: +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* H5MF__close_delete() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_try_close + * + * Purpose: This is called by H5Fformat_convert() to close and delete + * free-space managers when downgrading persistent free-space + * to non-persistent. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi + * Jan 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_try_close(H5F_t *f, hid_t dxpl_id) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + + /* If there have been no file space allocations / deallocation so + * far, must call H5MF_tidy_self_referential_fsm_hack() to float + * all self referential FSMs and release file space allocated to + * them. Otherwise, the function will be called after the format + * conversion, and will become very confused. + * + * The situation is further complicated if a cache image exists + * and had not yet been loaded into the metadata cache. In this + * case, call H5AC_force_cache_image_load() instead of + * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load() + * will load the cache image, and then call + * H5MF_tidy_self_referential_fsm_hack() to discard the cache image + * block. + */ + if(f->shared->first_alloc_dealloc) { + if(H5AC_cache_image_pending(f)) { + if(H5AC_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed") + } /* end if */ + else { + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end else */ + } /* end if */ + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + if(H5F_PAGED_AGGR(f)) { + H5F_mem_page_t ptype; /* Memory type for iteration */ + /* Iterate over all the free space types that have managers and + * get each free list's space + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring ) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + } /* end for */ + } /* end if */ + else { + H5FD_mem_t type; /* Memory type for iteration */ + + /* Iterate over all the free space types that have managers and + * get each free list's space + */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { /* test to see if we need to switch rings -- do so if required */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; if(needed_ring != curr_ring) { if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)") + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") curr_ring = needed_ring; } /* end if */ - if(H5MF__alloc_close(f, dxpl_id, type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space") - } /* end if */ - } /* end for */ - - /* Set the value(s) to return */ - /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */ - if(tot_space) - *tot_space = tot_fs_size + ma_size + sda_size; - if(meta_size) - *meta_size = tot_meta_size; + if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + } /* end for */ + } /* end else */ done: /* Reset the ring in the DXPL */ @@ -1079,75 +1807,370 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_get_freespace() */ +} /* H5MF_try_close() */ /*------------------------------------------------------------------------- - * Function: H5MF_try_shrink + * Function: H5MF__close_aggrfs * - * Purpose: Try to shrink the size of a file with a block or absorb it - * into a block aggregator. + * Purpose: Close the free space tracker(s) for a file: non-paged aggregation * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Feb 14 2008 + * Tuesday, January 22, 2008 * *------------------------------------------------------------------------- */ -htri_t -H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, - hsize_t size) +static herr_t +H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id) { - H5MF_free_section_t *node = NULL; /* Free space section pointer */ - H5MF_sect_ud_t udata; /* User data for callback */ - htri_t ret_value = FAIL; /* Return value */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5FD_mem_t type; /* Memory type for iteration */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size); +HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - /* check arguments */ + /* check args */ HDassert(f); HDassert(f->shared); HDassert(f->shared->lf); - HDassert(H5F_addr_defined(addr)); - HDassert(size > 0); + HDassert(f->shared->sblock); - /* Create free space section for block */ - if(NULL == (node = H5MF_sect_simple_new(addr, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */ - udata.allow_eoa_shrink_only = FALSE; - - /* Call the "can shrink" callback for the section */ - if((ret_value = H5MF_sect_simple_can_shrink((const H5FS_section_info_t *)node, &udata)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container") - else if(ret_value > 0) { - /* Shrink or absorb the section */ - if(H5MF_sect_simple_shrink((H5FS_section_info_t **)&node, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container") + /* Free the space in aggregators */ + /* (for space not at EOA, it may be put into free space managers) */ + if(H5MF_free_aggrs(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + + /* Trying shrinking the EOA for the file */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* Making free-space managers persistent for superblock version >= 2 */ + if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 + && f->shared->fs_persist) { + H5O_fsinfo_t fsinfo; /* File space info message */ + haddr_t final_eoa; /* Final eoa -- for sanity check */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + + /* superblock extension and free space manager message should + * exist at this point -- verify at least the former. + */ + HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + + /* file space for all non-empty free space managers should be + * allocated at this point, and these free space managers should + * be written to file and thus their headers and section info + * entries in the metadata cache should be clean. + */ + + /* gather data for the free space manager superblock extension message. + * + * In passing, verify that all the free space managers are closed. + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + fsinfo.fs_addr[type-1] = f->shared->fs_addr[type]; + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + + /* Write the free space manager message -- message must already exist */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + + /* Close the free space managers */ + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(f->shared->fs_man[type]) { + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN); + + if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") + f->shared->fs_man[type] = NULL; + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + } /* end if */ + f->shared->fs_addr[type] = HADDR_UNDEF; + } /* end for */ + + /* verify that we haven't dirtied any metadata cache entries + * from the metadata free space manager ring out. + */ + HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + + /* verify that the aggregators are still shutdown. */ + HDassert(f->shared->sdata_aggr.tot_size == 0); + HDassert(f->shared->sdata_aggr.addr == 0); + HDassert(f->shared->sdata_aggr.size == 0); + + HDassert(f->shared->meta_aggr.tot_size == 0); + HDassert(f->shared->meta_aggr.addr == 0); + HDassert(f->shared->meta_aggr.size == 0); + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* get the eoa, and verify that it has the expected value */ + if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* f->shared->eoa_post_fsm_fsalloc is undefined if there has + * been no file space allocation or deallocation since file + * open. + */ + HDassert((f->shared->first_alloc_dealloc) || (final_eoa == f->shared->eoa_post_fsm_fsalloc)); } /* end if */ + else { /* super_vers can be 0, 1, 2 */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + } /* end else */ + + /* Free the space in aggregators (again) */ + /* (in case any free space information re-started them) */ + if(H5MF_free_aggrs(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - /* Free section node allocated */ - if(node && H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__close_aggrfs() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__close_pagefs + * + * Purpose: Close the free space tracker(s) for a file: paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this + * iteration. + */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + H5O_fsinfo_t fsinfo; /* File space info message */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); +HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_try_shrink() */ + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(f->shared->sblock); + HDassert(f->shared->fs_page_size); + HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2); + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Trying shrinking the EOA for the file */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* Set up file space info message */ + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + + if(f->shared->fs_persist) { + haddr_t final_eoa; /* final eoa -- for sanity check */ + + /* superblock extension and free space manager message should + * exist at this point -- verify at least the former. + */ + HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + + /* file space for all non-empty free space managers should be + * allocated at this point, and these free space managers should + * be written to file and thus their headers and section info + * entries in the metadata cache should be clean. + */ + + /* gather data for the free space manager superblock extension message. + * Only need addresses of FSMs and eoa prior to allocation of + * file space for the self referential free space managers. Other + * data was gathered above. + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype-1] = f->shared->fs_addr[ptype]; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + + /* Write the free space manager message -- message must already exist */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + + /* Close the free space managers */ + /* use H5MF__close_fstype() for this? */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + if(f->shared->fs_man[ptype]) { + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + HDassert(f->shared->fs_state[ptype] == H5F_FS_STATE_OPEN); + + if(H5FS_close(f, dxpl_id, f->shared->fs_man[ptype]) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") + f->shared->fs_man[ptype] = NULL; + f->shared->fs_state[ptype] = H5F_FS_STATE_CLOSED; + } /* end if */ + f->shared->fs_addr[ptype] = HADDR_UNDEF; + } /* end for */ + + /* verify that we haven't dirtied any metadata cache entries + * from the metadata free space manager ring out. + */ + HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* get the eoa, and verify that it has the expected value */ + if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* f->shared->eoa_post_fsm_fsalloc is undefined if there has + * been no file space allocation or deallocation since file + * open. + * + * If there is a cache image in the file at file open, + * f->shared->first_alloc_dealloc will always be FALSE unless + * the file is opened R/O, as otherwise, the image will have been + * read and discarded by this point. + * + * If a cache image was created on file close, the actual EOA + * should be in f->shared->eoa_post_mdci_fsalloc. Note that in + * this case, it is conceivable that f->shared->first_alloc_dealloc + * will still be TRUE, as the cache image is allocated directly from + * the file driver layer. However, as this possibility seems remote, + * it is ignored in the following assert. + */ + HDassert((f->shared->first_alloc_dealloc) || + (final_eoa == f->shared->eoa_post_fsm_fsalloc) || + ((H5F_addr_defined(f->shared->eoa_post_mdci_fsalloc)) && + (final_eoa == f->shared->eoa_post_mdci_fsalloc))); + } /* end if */ + else { + /* Iterate over all the free space types that have managers + * and get each free list's space + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + + /* Write file space info message to superblock extension object header */ + /* Create the superblock extension object header in advance if needed */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + } /* end else */ + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__close_pagefs() */ /*------------------------------------------------------------------------- @@ -1165,17 +2188,16 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) { - H5FD_mem_t type; /* Memory type for iteration */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ - H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this - * iteration. - */ + H5F_mem_t type; + H5F_mem_page_t ptype; /* Memory type for iteration */ hbool_t eoa_shrank; /* Whether an EOA shrink occurs */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ htri_t status; /* Status value */ H5MF_sect_ud_t udata; /* User data for callback */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -1187,13 +2209,10 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) /* Construct user data for callbacks */ udata.f = f; udata.dxpl_id = dxpl_id; - udata.allow_sect_absorb = FALSE; - udata.allow_eoa_shrink_only = TRUE; + udata.allow_sect_absorb = FALSE; + udata.allow_eoa_shrink_only = TRUE; - /* Set the ring type in the DXPL. In most cases, we will - * need H5AC_RING_RDFSM, so initialy set the ring type in - * the DXPL to that value. We will alter this later if needed. - */ + /* Set the ring type in the DXPL */ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(1)") reset_ring = TRUE; @@ -1203,36 +2222,62 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) do { eoa_shrank = FALSE; - /* Check the last section of each free-space manager */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(H5F_PAGED_AGGR(f)) { + /* Check the last section of each free-space manager */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + if(f->shared->fs_man[ptype]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; - } /* end if */ + udata.alloc_type = (H5FD_mem_t)((H5FD_mem_t)ptype < H5FD_MEM_NTYPES ? ptype : ((ptype % H5FD_MEM_NTYPES) + 1)); - if(f->shared->fs_man[type]) { - udata.alloc_type = type; - if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") - else if(status > 0) - eoa_shrank = TRUE; - } /* end if */ - } /* end for */ + if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[ptype], &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end if */ + } /* end for */ + } /* end if */ + else { + /* Check the last section of each free-space manager */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(f->shared->fs_man[type]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ - /* check the two aggregators */ - if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") - else if(status > 0) - eoa_shrank = TRUE; + udata.alloc_type = type; + + if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end if */ + } /* end for */ + + /* check the two aggregators */ + if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end else */ } while(eoa_shrank); done: @@ -1246,6 +2291,370 @@ done: /*------------------------------------------------------------------------- + * Function: H5MF_get_freespace + * + * Purpose: Retrieve the amount of free space in the file + * + * Return: Success: Amount of free space in file + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, October 6, 2003 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size) +{ + haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ + hsize_t ma_size = 0; /* Size of "metadata aggregator" */ + haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ + hsize_t sda_size = 0; /* Size of "small data aggregator" */ + hsize_t tot_fs_size = 0; /* Amount of all free space managed */ + hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */ + H5FD_mem_t tt; /* Memory type for iteration */ + H5F_mem_page_t type; /* Memory type for iteration */ + H5F_mem_page_t start_type; /* Memory type for iteration */ + H5F_mem_page_t end_type; /* Memory type for iteration */ + htri_t fs_started[H5F_MEM_PAGE_NTYPES]; /* Indicate whether the free-space manager has been started */ + haddr_t fs_eoa[H5FD_MEM_NTYPES]; /* EAO for each free-space manager */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Determine start/end points for loop */ + if(H5F_PAGED_AGGR(f)) { + start_type = H5F_MEM_PAGE_META; + end_type = H5F_MEM_PAGE_NTYPES; + } /* end if */ + else { + start_type = (H5F_mem_page_t)H5FD_MEM_SUPER; + end_type = (H5F_mem_page_t)H5FD_MEM_NTYPES; + } /* end else */ + + for(tt = H5FD_MEM_SUPER; tt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, tt)) + if(HADDR_UNDEF == (fs_eoa[tt] = H5F_get_eoa(f, tt))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + if(!H5F_PAGED_AGGR(f)) { + /* Retrieve metadata aggregator info, if available */ + if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") + + /* Retrieve 'small data' aggregator info, if available */ + if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") + } /* end if */ + + /* Iterate over all the free space types that have managers and get each free list's space */ + for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) { + fs_started[type] = FALSE; + + /* Check if the free space for the file has been initialized */ + if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) { + if(H5MF_open_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + HDassert(f->shared->fs_man[type]); + fs_started[type] = TRUE; + } /* end if */ + + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ + + /* Check if there's free space of this type */ + if(f->shared->fs_man[type]) { + hsize_t type_fs_size = 0; /* Amount of free space managed for each type */ + hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */ + + /* Retrieve free space size from free space manager */ + if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") + if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats") + + /* Increment total free space for types */ + tot_fs_size += type_fs_size; + tot_meta_size += type_meta_size; + } /* end if */ + } /* end for */ + + /* Close the free-space managers if they were opened earlier in this routine */ + for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ + + if(fs_started[type]) + if(H5MF__close_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space") + } /* end for */ + + /* Set the value(s) to return */ + /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */ + if(tot_space) + *tot_space = tot_fs_size + ma_size + sda_size; + if(meta_size) + *meta_size = tot_meta_size; + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_get_freespace() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_get_free_sections() + * + * Purpose: To retrieve free-space section information for + * paged or non-paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +ssize_t +H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + size_t total_sects = 0; /* Total number of sections */ + H5MF_sect_iter_ud_t sect_udata; /* User data for callback */ + H5F_mem_page_t start_type, end_type; /* Memory types to iterate over */ + H5F_mem_page_t ty; /* Memory type for iteration */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + ssize_t ret_value = -1; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, -1) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + + /* H5MF_tidy_self_referential_fsm_hack() will fail if any self + * referential FSM is opened prior to the call to it. Thus call + * it here if necessary and if it hasn't been called already. + * + * The situation is further complicated if a cache image exists + * and had not yet been loaded into the metadata cache. In this + * case, call H5AC_force_cache_image_load() instead of + * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load() + * will load the cache image, and then call + * H5MF_tidy_self_referential_fsm_hack() to discard the cache image + * block. + */ + if(f->shared->first_alloc_dealloc) { + if(H5AC_cache_image_pending(f)) { + if(H5AC_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed") + } /* end if */ + else { + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end else */ + } /* end if */ + + if(type == H5FD_MEM_DEFAULT) { + start_type = H5F_MEM_PAGE_SUPER; + end_type = H5F_MEM_PAGE_NTYPES; + } /* end if */ + else { + start_type = end_type = (H5F_mem_page_t)type; + if(H5F_PAGED_AGGR(f)) /* set to the corresponding LARGE free-space manager */ + end_type = (H5F_mem_page_t)(end_type + H5FD_MEM_NTYPES); + else + H5_INC_ENUM(H5F_mem_page_t, end_type); + } /* end else */ + + /* Set up user data for section iteration */ + sect_udata.sects = sect_info; + sect_udata.sect_count = nsects; + sect_udata.sect_idx = 0; + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(0)") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Iterate over memory types, retrieving the number of sections of each type */ + for(ty = start_type; ty < end_type; H5_INC_ENUM(H5F_mem_page_t, ty)) { + hbool_t fs_started = FALSE; /* The free-space manager is opened or not */ + size_t nums = 0; /* The number of free-space sections */ + + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ty)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) { + if(H5MF_open_fstype(f, dxpl_id, ty) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't open the free space manager") + HDassert(f->shared->fs_man[ty]); + fs_started = TRUE; + } /* end if */ + + /* Check if there's free space sections of this type */ + if(f->shared->fs_man[ty]) + if(H5MF__get_free_sects(f, dxpl_id, f->shared->fs_man[ty], §_udata, &nums) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get section info for the free space manager") + + /* Increment total # of sections */ + total_sects += nums; + + /* Close the free space manager of this type, if we started it here */ + if(fs_started) + if(H5MF__close_fstype(f, dxpl_id, ty) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space") + if((H5F_PAGED_AGGR(f)) && (type != H5FD_MEM_DEFAULT)) + ty = (H5F_mem_page_t)(ty + H5FD_MEM_NTYPES - 2); + } /* end for */ + + /* Set return value */ + ret_value = (ssize_t)total_sects; + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, -1) +} /* H5MF_get_free_sections() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sects_cb() + * + * Purpose: Iterator callback for each free-space section + * Retrieve address and size into user data + * + * Return: Always succeed + * + * Programmer: Vailin Choi + * July 1st, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata) +{ + H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; + H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(udata->sect_idx < udata->sect_count) { + udata->sects[udata->sect_idx].addr = sect->sect_info.addr; + udata->sects[udata->sect_idx].size = sect->sect_info.size; + udata->sect_idx++; + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sects_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__get_free_sects + * + * Purpose: Retrieve section information for the specified free-space manager. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums) +{ + hsize_t hnums = 0; /* # of sections */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(f); + HDassert(sect_udata); + HDassert(nums); + HDassert(fspace); + + /* Query how many sections of this type */ + if(H5FS_sect_stats(fspace, NULL, &hnums) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") + H5_CHECKED_ASSIGN(*nums, size_t, hnums, hsize_t); + + /* Check if we should retrieve the section info */ + if(sect_udata->sects && *nums > 0) + /* Iterate over all the free space sections of this type, adding them to the user's section info */ + if(H5FS_sect_iterate(f, dxpl_id, fspace, H5MF_sects_cb, sect_udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF__get_free_sects() */ + + +/*------------------------------------------------------------------------- * Function: H5MF_settle_raw_data_fsm() * * Purpose: Handle any tasks required before the metadata cache @@ -1253,19 +2662,48 @@ done: * and any metadata free space managers that reside in the * raw data free space manager ring. * - * Specifically, any metadata managers that DON'T handle - * space allocation for free space manager header or section - * info will reside in the raw data free space manager ring. - * As of this writing, the plan is to move to only two free space - * managers, one for raw data and one for metadata -- which - * means that only the raw data free space manager will reside - * in the free space manager ring. However, this has not been - * fully implemented yet, so this code must support the - * possibilty of multiple metadata free space managers, at most - * two of which handle free space manager header or section info, - * and thus reside in the metadata free space manager ring. - * - * At present, the task list is: + * Specifically, this means any metadata managers that DON'T + * handle space allocation for free space manager header or + * section info will reside in the raw data free space manager + * ring. + * + * In the absence of page allocation, there is at most one + * free space manager per memory type defined in H5F_mem_t. + * Of these, the one that allocates H5FD_MEM_DRAW will + * always reside in the raw data free space manager ring. + * If there is more than one metadata free space manager, + * all that don't handle H5FD_MEM_FSPACE_HDR or + * H5FD_MEM_FSPACE_SINFO (which map to H5FD_MEM_OHDR and + * H5FD_MEM_LHEAP respectively) will reside in the raw + * data free space manager ring as well + * + * With page allocation, the situation is conceptually + * identical, but more complex in practice. + * + * In the worst case (multi file driver) page allocation + * can result in two free space managers for each memory + * type -- one for small (less than on equal to one page) + * allocations, and one for large (greater than one page) + * allocations. + * + * In the more common one file case, page allocation will + * result in a total of three free space managers -- one for + * small (<= one page) raw data allocations, one for small + * metadata allocations (i.e, all memory types other than + * H5FD_MEM_DRAW), and one for all large (> one page) + * allocations. + * + * Despite these complications, the solution is the same in + * the page allocation case -- free space managers (be they + * small data or large) are assigned to the raw data free + * space manager ring if they don't allocate file space for + * free space managers. Note that in the one file case, the + * large free space manager must be assigned to the metadata + * free space manager ring, as it both allocates pages for + * the metadata free space manager, and allocates space for + * large (> 1 page) metadata cache entries. + * + * At present, the task list for this routine is: * * 1) Reduce the EOA to the extent possible. To do this: * @@ -1320,26 +2758,43 @@ done: herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) { + int pass_count; + hsize_t alloc_size; + H5F_mem_t mem_type; /* Memory type for iteration */ + H5F_mem_page_t fsm_type; /* FSM type for iteration */ + H5O_fsinfo_t fsinfo; /* Free space manager info message */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) - /* check args */ + /* Check args */ HDassert(f); HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info */ - if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5O_fsinfo_t fsinfo; /* Free space manager info message */ - H5FD_mem_t type; /* Memory type for iteration */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ - H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ - hbool_t fsm_opened[H5FD_MEM_NTYPES]; /* State of FSM */ - hbool_t fsm_visited[H5FD_MEM_NTYPES]; /* State of FSM */ + /* Only need to settle things if we are persisting the free space info + * and allocation/deallocation has occurred. + */ + if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + hbool_t fsm_opened[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ + hbool_t fsm_visited[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ + + /* Sanity check */ + HDassert(f->shared->sblock); + + /* should only be called if file is opened R/W */ + HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + + /* shouldn't be called unless we have a superblock supporting the + * superblock extension. + */ + HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2); /* Initialize fsm_opened and fsm_visited */ HDmemset(fsm_opened, 0, sizeof(fsm_opened)); @@ -1356,8 +2811,12 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * * Note that while the raw data aggregator should not be restarted during * the close process, this need not be the case for the metadata aggregator. + * + * Note also that the aggregators will not exist if page aggregation + * is enabled -- skip this if so. */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + /* Vailin -- is this correct? */ + if(!H5F_PAGED_AGGR(f) && (H5MF_free_aggrs(f, dxpl_id) < 0)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators") /* Set the ring type in the DXPL. In most cases, we will @@ -1379,7 +2838,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * In the case of the raw data free space manager, and any other free * space manager that does not allocate space for free space managers, * allocations should be complete at this point, as all raw data should - * have space allocated and be flushed to file at this point. Thus we + * have space allocated and be flushed to file by now. Thus we * can examine such free space managers and only re-allocate space for * them if they contain free space. Do this later in this function after * the EOA has been reduced to the extent possible. @@ -1390,79 +2849,100 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * are possible), the matter is more ticklish due to the self- * referential nature of the problem. These FSMs are dealt with in * H5MF_settle_meta_data_fsm(). + * + * Since paged allocation may be enabled, there may be up to two + * free space managers per memory type -- one for small and one for + * large allocation. Hence we must loop over the memory types twice + * setting the allocation size accordingly if paged allocation is + * enabled. */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5FS_stat_t fs_stat; /* Information for free-space manager */ - H5FD_mem_t fsm_type; /* File memory type for FSM */ - - /* There is potentially a many-to-one mapping from memory types to - * free space managers. Use the fsm_visited[] array to avoid visiting - * a given FSM more than once. Use fsm_opened[] to track which FSMs - * must be closed at the end of this function. - */ - fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type); - if(!fsm_visited[fsm_type]) { - fsm_visited[fsm_type] = TRUE; - - /* If there is no active FSM for this type, but such a FSM has - * space allocated in file, open it so that we can free its file - * space. - */ - if(NULL == f->shared->fs_man[fsm_type]) { - if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) { - /* Sanity check */ - HDassert(fsm_opened[fsm_type] == FALSE); - - /* Start up FSM for the file memory type */ - if(H5MF__alloc_open(f, dxpl_id, fsm_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - fsm_opened[fsm_type] = TRUE; - } /* end if */ + for(pass_count = 0; pass_count <= 1; pass_count++) { + if(pass_count == 0) + alloc_size = 1; + else if ( H5F_PAGED_AGGR(f) ) + alloc_size = f->shared->fs_page_size + 1; + else /* no need for a second pass */ + break; + + for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { + H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + + if(pass_count == 0) { /* this is the first pass */ + HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER); } /* end if */ + else if(H5F_PAGED_AGGR(f)) { /* page alloc active */ + HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); + } /* end else-if */ + else /* paged allocation disabled -- should be unreachable */ + HDassert(FALSE); + + if(!fsm_visited[fsm_type]) { + fsm_visited[fsm_type] = TRUE; + + /* If there is no active FSM for this type, but such a FSM has + * space allocated in file, open it so that we can free its file + * space. + */ + if(NULL == f->shared->fs_man[fsm_type]) { + if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) { + /* Sanity check */ + HDassert(fsm_opened[fsm_type] == FALSE); - /* Check for an actual FSM for this type now */ - /* (Possibly opened in previous step) */ - if(f->shared->fs_man[fsm_type]) { - /* Test to see if we need to switch rings -- do so if required */ - if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - curr_ring = needed_ring; + if(H5MF_open_fstype(f, dxpl_id, fsm_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + fsm_opened[fsm_type] = TRUE; + } /* end if */ } /* end if */ - /* Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + if(f->shared->fs_man[fsm_type]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ - /* Check if the free space manager has space in the file */ - if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) { - /* Delete the free space manager in the file. Will - * reallocate later if the free space manager contains - * any free space. - */ - if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") - f->shared->fs_addr[fsm_type] = HADDR_UNDEF; + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + /* Check if the free space manager has space in the file */ + if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) { + /* Delete the free space manager in the file. Will + * reallocate later if the free space manager contains + * any free space. + */ + if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id, TRUE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[fsm_type] = HADDR_UNDEF; + } /* end if */ } /* end if */ - } /* end if */ - /* note that we are tracking opened FSM -- we will close them - * at the end of the function. - */ - } /* end if */ + /* note that we are tracking opened FSM -- we will close them + * at the end of the function. + */ + } /* end if */ + } /* end for */ } /* end for */ + /* c) Delete the free space manager superblock extension message * if allocated. * * Must do this since the routine that writes / creates superblock * extension messages will choke if the target message is * unexpectedly either absent or present. + * + * Update: This is probably unecessary, as I gather that the + * file space manager info message is guaranteed to exist. + * Leave it in for now, but consider removing it. */ if(H5F_addr_defined(f->shared->sblock->ext_addr)) if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0) @@ -1486,11 +2966,16 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * those addresses are unknown. This is OK -- we will write the correct * values to the message at free space manager shutdown. */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type - 1] = HADDR_UNDEF; + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + fsinfo.fs_addr[fsm_type - 1] = HADDR_UNDEF; fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; fsinfo.threshold = f->shared->fs_threshold; - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_NO_FLAGS_SET) < 0) + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") @@ -1514,102 +2999,117 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) */ /* Reinitialize fsm_visited */ - HDmemset(fsm_visited, 0, sizeof(fsm_visited)); - - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5FD_mem_t fsm_type; /* File memory type for FSM */ - - fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type); - - /* test to see if we need to switch rings -- do so if required */ - if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - curr_ring = needed_ring; - } /* end if */ - - /* Since there can be a many-to-one mapping from memory types - * to free space managers, ensure that we don't visit any FSM - * more than once. - */ - if(!fsm_visited[fsm_type]) { - fsm_visited[fsm_type] = TRUE; - - if(f->shared->fs_man[fsm_type]) { - /* Only allocate file space if the target free space manager - * doesn't allocate file space for free space managers. Note - * that this is also the deciding factor as to whether a FSM - * in in the raw data FSM ring. - */ - if((fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - && (fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) { - H5FS_stat_t fs_stat; /* Information for free-space manager */ + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + fsm_visited[fsm_type] = FALSE; + + for(pass_count = 0; pass_count <= 1; pass_count++) { + if(pass_count == 0) + alloc_size = 1; + else if(H5F_PAGED_AGGR(f)) + alloc_size = f->shared->fs_page_size + 1; + else /* no need for a second pass */ + break; + + for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { + H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + + if(pass_count == 0) { /* this is the first pass */ + HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER); + } /* end if */ + else if(H5F_PAGED_AGGR(f)) { /* page alloc active */ + HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); + } /* end else-if */ + else /* paged allocation disabled -- should be unreachable */ + HDassert(FALSE); - /* The current ring should be H5AC_RING_RDFSM */ - HDassert(curr_ring == H5AC_RING_RDFSM); + /* test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; - /* Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 ) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ - /* If the free space manager contains section info, - * allocate space for the header and sinfo (note that - * space must not be allocated at present -- verify - * verify this with assertions). + /* Since there can be a many-to-one mapping from memory types + * to free space managers, ensure that we don't visit any FSM + * more than once. + */ + if(!fsm_visited[fsm_type]) { + fsm_visited[fsm_type] = TRUE; + + if(f->shared->fs_man[fsm_type]) { + /* Only allocate file space if the target free space manager + * doesn't allocate file space for free space managers. Note + * that this is also the deciding factor as to whether a FSM + * in in the raw data FSM ring. */ - if(fs_stat.serial_sect_count > 0) { - /* Sanity check */ - HDassert(!H5F_addr_defined(fs_stat.addr)); - - /* Allocate FSM header */ - if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header") - - /* Allocate FSM section info */ - HDassert(!H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.alloc_sect_size == 0); - if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 ) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info") + if(!H5MF__fsm_type_is_self_referential(f, fsm_type)) { + /* The current ring should be H5AC_RING_RDFSM */ + HDassert(curr_ring == H5AC_RING_RDFSM); + + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + + /* If the free space manager contains section info, + * allocate space for the header and sinfo (note that + * space must not be allocated at present -- verify + * verify this with assertions). + */ + if(fs_stat.serial_sect_count > 0) { + /* Sanity check */ + HDassert(!H5F_addr_defined(fs_stat.addr)); + + /* Allocate FSM header */ + if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header") + + /* Allocate FSM section info */ + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info") #ifndef NDEBUG - /* Re-Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - - HDassert(H5F_addr_defined(fs_stat.addr)); - HDassert(H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.serial_sect_count > 0); - HDassert(fs_stat.alloc_sect_size > 0); - HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size); + /* Re-Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.serial_sect_count > 0); + HDassert(fs_stat.alloc_sect_size > 0); + HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size); #endif /* NDEBUG */ + } /* end if */ + else { + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.serial_sect_count == 0); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end else */ } /* end if */ - else { - HDassert(!H5F_addr_defined(fs_stat.addr)); - HDassert(!H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.serial_sect_count == 0); - HDassert(fs_stat.alloc_sect_size == 0); - } /* end else */ } /* end if */ - } /* end if */ - /* Close any opened FSMs */ - if(fsm_opened[fsm_type]) { - if(H5MF__alloc_close(f, dxpl_id, fsm_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager") - fsm_opened[fsm_type] = FALSE; + /* Close any opened FSMs */ + if(fsm_opened[fsm_type]) { + if(H5MF__close_fstype(f, dxpl_id, fsm_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager") + fsm_opened[fsm_type] = FALSE; + } /* end if */ } /* end if */ - } /* end if */ + } /* end for */ } /* end for */ /* verify that all opened FSMs were closed */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - HDassert(!fsm_opened[type]); + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + HDassert(!fsm_opened[fsm_type]); /* Indicate that the FSM was settled successfully */ *fsm_settled = TRUE; @@ -1630,11 +3130,11 @@ done: * * Purpose: If the free space manager is persistent, handle any tasks * required before the metadata cache can serialize or flush - * the metadata free space manager(sI) that handle file space + * the metadata free space manager(s) that handle file space * allocation for free space managers. * * In most cases, there will be only one manager assigned - * to this role. However, since for reason or reason unknown, + * to this role. However, since for reasons unknown, * free space manager headers and section info blocks are * different classes of memory, it is possible that two free * space managers will be involved. @@ -1642,7 +3142,7 @@ done: * On entry to this function, the raw data settle routine * (H5MF_settle_raw_data_fsm()) should have: * - * 1) Freed the accumulators. + * 1) Freed the aggregators. * * 2) Freed all file space allocated to the free space managers. * @@ -1678,9 +3178,12 @@ done: * 1) Verify that the free space manager(s) involved in file * space allocation for free space managers are still floating. * - * 2) Free the accumulators. + * 2) Free the aggregators. * - * 3) Reduce the EOA to the extent possible. + * 3) Reduce the EOA to the extent possible, and make note + * of the resulting value. This value will be stored + * in the fsinfo superblock extension message and be used + * in the subsequent file open. * * 4) Re-allocate space for any free space manager(s) that: * @@ -1714,6 +3217,9 @@ done: * severe time pressure at the moment, the above brute * force solution is attractive. * + * 5) Make note of the EOA -- used for sanity checking on + * FSM shutdown. + * * Return: SUCCEED/FAIL * * Programmer: John Mainzer @@ -1724,10 +3230,23 @@ done: herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) { + H5F_mem_page_t sm_fshdr_fs_type; /* small fs hdr fsm */ + H5F_mem_page_t sm_fssinfo_fs_type; /* small fs sinfo fsm */ + H5F_mem_page_t lg_fshdr_fs_type; /* large fs hdr fsm */ + H5F_mem_page_t lg_fssinfo_fs_type; /* large fs sinfo fsm */ + H5FS_t *sm_hdr_fspace = NULL; /* ptr to sm FSM hdr alloc FSM */ + H5FS_t *sm_sinfo_fspace = NULL; /* ptr to sm FSM sinfo alloc FSM */ + H5FS_t *lg_hdr_fspace = NULL; /* ptr to lg FSM hdr alloc FSM */ + H5FS_t *lg_sinfo_fspace = NULL; /* ptr to lg FSM sinfo alloc FSM */ + haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + /* for self referential FSMs */ + H5FS_stat_t fs_stat; /* Information for hdr FSM */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ hbool_t reset_ring = FALSE; /* Whether we set the ring */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -1736,29 +3255,49 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info */ - if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5FS_t *hdr_fspace; /* Ptr to FSM hdr alloc FSM */ - H5FS_t *sinfo_fspace; /* Ptr to FSM sinfo alloc FSM */ - H5FS_stat_t fs_stat; /* Information for FSM */ - H5FD_mem_t hdr_fsm_alloc_type; /* FSM hdr alloc type */ - H5FD_mem_t sinfo_fsm_alloc_type; /* FSM info alloc type */ - - /* Get the file memory types for the FSM header & section info */ - hdr_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR); - sinfo_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO); - - /* Sanity checks */ - HDassert(hdr_fsm_alloc_type > H5FD_MEM_NOLIST); - HDassert(hdr_fsm_alloc_type < H5FD_MEM_NTYPES); - HDassert(sinfo_fsm_alloc_type > H5FD_MEM_NOLIST); - HDassert(sinfo_fsm_alloc_type < H5FD_MEM_NTYPES); - HDassert(!H5F_addr_defined(f->shared->fs_addr[hdr_fsm_alloc_type])); - HDassert(!H5F_addr_defined(f->shared->fs_addr[sinfo_fsm_alloc_type])); - - /* Note that in most cases, hdr_fspace will equal sinfo_fspace. */ - hdr_fspace = f->shared->fs_man[hdr_fsm_alloc_type]; - sinfo_fspace = f->shared->fs_man[sinfo_fsm_alloc_type]; + /* Only need to settle things if we are persisting the free space info + * and allocation/deallocation has occurred. + */ + if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + /* Sanity check */ + HDassert(f->shared->lf); + + /* should only be called if file is opened R/W */ + HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); + + HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); + + HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER); + + HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])); + HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type])); + + /* Note that in most cases, sm_hdr_fspace will equal sm_sinfo_fspace. */ + sm_hdr_fspace = f->shared->fs_man[sm_fshdr_fs_type]; + sm_sinfo_fspace = f->shared->fs_man[sm_fssinfo_fs_type]; + + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); + + HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); + + HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES); + + HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])); + HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])); + + /* Note that in most cases, lg_hdr_fspace will equal lg_sinfo_fspace. */ + lg_hdr_fspace = f->shared->fs_man[lg_fshdr_fs_type]; + lg_sinfo_fspace = f->shared->fs_man[lg_fssinfo_fs_type]; + } /* end if */ /* Set the ring in the dxpl appropriately for subsequent calls */ if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0) @@ -1766,41 +3305,109 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) reset_ring = TRUE; #ifndef NDEBUG - /* Verify that hdr_fspace is floating if it exists */ - if(hdr_fspace) { + /* Verify that sm_hdr_fspace is floating if it exists */ + if(sm_hdr_fspace) { /* Query free space manager info for this type */ - if(H5FS_stat_info(f, hdr_fspace, &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + if(H5FS_stat_info(f, sm_hdr_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") HDassert(!H5F_addr_defined(fs_stat.addr)); HDassert(!H5F_addr_defined(fs_stat.sect_addr)); HDassert(fs_stat.alloc_sect_size == 0); } /* end if */ - /* Verify that sinfo_fspace is floating if it exists */ - if((sinfo_fspace) && (hdr_fspace != sinfo_fspace)) { + /* Verify that sm_sinfo_fspace is floating if it exists and is distinct */ + if((sm_sinfo_fspace) && (sm_hdr_fspace != sm_sinfo_fspace)) { /* Query free space manager info for this type */ - if(H5FS_stat_info(f, sinfo_fspace, &fs_stat) < 0) + if(H5FS_stat_info(f, sm_sinfo_fspace, &fs_stat) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") HDassert(!H5F_addr_defined(fs_stat.addr)); HDassert(!H5F_addr_defined(fs_stat.sect_addr)); HDassert(fs_stat.alloc_sect_size == 0); } /* end if */ + + if(H5F_PAGED_AGGR(f)) { + /* Verify that lg_hdr_fspace is floating if it exists */ + if(lg_hdr_fspace) { + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, lg_hdr_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (3)") + + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end if */ + + /* Verify that lg_sinfo_fspace is floating if it + * exists and is distinct + */ + if((lg_sinfo_fspace) && (lg_hdr_fspace != lg_sinfo_fspace)) { + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, lg_sinfo_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (4)") + + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end if */ + } /* end if */ #endif /* NDEBUG */ /* Free the space in the metadata aggregator. Do this via the * H5MF_free_aggrs() call. Note that the raw data aggregator must * have already been freed. Sanity checks for this? + * + * Note that the aggregators will not exist if paged aggregation + * is enabled -- don't attempt to free if this is the case. */ + /* Vailin -- is this correct? */ /* (for space not at EOF, it may be put into free space managers) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + if((!H5F_PAGED_AGGR(f)) && (H5MF_free_aggrs(f, dxpl_id) < 0)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators") /* Trying shrinking the EOA for the file */ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + /* At this point, the EOA should be set to a value that contains + * the allocation for all user data, all non self referential FSMs, + * the superblock and all superblock extension messages. + * + * Make note of the current EOA. We will store this value in the + * free space manager superblock extension message. Since space for + * everything other than the self referential FSMs (and possibly the + * cache image) has been allocated at this point, this allows us to + * to float the self referential FSMs on the first file space allocation / + * deallocaiton and then set the EOA to this value before we handle + * the allocation / deallocation. (If a cache image exists, the + * first allocation / deallocation will be the deallocation of space + * for the cache image). + * + * WARNING: This approach settling the self referential free space + * managers and allocating space for them in the file will + * not work as currently implemented with the split and + * multi file drivers, as the self referential free space + * manager header and section info can be stored in up to + * two different files -- requiring that up to two EOA's + * be stored in the the free space managers super block + * extension message. + * + * As of this writing, we are solving this problem by + * simply not supporting persistant FSMs with the split + * and multi file drivers. + * + * Current plans are to do away with the multi file + * driver, so this should be a non-issue in this case. + * + * We should be able to support the split file driver + * without a file format change. However, the code to + * do so does not exist at present. + */ + if(HADDR_UNDEF == (eoa_pre_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + + /* ******************* PROBLEM: ******************** * * If the file has an alignement other than 1, and if @@ -1849,15 +3456,39 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * This is isn't good, but due to schedule pressure, we will just drop * the fragment on the floor for now. */ - if(hdr_fspace) - if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, hdr_fspace, - &(f->shared->fs_addr[hdr_fsm_alloc_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate hdr FSM file space") + if(sm_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space") + + if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space") + + if(H5F_PAGED_AGGR(f)) { + if(lg_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space") + + if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space") + } /* end if */ - if(sinfo_fspace && (sinfo_fspace != hdr_fspace)) - if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, sinfo_fspace, - &(f->shared->fs_addr[sinfo_fsm_alloc_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sinfo FSM file space") + /* Get the eoa after allocation of file space for the self referential + * free space managers. Assuming no cache image, this should be the + * final EOA of the file. + */ + if(HADDR_UNDEF == (eoa_post_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* All free space managers should have file space allocated for them + * now, and should see no further allocations / deallocations. Store + * the pre and post file space allocaton for self referential FSMs EOA + * for use when we actually write the free space manager superblock + * extension message. + */ + f->shared->eoa_pre_fsm_fsalloc = eoa_pre_fsm_fsalloc; + f->shared->eoa_post_fsm_fsalloc = eoa_post_fsm_fsalloc; /* Indicate that the FSM was settled successfully */ *fsm_settled = TRUE; @@ -1868,479 +3499,410 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* H5MF_settle_meta_data_fsm() */ /*------------------------------------------------------------------------- - * Function: H5MF__close_delete + * Function: H5MF__fsm_type_is_self_referential() * - * Purpose: Common code for closing and deleting freespace managers from - * the file. + * Purpose: Return TRUE if the indicated free space manager allocates + * file space for free space managers. Return FALSE otherwise. * - * Return: SUCCEED/FAIL + * Return: TRUE/FALSE * - * Programmer: Vailin Choi - * Jan 2016 + * Programmer: John Mainzer + * 12/6/16 * *------------------------------------------------------------------------- */ -static herr_t -H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl) +hbool_t +H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type) { - H5FD_mem_t type; /* Memory type for iteration */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ - herr_t ret_value = SUCCEED; /* Return value */ + H5F_mem_page_t sm_fshdr_fsm; + H5F_mem_page_t sm_fssinfo_fsm; + H5F_mem_page_t lg_fshdr_fsm; + H5F_mem_page_t lg_fssinfo_fsm; + hbool_t result = FALSE; - FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ + FUNC_ENTER_NOAPI_NOINIT_NOERR - /* check args */ + /* Sanity check */ HDassert(f); HDassert(f->shared); + HDassert(fsm_type >= H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); - /* Iterate over all the free space types that have managers and get each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; + result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm) + || (fsm_type == lg_fshdr_fsm) || (fsm_type == lg_fssinfo_fsm); + } /* end if */ + else { + /* In principle, fsm_type should always be less than + * H5F_MEM_PAGE_LARGE_SUPER whenever paged aggregation + * is not enabled. However, since there is code that does + * not observe this prinicple, force the result to FALSE if + * fsm_type is greater than or equal to H5F_MEM_PAGE_LARGE_SUPER. + */ + if(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER) + result = FALSE; else - needed_ring = H5AC_RING_RDFSM; - - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (4)") - curr_ring = needed_ring; - } /* end if */ - - /* If the free space manager for this type is open, close it */ - if(f->shared->fs_man[type]) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info") - f->shared->fs_man[type] = NULL; - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - } /* end if */ -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* If there is free space manager info for this type, delete it */ - if(H5F_addr_defined(f->shared->fs_addr[type])) { - haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ - - /* Put address into temporary variable and reset it */ - /* (Avoids loopback in file space freeing routine) */ - tmp_fs_addr = f->shared->fs_addr[type]; - f->shared->fs_addr[type] = HADDR_UNDEF; - - /* Shift to "deleting" state, to make certain we don't track any - * file space freed as a result of deleting the free space manager. - */ - f->shared->fs_state[type] = H5F_FS_STATE_DELETING; - -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* Delete free space manager for this type */ - if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager") - - /* Shift [back] to closed state */ - HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - - /* Sanity check that the free space manager for this type wasn't started up again */ - HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); - } /* end if */ - } /* end for */ + result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm); + } /* end else */ -done: -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5MF__close_delete() */ + FUNC_LEAVE_NOAPI(result) +} /* H5MF__fsm_type_is_self_referential() */ /*------------------------------------------------------------------------- - * Function: H5MF_try_close + * Function: H5MF__fsm_is_self_referential() * - * Purpose: This is called by H5Fformat_convert() to close and delete - * free-space managers when downgrading persistent free-space - * to non-persistent. + * Purpose: Return TRUE if the indicated free space manager allocates + * file space for free space managers. Return FALSE otherwise. * - * Return: SUCCEED/FAIL + * Return: TRUE/FALSE * - * Programmer: Vailin Choi - * Jan 2016 + * Programmer: John Mainzer + * 12/6/16 * *------------------------------------------------------------------------- */ -herr_t -H5MF_try_close(H5F_t *f, hid_t dxpl_id) +static hbool_t +H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5F_mem_page_t sm_fshdr_fsm; + H5F_mem_page_t sm_fssinfo_fsm; + hbool_t result = FALSE; - FUNC_ENTER_NOAPI(FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ + FUNC_ENTER_STATIC_NOERR - /* check args */ + /* Sanity check */ HDassert(f); + HDassert(f->shared); + HDassert(fspace); - /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - reset_ring = TRUE; + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); - /* Close and delete freespace managers from the file */ - if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to close delete free-space managers") + if(H5F_PAGED_AGGR(f)) { + H5F_mem_page_t lg_fshdr_fsm; + H5F_mem_page_t lg_fssinfo_fsm; -done: - /* Reset the ring in the DXPL */ - if(reset_ring) - if(H5AC_reset_ring(dxpl, orig_ring) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_try_close() */ + result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || + (fspace == f->shared->fs_man[sm_fssinfo_fsm]) || + (fspace == f->shared->fs_man[lg_fshdr_fsm]) || + (fspace == f->shared->fs_man[lg_fssinfo_fsm]); + } /* end if */ + else + result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || + (fspace == f->shared->fs_man[sm_fssinfo_fsm]); + + FUNC_LEAVE_NOAPI(result) +} /* H5MF__fsm_is_self_referential() */ /*------------------------------------------------------------------------- - * Function: H5MF_close - * - * Purpose: Close the free space tracker(s) for a file - * - * Return: SUCCEED/FAIL + * Function: H5MF_tidy_self_referential_fsm_hack + * + * Purpose: As discussed in the comments of the settle routines above, + * the existence of self referential free space managers + * as currently implemented creates the possibility of + * infinite loops at file close. + * + * As a hack to avoid this, we have added code to settle + * self referential free space managers, and then allocate + * space for them directly from the file driver. + * + * To avoid dropping ever increasing amounts of file space + * on the floor with each subsequent file close/open cycle, + * we need to clean this up on file open. To avoid this, + * this function is called on the first file space allocation + * or deallocation after file open to float the self referential + * free space managers and reduce the EOA to the value it + * had before the direct allocation of space for the self + * referential free space managers. + * + * The function proceeds as follows: + * + * 1) Verify that f->shared->first_alloc_dealloc is TRUE, + * and then set it to FALSE. + * + * 2) Get the current EOA. Verify that it is greater than + * or equal to f->shared->eoa_pre_fsm_fsalloc. If the + * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, + * no self referential FSMs were stored, and we are done. + * + * NOTE: This will have to be reworked somewhat for + * cache image. + * + * 3) Load the self referential FSMs. In passing verify that + * the lowest address of a FSM header is equal to + * f->shared->eoa_pre_fsm_fsalloc.' + * + * Note that we don't have to use any special I/O for + * this -- we can use the regular I/O methods even if + * paged aggregation and page buffering is enabled. + * + * 4) Float the FSMs. Ensure that the file space is NOT + * released. + * + * 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, + * and then set f->shared->eoa_pre_fsm_fsalloc to + * HADDR_UNDEF. + * + * If page buffering, verify that the new EOA is + * on a page boundary, and expunge any pages in the + * page buffer after the new EOA. + * + * Note that this function is also called from test code + * when it is necessary to startup a self referential + * free space manager prior to the first file space + * allocation / deallocation. Failure to do so will + * result in assertion failures in this function on + * the first file space allocation / deallocation. + * + * Return: SUCCEED/FAIL * - * Programmer: Quincey Koziol - * Tuesday, January 22, 2008 + * Programmer: John Mainzer + * 12/11/16 * - * Modifications: - * Vailin Choi; July 2012 - * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY, - * modifications are needed to shrink EOA if the last section of each free-space manager - * and the remaining space in the two aggregators are at EOA. - *------------------------------------------------------------------------- */ herr_t -H5MF_close(H5F_t *f, hid_t dxpl_id) +H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id) { + haddr_t eoa; /* EOA of file */ + hsize_t tail_size = 0; /* Size of chunk to free */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t type; /* Memory type for iteration */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + haddr_t first_srfsm_hdr = HADDR_UNDEF; /* Addr of first self referential */ + /* fsm header in file */ + H5FS_stat_t fs_stat; /* Information for hdr FSM */ + H5F_mem_page_t sm_fshdr_fs_type; /* Small fs hdr fsm */ + H5F_mem_page_t sm_fssinfo_fs_type; /* Small fs sinfo fsm */ + H5F_mem_page_t lg_fshdr_fs_type; /* Large fs hdr fsm */ + H5F_mem_page_t lg_fssinfo_fs_type; /* Large fs sinfo fsm */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ /* check args */ HDassert(f); HDassert(f->shared); - HDassert(f->shared->lf); - HDassert(f->shared->sblock); + HDassert(f->shared->fs_persist); + HDassert(f->shared->first_alloc_dealloc); - /* Set the ring type in the DXPL. In most cases, we will - * need H5AC_RING_RDFSM, so initialy set the ring in - * the DXPL to that value. We will alter this later if - * needed. + /* Set the ring type in the DXPL. Since we are only dealing with + * self referential FSMs, we will only need H5AC_RING_MDFSM. */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - curr_ring = H5AC_RING_RDFSM; - /* Free the space in aggregators */ - /* (for space not at EOF, it may be put into free space managers) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + /* 1) Verify that f->shared->first_alloc_dealloc is TRUE, + * and then set it to FALSE. + */ + HDassert(f->shared->first_alloc_dealloc); + f->shared->first_alloc_dealloc = FALSE; - /* Trying shrinking the EOA for the file */ - if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") - /* Making free-space managers persistent for superblock version >= 2 */ - if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 - && f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5O_fsinfo_t fsinfo; /* Free space manager info message */ + /* 2) Get the current EOA. Verify that it is greater than + * or equal to f->shared->eoa_pre_fsm_fsalloc. If the + * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, + * no self referential FSMs were stored, and we are done. + * + * NOTE: This will have to be reworked somewhat for + * cache image. + */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + HDassert(H5F_addr_le(f->shared->eoa_pre_fsm_fsalloc, eoa)); - /* Superblock extension and free space manager message should - * exist at this point -- verify at least the former. - */ - HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + if(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)) + HGOTO_DONE(SUCCEED) - /* Note that unlike the previous version of this code, we do not - * delete free space managers that have no section to store. - * - * Can't do this, as that would involve freeing file space, which would - * dirty the free space manager in question. - * - * Fortunately, the code doesn't seem to care about this. - */ - /* Gather data for the free space manager superblock extension message. - * - * In passing, verify that all the free space managers are closed. - */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type - 1] = f->shared->fs_addr[type]; - fsinfo.strategy = f->shared->fs_strategy; - fsinfo.threshold = f->shared->fs_threshold; + /* 3) Load the self referential FSMs. In passing verify that + * the lowest address of a FSM header is equal to + * f->shared->eoa_pre_fsm_fsalloc.' + * + * Note that we don't have to use any special I/O for + * this -- we can use the regular I/O methods even if + * paged aggregation and page buffering is enabled. + */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); + HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - /* Write the free space manager message -- message must already exist */ - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - /* Final close of free-space managers */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - if(f->shared->fs_man[type]) { - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ + HDassert(NULL == f->shared->fs_man[sm_fshdr_fs_type]); + HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])) { + first_srfsm_hdr = f->shared->fs_addr[sm_fshdr_fs_type]; - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)") - curr_ring = needed_ring; - } /* end if */ + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, sm_fshdr_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN); - if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") - f->shared->fs_man[type] = NULL; - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - } /* end if */ - f->shared->fs_addr[type] = HADDR_UNDEF; - } /* end for */ + HDassert(f->shared->fs_man[sm_fshdr_fs_type]); + } /* end if */ - /* Verify that we haven't dirtied any metadata cache entries - * from the metadata free space manager ring out. - */ - HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + if((sm_fshdr_fs_type != sm_fssinfo_fs_type) && + (H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type]))) { - /* Verify that the aggregators are still shutdown. */ - HDassert(f->shared->sdata_aggr.tot_size == 0); - HDassert(f->shared->sdata_aggr.addr == 0); - HDassert(f->shared->sdata_aggr.size == 0); - HDassert(f->shared->meta_aggr.tot_size == 0); - HDassert(f->shared->meta_aggr.addr == 0); - HDassert(f->shared->meta_aggr.size == 0); - } /* end if */ - else { /* super_vers can be 0, 1, 2 */ - /* Close and delete freespace managers from the file */ - if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[sm_fssinfo_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[sm_fssinfo_fs_type]; - /* moved code that was for both the persistant and non persistant free - * space managers to the non-persistant case. In the persistant - * case, the EOA should already be as shrunked as it can be, and the - * aggregators should alread be shut down. - */ + HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - /* Free the space in aggregators (again) */ - /* (in case any free space information re-started them) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, sm_fssinfo_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - /* Trying shrinking the EOA for the file */ - /* (in case any free space is now at the EOA) */ - if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") - } /* end else */ + HDassert(f->shared->fs_man[sm_fssinfo_fs_type]); + } /* end if */ -done: - /* Reset the ring in the DXPL */ - if(reset_ring) - if(H5AC_reset_ring(dxpl, orig_ring) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_close() */ + HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); - -/*------------------------------------------------------------------------- - * Function: H5MF_sects_cb() - * - * Purpose: Iterator callback for each free-space section - * Retrieve address and size into user data - * - * Return: Always succeed - * - * Programmer: Vailin Choi - * July 1st, 2009 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata) -{ - H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; - H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata; + HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES); - FUNC_ENTER_NOAPI_NOINIT_NOERR + HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); + HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - if(udata->sect_idx < udata->sect_count) { - udata->sects[udata->sect_idx].addr = sect->sect_info.addr; - udata->sects[udata->sect_idx].size = sect->sect_info.size; - udata->sect_idx++; - } /* end if */ + if(H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])) { + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[lg_fshdr_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[lg_fshdr_fs_type]; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sects_cb() */ + HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); - -/*------------------------------------------------------------------------- - * Function: H5MF_get_free_sections() - * - * Purpose: To iterate over one or all free-space managers for: - * # of sections - * section info as defined in H5F_sect_info_t - * - * Return: SUCCEED/FAIL - * - * Programmer: Vailin Choi - * July 1st, 2009 - * - *------------------------------------------------------------------------- - */ -ssize_t -H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info) -{ - size_t total_sects = 0; /* total number of sections */ - H5MF_sect_iter_ud_t sect_udata; /* User data for callback */ - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t start_type, end_type; /* Memory types to iterate over */ - H5FD_mem_t ty; /* Memory type for iteration */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - ssize_t ret_value = -1; /* Return value */ + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, lg_fshdr_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + HDassert(f->shared->fs_man[lg_fshdr_fs_type]); + } /* end if */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + if(lg_fshdr_fs_type != lg_fssinfo_fs_type && H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])) { + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[lg_fssinfo_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[lg_fssinfo_fs_type]; - /* check args */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); + HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - /* Determine start/end points for loop */ - if(type == H5FD_MEM_DEFAULT) { - start_type = H5FD_MEM_SUPER; - end_type = H5FD_MEM_NTYPES; + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, lg_fssinfo_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + HDassert(f->shared->fs_man[lg_fssinfo_fs_type]); + } /* end if */ + } /* end if */ + HDassert(H5F_addr_eq(first_srfsm_hdr, f->shared->eoa_pre_fsm_fsalloc)); + + /* 4) Float the FSMs. Ensure that the file space is NOT released. */ + if(f->shared->fs_man[sm_fshdr_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[sm_fshdr_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[sm_fshdr_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[sm_fshdr_fs_type] = HADDR_UNDEF; } /* end if */ - else { - start_type = end_type = type; - H5_INC_ENUM(H5FD_mem_t, end_type); - } /* end else */ - /* Set up user data for section iteration */ - sect_udata.sects = sect_info; - sect_udata.sect_count = nsects; - sect_udata.sect_idx = 0; + if(sm_fshdr_fs_type != sm_fssinfo_fs_type && f->shared->fs_man[sm_fssinfo_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[sm_fssinfo_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - /* Set the ring type in the DXPL. Note that if we are - * scanning a number of different free space managers, - * we may have to change the ring - */ - if((H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - curr_ring = needed_ring; - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &orig_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - reset_ring = TRUE; - - /* Iterate over memory types, retrieving the number of sections of each type */ - for(ty = start_type; ty < end_type; H5_INC_ENUM(H5FD_mem_t, ty)) { - hbool_t fs_started = FALSE; + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[sm_fssinfo_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[sm_fssinfo_fs_type] = HADDR_UNDEF; + } /* end if */ - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(H5F_PAGED_AGGR(f)) { + if(f->shared->fs_man[lg_fshdr_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[lg_fshdr_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[lg_fshdr_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") + f->shared->fs_addr[lg_fshdr_fs_type] = HADDR_UNDEF; } /* end if */ - /* Open free space manager of this type, if it isn't already */ - if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) { - if(H5MF__alloc_open(f, dxpl_id, ty) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space") - HDassert(f->shared->fs_man[ty]); - fs_started = TRUE; - } /* end if */ - - /* Check if f there's free space sections of this type */ - if(f->shared->fs_man[ty]) { - hsize_t hnums = 0; /* Total # of sections */ - size_t nums; /* Total # of sections, cast to a size_t */ - - /* Query how many sections of this type */ - if(H5FS_sect_stats(f->shared->fs_man[ty], NULL, &hnums) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") - H5_CHECKED_ASSIGN(nums, size_t, hnums, hsize_t); - - /* Increment total # of sections */ - total_sects += nums; - - /* Check if we should retrieve the section info */ - if(sect_info && nums > 0) { - /* Iterate over all the free space sections of this type, adding them to the user's section info */ - if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ty], H5MF_sects_cb, §_udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections") - } /* end if */ + if(lg_fshdr_fs_type != lg_fssinfo_fs_type && f->shared->fs_man[lg_fssinfo_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[lg_fssinfo_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[lg_fssinfo_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") + f->shared->fs_addr[lg_fssinfo_fs_type] = HADDR_UNDEF; } /* end if */ + } /* end if */ - /* Close the free space manager of this type, if we started it here */ - if(fs_started) - if(H5MF__alloc_close(f, dxpl_id, ty) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space") - } /* end for */ + /* 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, + * and then set f->shared->eoa_pre_fsm_fsalloc to + * HADDR_UNDEF. + * + * If page buffering, verify that the new EOA is + * on a page boundary, and expunge any pages in the + * page buffer after the new EOA. + */ + if(!H5F_PAGED_AGGR(f)) { + /* Verify that the aggregators are still shutdown. */ + HDassert(f->shared->sdata_aggr.tot_size == 0); + HDassert(f->shared->sdata_aggr.addr == 0); + HDassert(f->shared->sdata_aggr.size == 0); - /* Set return value */ - ret_value = (ssize_t)total_sects; + HDassert(f->shared->meta_aggr.tot_size == 0); + HDassert(f->shared->meta_aggr.addr == 0); + HDassert(f->shared->meta_aggr.size == 0); + } /* end if */ + + tail_size = (hsize_t)(eoa - f->shared->eoa_pre_fsm_fsalloc); + + /* Release file space allocated to self referential FSMs */ + if(H5F_free(f, dxpl_id, H5FD_MEM_DEFAULT, f->shared->eoa_pre_fsm_fsalloc, tail_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + HDassert(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)); + + f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + HDassert((!H5F_PAGED_AGGR(f)) || (0 == (eoa % f->shared->fs_page_size))); done: /* Reset the ring in the DXPL */ @@ -2349,5 +3911,5 @@ done: HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5MF_get_free_sections() */ +} /* H5MF_tidy_self_referential_fsm_hack() */ diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c index 1510645..de98bf1 100644 --- a/src/H5MFaggr.c +++ b/src/H5MFaggr.c @@ -57,8 +57,10 @@ /********************/ /* Local Prototypes */ /********************/ -static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, +static herr_t H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr); +static haddr_t H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, + H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size); /*********************/ @@ -78,86 +80,9 @@ static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, /*------------------------------------------------------------------------- - * Function: H5MF_vfd_alloc - * - * Purpose: Allocate SIZE bytes of file memory via H5FD_alloc() and return - * the relative address where that contiguous chunk of file memory - * exists. - * The TYPE argument describes the purpose for which the storage - * is being requested. - * - * Return: Success: The file address of new chunk. - * Failure: HADDR_UNDEF - * - * Programmer: Quincey Koziol - * January 2, 2017 - * - *------------------------------------------------------------------------- - */ -haddr_t -H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, hsize_t size, - hbool_t keep_fragment) -{ - haddr_t eoa; /* Initial EOA for the file */ - haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ - hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */ - haddr_t ret_value = HADDR_UNDEF; /* Return value */ - - FUNC_ENTER_NOAPI(HADDR_UNDEF) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); -#endif /* H5MF_ALLOC_DEBUG */ - - /* check arguments */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); - HDassert(size > 0); - - /* Get the EOA for the file -- need for sanity check below */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") - - /* Check for overlap into temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "hdr file space alloc will overlap into 'temporary' file space") - - /* Allocate space for the header */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space for hdr") - - /* Sanity check for overlapping into file's temporary allocation space */ - HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); - - /* If the file alignment is 1, there should be no eoa fragment */ - HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); - - /* Check if fragment was generated and we want to keep it */ - if(keep_fragment && eoa_frag_size > 0) { - /* Sanity check */ - HDassert(H5F_addr_defined(eoa_frag_addr)); - - /* Put fragment on the free list */ - if(H5MF_xfree(f, alloc_type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment") - } /* end if */ - -done: -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); -#endif /* H5MF_ALLOC_DEBUG */ -#ifdef H5MF_ALLOC_DEBUG_DUMP -H5MF_sects_dump(f, dxpl_id, stderr); -#endif /* H5MF_ALLOC_DEBUG_DUMP */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_vfd_alloc() */ - - -/*------------------------------------------------------------------------- * Function: H5MF_aggr_vfd_alloc * - * Purpose: Allocate SIZE bytes of file memory via H5MF_aggr_alloc() + * Purpose: Allocate SIZE bytes of file memory via H5MF__aggr_alloc() * and return the relative address where that contiguous chunk * of file memory exists. * The TYPE argument describes the purpose for which the storage @@ -177,9 +102,9 @@ H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size haddr_t ret_value = HADDR_UNDEF; /* Return value */ FUNC_ENTER_NOAPI(HADDR_UNDEF) -#ifdef H5MF_ALLOC_DEBUG +#ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); -#endif /* H5MF_ALLOC_DEBUG */ +#endif /* H5MF_AGGR_DEBUG */ /* check arguments */ HDassert(f); @@ -190,12 +115,12 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ /* Couldn't find anything from the free space manager, go allocate some */ if(alloc_type != H5FD_MEM_DRAW && alloc_type != H5FD_MEM_GHEAP) { /* Handle metadata differently from "raw" data */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size))) + if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate metadata") } /* end if */ else { /* Allocate "raw" data: H5FD_MEM_DRAW and H5FD_MEM_GHEAP */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size))) + if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate raw data") } /* end else */ @@ -203,19 +128,16 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); done: -#ifdef H5MF_ALLOC_DEBUG +#ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); -#endif /* H5MF_ALLOC_DEBUG */ -#ifdef H5MF_ALLOC_DEBUG_DUMP -H5MF_sects_dump(f, dxpl_id, stderr); -#endif /* H5MF_ALLOC_DEBUG_DUMP */ +#endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5MF_aggr_vfd_alloc() */ /*------------------------------------------------------------------------- - * Function: H5MF_aggr_alloc + * Function: H5MF__aggr_alloc * * Purpose: Try to allocate SIZE bytes of memory from an aggregator * block if possible. @@ -228,8 +150,8 @@ H5MF_sects_dump(f, dxpl_id, stderr); * *------------------------------------------------------------------------- */ -haddr_t -H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, +static haddr_t +H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size) { haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ @@ -237,7 +159,7 @@ H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ - FUNC_ENTER_NOAPI(HADDR_UNDEF) + FUNC_ENTER_STATIC #ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); #endif /* H5MF_AGGR_DEBUG */ @@ -257,11 +179,11 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") /* - * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_VFD, + * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_NONE, * allocate "generic" space and sub-allocate out of that, if possible. - * Otherwise just allocate through H5FD_alloc(). + * Otherwise just allocate through H5F_alloc(). */ - if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FILE_SPACE_VFD) { + if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE) { haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */ hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */ hsize_t alignment; /* Alignment of this section */ @@ -273,12 +195,12 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz #endif /* H5MF_AGGR_DEBUG */ /* Turn off alignment if allocation < threshold */ - alignment = f->shared->alignment; - if(!((alignment > 1) && (size >= f->shared->threshold))) + alignment = H5F_ALIGNMENT(f); + if(!((alignment > 1) && (size >= H5F_THRESHOLD(f)))) alignment = 0; /* no alignment */ /* Generate fragment if aggregator is mis-aligned */ - if(alignment && aggr->addr > 0 && (aggr_mis_align = (aggr->addr + H5FD_get_base_addr(f->shared->lf)) % alignment)) { + if(alignment && H5F_addr_gt(aggr->addr, 0) && (aggr_mis_align = (aggr->addr + H5F_BASE_ADDR(f)) % alignment)) { aggr_frag_addr = aggr->addr; aggr_frag_size = alignment - aggr_mis_align; } /* end if */ @@ -288,7 +210,7 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz /* Check if the space requested is larger than the space left in the block */ if((size + aggr_frag_size) > aggr->size) { - htri_t was_extended = FALSE; /* Whether the file was extended */ + htri_t extended = FALSE; /* Whether the file was extended */ /* Check if the block asked for is too large for 'normal' aggregator block */ if(size >= aggr->alloc_size) { @@ -298,32 +220,27 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr)) HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0) + if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space") - else if(was_extended) { + else if (extended) { /* aggr->size is unchanged */ ret_value = aggr->addr + aggr_frag_size; aggr->addr += ext_size; aggr->tot_size += ext_size; - } /* end else-if */ - else { - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - + } else { /* Release "other" aggregator, if it exists, is at the end of the allocated space, * has allocated more than one block and the unallocated space is greater than its * allocation block size. */ if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) && (other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) { - if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) + if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block") } /* end if */ /* Allocate space from the VFD (i.e. at the end of the file) */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block") + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size, &eoa_frag_addr, &eoa_frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") } /* end else */ } /* end if */ else { @@ -341,9 +258,9 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr)) HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0) + if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space") - else if(was_extended) { + else if(extended) { aggr->addr += aggr_frag_size; aggr->size += (ext_size - aggr_frag_size); aggr->tot_size += ext_size; @@ -351,23 +268,19 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); else { haddr_t new_space; /* Address of new space allocated */ - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + aggr->alloc_size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - /* Release "other" aggregator, if it exists, is at the end of the allocated space, * has allocated more than one block and the unallocated space is greater than its * allocation block size. */ if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) && (other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) { - if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) + if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block") } /* end if */ /* Allocate space from the VFD (i.e. at the end of the file) */ - if(HADDR_UNDEF == (new_space = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block") + if(HADDR_UNDEF == (new_space = H5F_alloc(f, dxpl_id, alloc_type, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") /* Return the unused portion of the block to a free list */ if(aggr->size > 0) @@ -408,7 +321,7 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment") /* Freeing any possible fragment due to alignment in the block after extension */ - if(was_extended && aggr_frag_size) + if(extended && aggr_frag_size) if(H5MF_xfree(f, alloc_type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation fragment") } /* end if */ @@ -425,12 +338,8 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); } /* end else */ } /* end if */ else { - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - /* Allocate data from the file */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, &eoa_frag_addr, &eoa_frag_size))) + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, type, size, &eoa_frag_addr, &eoa_frag_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") /* Check if fragment was generated */ @@ -444,15 +353,15 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); /* Post-condition sanity check */ - if(f->shared->alignment && size >= f->shared->threshold) - HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % f->shared->alignment)); + if(H5F_ALIGNMENT(f) && size >= H5F_THRESHOLD(f)) + HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % H5F_ALIGNMENT(f))); done: #ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: ret_value = %a\n", FUNC, ret_value); #endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_aggr_alloc() */ +} /* end H5MF__aggr_alloc() */ /*------------------------------------------------------------------------- @@ -524,7 +433,7 @@ H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, else { hsize_t extra = (extra_requested < aggr->alloc_size) ? aggr->alloc_size : extra_requested; - if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, (aggr->addr + aggr->size), extra)) < 0) + if((ret_value = H5F_try_extend(f, dxpl_id, type, (aggr->addr + aggr->size), extra)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") else if(ret_value == TRUE) { /* Shift the aggregator block by the extra requested */ @@ -905,7 +814,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5MF_aggr_free + * Function: H5MF__aggr_free * * Purpose: Free the aggregator's space in the file. * @@ -919,11 +828,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) +H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC /* Sanity check */ HDassert(f); @@ -936,7 +845,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) HDassert(f->shared->feature_flags & aggr->feature_flag); /* Free the remaining space at EOA in the aggregator */ - if(H5FD_free(f->shared->lf, dxpl_id, type, f, aggr->addr, aggr->size) < 0) + if(H5F_free(f, dxpl_id, type, aggr->addr, aggr->size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregation block") /* Reset the aggregator */ @@ -946,7 +855,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_aggr_free() */ +} /* H5MF__aggr_free() */ /*------------------------------------------------------------------------- @@ -978,13 +887,13 @@ H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id) if((ma_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr))) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") if(ma_status > 0) - if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0) + if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") if((sda_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DRAW, &(f->shared->sdata_aggr))) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") if(sda_status > 0) - if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0) + if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") ret_value = (ma_status || sda_status); diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c index 57bac9e..817ea00 100644 --- a/src/H5MFdbg.c +++ b/src/H5MFdbg.c @@ -116,7 +116,9 @@ H5MF_sects_debug_cb(H5FS_section_info_t *_sect, void *_udata) /* Print generic section information */ HDfprintf(udata->stream, "%*s%-*s %s\n", udata->indent, "", udata->fwidth, "Section type:", - (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" : "unknown")); + (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" : + (sect->sect_info.type == H5MF_FSPACE_SECT_SMALL ? "small" : + (sect->sect_info.type == H5MF_FSPACE_SECT_LARGE ? "large" : "unknown")))); HDfprintf(udata->stream, "%*s%-*s %a\n", udata->indent, "", udata->fwidth, "Section address:", sect->sect_info.addr); @@ -171,7 +173,7 @@ H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int ind for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) if(H5F_addr_eq(f->shared->fs_addr[type], fs_addr)) { if(!f->shared->fs_man[type]) - if(H5MF__alloc_open(f, dxpl_id, type) < 0) + if(H5MF_open_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") if(f->shared->fs_man[type]) { @@ -217,11 +219,6 @@ herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream) { haddr_t eoa; /* End of allocated space in the file */ - haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ - hsize_t ma_size = 0; /* Size of "metadata aggregator" */ - haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ - hsize_t sda_size = 0; /* Size of "small data aggregator" */ - H5FD_mem_t type; /* Memory type for iteration */ int indent = 0; /* Amount to indent */ int fwidth = 50; /* Field width */ herr_t ret_value = SUCCEED; /* Return value */ @@ -244,56 +241,90 @@ HDfprintf(stderr, "%s: Dumping file free space sections\n", FUNC); HDfprintf(stderr, "%s: for type = H5FD_MEM_DEFAULT, eoa = %a\n", FUNC, eoa); #endif /* H5MF_ALLOC_DEBUG */ - /* Retrieve metadata aggregator info, if available */ - H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1)); -#endif /* H5MF_ALLOC_DEBUG */ + if(H5F_PAGED_AGGR(f)) { /* File space paging */ + H5F_mem_page_t ptype; /* Memory type for iteration -- page fs */ - /* Retrieve 'small data' aggregator info, if available */ - H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1)); -#endif /* H5MF_ALLOC_DEBUG */ - - /* Iterate over all the free space types that have managers and dump each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - /* Print header for type */ - HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)type); - - /* Check for this type being mapped to another type */ - if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[type] || - type == f->shared->fs_type_map[type]) { - /* Retrieve the 'eoa' for this file memory type */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") - HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa); + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + /* Print header for type */ + HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)ptype); /* Print header for sections */ HDfprintf(stream, "%*sSections:\n", indent + 3, ""); /* If there is a free space manager for this type, iterate over them */ - if(f->shared->fs_man[type]) { + if(f->shared->fs_man[ptype]) { H5MF_debug_iter_ud_t udata; /* User data for callbacks */ /* Prepare user data for section iteration callback */ - udata.fspace = f->shared->fs_man[type]; + udata.fspace = f->shared->fs_man[ptype]; udata.stream = stream; udata.indent = indent + 6; udata.fwidth = MAX(0, fwidth - 6); /* Iterate over all the free space sections */ - if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[type], H5MF_sects_debug_cb, &udata) < 0) + if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ptype], H5MF_sects_debug_cb, &udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") } /* end if */ else /* No sections of this type */ HDfprintf(stream, "%*s<none>\n", indent + 6, ""); - } /* end if */ - else - HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[type]); + } /* end for */ + } /* end if */ + else { /* not file space paging */ + H5FD_mem_t atype; /* Memory type for iteration -- aggr fs */ + haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ + hsize_t ma_size = 0; /* Size of "metadata aggregator" */ + haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ + hsize_t sda_size = 0; /* Size of "small data aggregator" */ + + /* Retrieve metadata aggregator info, if available */ + H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1)); +#endif /* H5MF_ALLOC_DEBUG */ + + /* Retrieve 'small data' aggregator info, if available */ + H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size); +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1)); +#endif /* H5MF_ALLOC_DEBUG */ - } /* end for */ + /* Iterate over all the free space types that have managers and dump each free list's space */ + for(atype = H5FD_MEM_DEFAULT; atype < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, atype)) { + /* Print header for type */ + HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)atype); + + /* Check for this type being mapped to another type */ + if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[atype] || atype == f->shared->fs_type_map[atype]) { + /* Retrieve the 'eoa' for this file memory type */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, atype))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa); + + /* Print header for sections */ + HDfprintf(stream, "%*sSections:\n", indent + 3, ""); + + /* If there is a free space manager for this type, iterate over them */ + if(f->shared->fs.aggr.fs_man[atype]) { + H5MF_debug_iter_ud_t udata; /* User data for callbacks */ + + /* Prepare user data for section iteration callback */ + udata.fspace = f->shared->fs_man[atype]; + udata.stream = stream; + udata.indent = indent + 6; + udata.fwidth = MAX(0, fwidth - 6); + + /* Iterate over all the free space sections */ + if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[atype], H5MF_sects_debug_cb, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") + } /* end if */ + else /* No sections of this type */ + HDfprintf(stream, "%*s<none>\n", indent + 6, ""); + } /* end if */ + else + HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[atype]); + } /* end for */ + } /* end else */ done: HDfprintf(stderr, "%s: Done dumping file free space sections\n", FUNC); diff --git a/src/H5MFpkg.h b/src/H5MFpkg.h index 43fc1cd..c4a0819 100644 --- a/src/H5MFpkg.h +++ b/src/H5MFpkg.h @@ -51,9 +51,38 @@ /* Define this to dump free space tracker contents after they've been modified */ /* #define H5MF_ALLOC_DEBUG_DUMP */ -/* Free space section types for file */ +/* Free-space section types for file */ /* (values stored in free space data structures in file) */ -#define H5MF_FSPACE_SECT_SIMPLE 0 /* Section is a range of actual bytes in file */ +#define H5MF_FSPACE_SECT_SIMPLE 0 /* For non-paged aggregation: section is a range of actual bytes in file */ +#define H5MF_FSPACE_SECT_SMALL 1 /* For paged aggregation: "small" meta/raw data section which is < fsp_size) */ +#define H5MF_FSPACE_SECT_LARGE 2 /* For paged aggregation: "large" Section which is >= fsp_size) */ + +/* For non-paged aggregation: map allocation request type to tracked free-space type */ +/* F -- pointer to H5F_t; T -- H5FD_mem_t */ +#define H5MF_ALLOC_TO_FS_AGGR_TYPE(F, T) \ + ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) ? (T) : (F)->shared->fs_type_map[T]) + +/* Get section class type based on size */ +#define H5MF_SECT_CLASS_TYPE(F, S) \ + ((H5F_PAGED_AGGR(F)) ? \ + ((S >= (F)->shared->fs_page_size) ? H5MF_FSPACE_SECT_LARGE : H5MF_FSPACE_SECT_SMALL) : H5MF_FSPACE_SECT_SIMPLE) + +/* Get section class cls */ +#define H5MF_SECT_CLS_TYPE(F, S) \ + ((H5F_PAGED_AGGR(F)) ? \ + ((S >= (F)->shared->fs_page_size) ? \ + H5MF_FSPACE_SECT_CLS_LARGE : H5MF_FSPACE_SECT_CLS_SMALL) : H5MF_FSPACE_SECT_CLS_SIMPLE) + +/* Calculate the mis-aligned fragment */ +#define H5MF_EOA_MISALIGN(F, E, A, FR) \ +{ \ + hsize_t m; \ + \ + if(H5F_addr_gt((E), 0) && ((m) = ((E) + H5F_BASE_ADDR(F)) % (A))) \ + (FR) = (A) - m; \ + else \ + (FR) = 0; \ +} /****************************/ @@ -129,6 +158,15 @@ typedef struct H5MF_sect_ud_t { H5F_blk_aggr_t *aggr; /* Aggregator block to operate on */ } H5MF_sect_ud_t; +/* Information about the current free-space manager to use */ +typedef struct H5MF_fs_t { + H5F_fs_state_t *fs_state; + haddr_t *fs_addr; + H5FS_t **fs_man; + hsize_t align_thres; /* Threshold for alignment */ + hsize_t alignment; /* Alignment */ +} H5MF_fs_t; + /*****************************/ /* Package Private Variables */ @@ -136,6 +174,8 @@ typedef struct H5MF_sect_ud_t { /* H5MF single section inherits serializable properties from H5FS_section_class_t */ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1]; +H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1]; +H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1]; /******************************/ @@ -143,22 +183,23 @@ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1]; /******************************/ /* Allocator routines */ -H5_DLL herr_t H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -H5_DLL herr_t H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); +H5_DLL herr_t H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +H5_DLL herr_t H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); + +H5_DLL htri_t H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr); +H5_DLL herr_t H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node); + H5_DLL herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream); -/* 'simple' section routines */ -H5_DLL H5MF_free_section_t *H5MF_sect_simple_new(haddr_t sect_off, +H5_DLL void H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type); + +/* 'simple/small/large' section routines */ +H5_DLL H5MF_free_section_t *H5MF_sect_new(unsigned ctype, haddr_t sect_off, hsize_t sect_size); -H5_DLL htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, - void *udata); -H5_DLL herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, - void *udata); -H5_DLL herr_t H5MF_sect_simple_free(H5FS_section_info_t *sect); +H5_DLL herr_t H5MF_sect_free(H5FS_section_info_t *sect); + /* Block aggregator routines */ -H5_DLL haddr_t H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, - H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size); H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, H5FD_mem_t type, haddr_t abs_blk_end, hsize_t extra_requested); H5_DLL htri_t H5MF_aggr_can_absorb(const H5F_t *f, const H5F_blk_aggr_t *aggr, diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h index e258677..70322c3 100644 --- a/src/H5MFprivate.h +++ b/src/H5MFprivate.h @@ -58,8 +58,6 @@ H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id); /* File space allocation routines */ H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); -H5_DLL haddr_t H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, - hsize_t size, hbool_t keep_fragment); H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); H5_DLL herr_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, @@ -80,6 +78,14 @@ H5_DLL htri_t H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled); H5_DLL herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled); +/* This function has to be declared in H5MFprivate.h as it is needed + * in our test code to allow us to manually start a self referential + * free space manager prior to the first file space allocations / + * deallocation without causing assertion failures on the first + * file space allocation / deallocation. + */ +H5_DLL herr_t H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id); + /* Debugging routines */ #ifdef H5MF_DEBUGGING H5_DLL herr_t H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, diff --git a/src/H5MFsection.c b/src/H5MFsection.c index e5a0cf0..617cb59 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -57,18 +57,47 @@ /* Local Prototypes */ /********************/ -/* 'simple' section callbacks */ -static H5FS_section_info_t *H5MF_sect_simple_deserialize(const H5FS_section_class_t *cls, +/* 'simple/small/large' section callbacks */ +static H5FS_section_info_t *H5MF_sect_deserialize(const H5FS_section_class_t *cls, hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size, unsigned *des_flags); +static herr_t H5MF_sect_valid(const H5FS_section_class_t *cls, + const H5FS_section_info_t *sect, hid_t dxpl_id); +static H5FS_section_info_t *H5MF_sect_split(H5FS_section_info_t *sect, + hsize_t frag_size); + + +/* 'simple' section callbacks */ static htri_t H5MF_sect_simple_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5MF_sect_simple_merge(H5FS_section_info_t *sect1, +static herr_t H5MF_sect_simple_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); -static herr_t H5MF_sect_simple_valid(const H5FS_section_class_t *cls, - const H5FS_section_info_t *sect, hid_t dxpl_id); -static H5FS_section_info_t *H5MF_sect_simple_split(H5FS_section_info_t *sect, - hsize_t frag_size); +static htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, + void *udata); + + +/* 'small' section callbacks */ +static herr_t H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata); +static htri_t H5MF_sect_small_can_merge(const H5FS_section_info_t *sect1, + const H5FS_section_info_t *sect2, void *udata); +static herr_t H5MF_sect_small_merge(H5FS_section_info_t **sect1, + H5FS_section_info_t *sect2, void *udata); +static htri_t H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_small_shrink(H5FS_section_info_t **_sect, + void *udata); + +/* 'large' section callbacks */ +static htri_t H5MF_sect_large_can_merge(const H5FS_section_info_t *sect1, + const H5FS_section_info_t *sect2, void *udata); +static herr_t H5MF_sect_large_merge(H5FS_section_info_t **sect1, + H5FS_section_info_t *sect2, void *udata); +static htri_t H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_large_shrink(H5FS_section_info_t **_sect, + void *udata); /*********************/ /* Package Variables */ @@ -89,17 +118,68 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Object methods */ NULL, /* Add section */ NULL, /* Serialize section */ - H5MF_sect_simple_deserialize, /* Deserialize section */ + H5MF_sect_deserialize, /* Deserialize section */ H5MF_sect_simple_can_merge, /* Can sections merge? */ H5MF_sect_simple_merge, /* Merge sections */ H5MF_sect_simple_can_shrink, /* Can section shrink container?*/ H5MF_sect_simple_shrink, /* Shrink container w/section */ - H5MF_sect_simple_free, /* Free section */ - H5MF_sect_simple_valid, /* Check validity of section */ - H5MF_sect_simple_split, /* Split section node for alignment */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ NULL, /* Dump debugging for section */ }}; +/* Class info for "small" free space sections */ +H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{ + /* Class variables */ + H5MF_FSPACE_SECT_SMALL, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ + + /* Class methods */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ + + /* Object methods */ + H5MF_sect_small_add, /* Add section */ + NULL, /* Serialize section */ + H5MF_sect_deserialize, /* Deserialize section */ + H5MF_sect_small_can_merge, /* Can sections merge? */ + H5MF_sect_small_merge, /* Merge sections */ + H5MF_sect_small_can_shrink, /* Can section shrink container?*/ + H5MF_sect_small_shrink, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ +}}; + +/* Class info for "large" free space sections */ +H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1] = {{ + /* Class variables */ + H5MF_FSPACE_SECT_LARGE, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ + + /* Class methods */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ + + /* Object methods */ + NULL, /* Add section */ + NULL, /* Serialize section */ + H5MF_sect_deserialize, /* Deserialize section */ + H5MF_sect_large_can_merge, /* Can sections merge? */ + H5MF_sect_large_merge, /* Merge sections */ + H5MF_sect_large_can_shrink, /* Can section shrink container?*/ + H5MF_sect_large_shrink, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ +}}; /*****************************/ /* Library Private Variables */ @@ -113,12 +193,15 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Declare a free list to manage the H5MF_free_section_t struct */ H5FL_DEFINE(H5MF_free_section_t); +/* + * "simple/small/large" section callbacks + */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_new + * Function: H5MF_sect_new * - * Purpose: Create a new 'simple' section and return it to the caller + * Purpose: Create a new section of "ctype" and return it to the caller * * Return: Pointer to new section on success/NULL on failure * @@ -129,9 +212,9 @@ H5FL_DEFINE(H5MF_free_section_t); *------------------------------------------------------------------------- */ H5MF_free_section_t * -H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) +H5MF_sect_new(unsigned ctype, haddr_t sect_off, hsize_t sect_size) { - H5MF_free_section_t *sect = NULL; /* 'Simple' free space section to add */ + H5MF_free_section_t *sect; /* 'Simple' free space section to add */ H5MF_free_section_t *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -148,7 +231,7 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) sect->sect_info.size = sect_size; /* Set the section's class & state */ - sect->sect_info.type = H5MF_FSPACE_SECT_SIMPLE; + sect->sect_info.type = ctype; sect->sect_info.state = H5FS_SECT_LIVE; /* Set return value */ @@ -156,13 +239,43 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_sect_simple_new() */ +} /* end H5MF_sect_new() */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_deserialize + * Function: H5MF_sect_free * - * Purpose: Deserialize a buffer into a "live" single section + * Purpose: Free a 'simple/small/large' section node + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 8, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_sect_free(H5FS_section_info_t *_sect) +{ + H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(sect); + + /* Release the section */ + sect = H5FL_FREE(H5MF_free_section_t, sect); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sect_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_deserialize + * + * Purpose: Deserialize a buffer into a "live" section * * Return: Success: non-negative * Failure: negative @@ -173,7 +286,7 @@ done: *------------------------------------------------------------------------- */ static H5FS_section_info_t * -H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, +H5MF_sect_deserialize(const H5FS_section_class_t *cls, hid_t H5_ATTR_UNUSED dxpl_id, const uint8_t H5_ATTR_UNUSED *buf, haddr_t sect_addr, hsize_t sect_size, unsigned H5_ATTR_UNUSED *des_flags) { @@ -183,11 +296,12 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, FUNC_ENTER_NOAPI_NOINIT /* Check arguments. */ + HDassert(cls); HDassert(H5F_addr_defined(sect_addr)); HDassert(sect_size); /* Create free space section for block */ - if(NULL == (sect = H5MF_sect_simple_new(sect_addr, sect_size))) + if(NULL == (sect = H5MF_sect_new(cls->type, sect_addr, sect_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") /* Set return value */ @@ -195,10 +309,80 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_sect_simple_deserialize() */ +} /* H5MF_sect_deserialize() */ /*------------------------------------------------------------------------- + * Function: H5MF_sect_valid + * + * Purpose: Check the validity of a section + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 8, 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls, + const H5FS_section_info_t +#ifdef NDEBUG + H5_ATTR_UNUSED +#endif /* NDEBUG */ + *_sect, hid_t H5_ATTR_UNUSED dxpl_id) +{ +#ifndef NDEBUG + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ +#endif /* NDEBUG */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(sect); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sect_valid() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_split + * + * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section + * SECT's addr and size are updated to point to the aligned section + * + * Return: Success: the fragment for aligning sect + * Failure: null + * + * Programmer: Vailin Choi, July 29, 2008 + * + *------------------------------------------------------------------------- + */ +static H5FS_section_info_t * +H5MF_sect_split(H5FS_section_info_t *sect, hsize_t frag_size) +{ + H5MF_free_section_t *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Allocate space for new section */ + if(NULL == (ret_value = H5MF_sect_new(sect->type, sect->addr, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") + + /* Set new section's info */ + sect->addr += frag_size; + sect->size -= frag_size; + +done: + FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value) +} /* end H5MF_sect_split() */ + +/* + * "simple" section callbacks + */ + +/*------------------------------------------------------------------------- * Function: H5MF_sect_simple_can_merge * * Purpose: Can two sections of this type merge? @@ -252,10 +436,10 @@ H5MF_sect_simple_can_merge(const H5FS_section_info_t *_sect1, *------------------------------------------------------------------------- */ static herr_t -H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5MF_sect_simple_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) { - H5MF_free_section_t *sect1 = (H5MF_free_section_t *)_sect1; /* File free section */ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ herr_t ret_value = SUCCEED; /* Return value */ @@ -263,16 +447,16 @@ H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); HDassert(sect2); HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)sect2) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") done: @@ -293,7 +477,7 @@ done: * *------------------------------------------------------------------------- */ -htri_t +static htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, void *_udata) { const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ @@ -392,7 +576,7 @@ done: * *------------------------------------------------------------------------- */ -herr_t +static herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) { H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ @@ -411,8 +595,8 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) /* Sanity check */ HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); - /* Release section's space at EOA with file driver */ - if(H5FD_free(udata->f->shared->lf, udata->dxpl_id, udata->alloc_type, udata->f, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) + /* Release section's space at EOA */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") } /* end if */ else { @@ -427,7 +611,7 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) /* Check for freeing section */ if(udata->shrink != H5MF_SHRINK_SECT_ABSORB_AGGR) { /* Free section */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)*sect) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") /* Mark section as freed, for free space manager */ @@ -438,100 +622,474 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5MF_sect_simple_shrink() */ +/* + * "small" section callbacks + */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_add + * + * Purpose: Perform actions on a small "meta" action before adding it to the free space manager: + * 1) Drop the section if it is at page end and its size <= page end threshold + * 2) Adjust section size to include page end threshold if + * (section size + threshold) is at page end + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) +{ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* Fractal heap free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t sect_end; + hsize_t rem, prem; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Do not adjust the section raw data or global heap data */ + if(udata->alloc_type == H5FD_MEM_DRAW || udata->alloc_type == H5FD_MEM_GHEAP) + HGOTO_DONE(ret_value); + + sect_end = (*sect)->sect_info.addr + (*sect)->sect_info.size; + rem = sect_end % udata->f->shared->fs_page_size; + prem = udata->f->shared->fs_page_size - rem; + + /* Drop the section if it is at page end and its size is <= pgend threshold */ + if(!rem && (*sect)->sect_info.size <= H5F_PGEND_META_THRES(udata->f) && (*flags & H5FS_ADD_RETURNED_SPACE)) { + if(H5MF_sect_free((H5FS_section_info_t *)(*sect)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + *sect = NULL; + *flags &= (unsigned)~H5FS_ADD_RETURNED_SPACE; + *flags |= H5FS_PAGE_END_NO_ADD; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section is dropped\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + /* Adjust the section if it is not at page end but its size + pgend threshold is at page end */ + else + if(prem <= H5F_PGEND_META_THRES(udata->f)) { + (*sect)->sect_info.size += prem; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section is adjusted {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_add() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_can_shrink + * + * Purpose: Can this section shrink the container? + * + * Note: A small section is allowed to shrink only at closing. + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect, void *_udata) +{ + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t eoa; /* End of address space in the file */ + haddr_t end; /* End of section to extend */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect); + HDassert(udata); + HDassert(udata->f); + + /* Retrieve the end of the file's address space */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* Compute address of end of section to check */ + end = sect->sect_info.addr + sect->sect_info.size; + + /* Check if the section is exactly at the end of the allocated space in the file */ + if(H5F_addr_eq(end, eoa) && sect->sect_info.size == udata->f->shared->fs_page_size) { + udata->shrink = H5MF_SHRINK_EOA; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Indicate shrinking can occur */ + HGOTO_DONE(TRUE) + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_can_shrink() */ + /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_free + * Function: H5MF_sect_small_shrink * - * Purpose: Free a 'single' section node + * Purpose: Shrink container with section * * Return: Success: non-negative - * Failure: negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 * - * Programmer: Quincey Koziol - * Tuesday, January 8, 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_small_shrink(H5FS_section_info_t **_sect, void *_udata) +{ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect); + HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(udata); + HDassert(udata->f); + HDassert(udata->shrink == H5MF_SHRINK_EOA); + HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); + + /* Release section's space at EOA */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + + /* Free section */ + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + + /* Mark section as freed, for free space manager */ + *sect = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_shrink() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_can_merge + * + * Purpose: Can two sections of this type merge? + * + * Note: Second section must be "after" first section + * The "merged" section cannot cross page boundary. + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ -herr_t -H5MF_sect_simple_free(H5FS_section_info_t *_sect) +static htri_t +H5MF_sect_small_can_merge(const H5FS_section_info_t *_sect1, + const H5FS_section_info_t *_sect2, void *_udata) { - H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */ + const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */ + const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR /* Check arguments. */ - HDassert(sect); + HDassert(sect1); + HDassert(sect2); + HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */ + HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)); - /* Release the section */ - sect = H5FL_FREE(H5MF_free_section_t, sect); + /* Check if second section adjoins first section */ + ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr); + if(ret_value > 0) + /* If they are on different pages, couldn't merge */ + if((sect1->sect_info.addr / udata->f->shared->fs_page_size) != (((sect2->sect_info.addr + sect2->sect_info.size - 1) / udata->f->shared->fs_page_size))) + ret_value = FALSE; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sect_simple_free() */ +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_can_merge() */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_valid + * Function: H5MF_sect_small_merge * - * Purpose: Check the validity of a section + * Purpose: Merge two sections of this type + * + * Note: Second section always merges into first node. + * If the size of the "merged" section is equal to file space page size, + * free the section. * * Return: Success: non-negative * Failure: negative * - * Programmer: Quincey Koziol - * Tuesday, January 8, 2008 + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ static herr_t -H5MF_sect_simple_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls, - const H5FS_section_info_t -#ifdef NDEBUG - H5_ATTR_UNUSED -#endif /* NDEBUG */ - *_sect, hid_t H5_ATTR_UNUSED dxpl_id) +H5MF_sect_small_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, + void *_udata) { -#ifndef NDEBUG - const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ -#endif /* NDEBUG */ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ + H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect1); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(sect2); + HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); + + /* Add second section's size to first section */ + (*sect1)->sect_info.size += sect2->sect_info.size; + + if((*sect1)->sect_info.size == udata->f->shared->fs_page_size) { + if(H5MF_xfree(udata->f, udata->alloc_type, udata->dxpl_id, (*sect1)->sect_info.addr, (*sect1)->sect_info.size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section") + + /* Need to free possible metadata page in the PB cache */ + /* This is in response to the data corruption bug from fheap.c with page buffering + page strategy */ + /* Note: Large metadata page bypasses the PB cache */ + /* Note: Update of raw data page (large or small sized) is handled by the PB cache */ + if(udata->f->shared->page_buf != NULL && udata->alloc_type != H5FD_MEM_DRAW) + if(H5PB_remove_entry(udata->f, (*sect1)->sect_info.addr) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section") + + if(H5MF_sect_free((H5FS_section_info_t *)(*sect1)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + *sect1 = NULL; + } /* end if */ + + /* Get rid of second section */ + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_merge() */ + +/* + * "Large" section callbacks + */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_can_merge (same as H5MF_sect_simple_can_merge) + * + * Purpose: Can two sections of this type merge? + * + * Note: Second section must be "after" first section + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_large_can_merge(const H5FS_section_info_t *_sect1, + const H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) +{ + const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */ + const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR /* Check arguments. */ + HDassert(sect1); + HDassert(sect2); + HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */ + HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)); + + ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_can_merge() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_merge (same as H5MF_sect_simple_merge) + * + * Purpose: Merge two sections of this type + * + * Note: Second section always merges into first node + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_large_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, + void H5_ATTR_UNUSED *_udata) +{ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ + H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect1); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(sect2); + HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); + + /* Add second section's size to first section */ + (*sect1)->sect_info.size += sect2->sect_info.size; + + /* Get rid of second section */ + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_merge() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_can_shrink + * + * Purpose: Can this section shrink the container? + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect, void *_udata) +{ + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t eoa; /* End of address space in the file */ + haddr_t end; /* End of section to extend */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ HDassert(sect); + HDassert(sect->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(udata); + HDassert(udata->f); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sect_simple_valid() */ + /* Retrieve the end of the file's address space */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* Compute address of end of section to check */ + end = sect->sect_info.addr + sect->sect_info.size; + + /* Check if the section is exactly at the end of the allocated space in the file */ + if(H5F_addr_eq(end, eoa) && sect->sect_info.size >= udata->f->shared->fs_page_size) { + /* Set the shrinking type */ + udata->shrink = H5MF_SHRINK_EOA; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Indicate shrinking can occur */ + HGOTO_DONE(TRUE) + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_can_shrink() */ + /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_split + * Function: H5MF_sect_large_shrink * - * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section - * SECT's addr and size are updated to point to the aligned section + * Purpose: Shrink a large-sized section * - * Return: Success: the fragment for aligning sect - * Failure: null + * Return: Success: non-negative + * Failure: negative * - * Programmer: Vailin Choi, July 29, 2008 + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ -static H5FS_section_info_t * -H5MF_sect_simple_split(H5FS_section_info_t *sect, hsize_t frag_size) +static herr_t +H5MF_sect_large_shrink(H5FS_section_info_t **_sect, void *_udata) { - H5MF_free_section_t *ret_value = NULL; /* Return value */ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + hsize_t frag_size = 0; /* Fragment size */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT - /* Allocate space for new section */ - if(NULL == (ret_value = H5MF_sect_simple_new(sect->addr, frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") + /* Check arguments. */ + HDassert(sect); + HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(udata); + HDassert(udata->f); + HDassert(udata->shrink == H5MF_SHRINK_EOA); + HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); + HDassert(H5F_PAGED_AGGR(udata->f)); - /* Set new section's info */ - sect->addr += frag_size; - sect->size -= frag_size; + /* Calculate possible mis-aligned fragment */ + H5MF_EOA_MISALIGN(udata->f, (*sect)->sect_info.addr, udata->f->shared->fs_page_size, frag_size); + + /* Free full pages from EOA */ + /* Retain partial page in the free-space manager so as to keep EOA at page boundary */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr+frag_size, (*sect)->sect_info.size-frag_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + + if(frag_size) /* Adjust section size for the partial page */ + (*sect)->sect_info.size = frag_size; + else { + /* Free section */ + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + + /* Mark section as freed, for free space manager */ + *sect = NULL; + } /* end else */ done: - FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value) -} /* end H5MF_sect_simple_split() */ + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_shrink() */ diff --git a/src/H5Oalloc.c b/src/H5Oalloc.c index 4f98cfa..40c9c38 100644 --- a/src/H5Oalloc.c +++ b/src/H5Oalloc.c @@ -2023,6 +2023,7 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh) /* Second message has been merged, delete it */ if(merged_msg) { H5O_chunk_proxy_t *curr_chk_proxy; /* Chunk that message is in */ + htri_t result; /* Release any information/memory for second message */ H5O_msg_free_mesg(curr_msg2); @@ -2050,6 +2051,13 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh) /* (Don't bother reducing size of message array for now -QAK) */ oh->nmesgs--; + /* The merge null message might span the entire chunk: scan for empty chunk to remove */ + if((result = H5O_remove_empty_chunks(f, dxpl_id, oh)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPACK, FAIL, "can't remove empty chunk") + else if(result > 0) + /* Get out of loop */ + break; + /* If the merged message is too large, shrink the chunk */ if(curr_msg->raw_size >= H5O_MESG_MAX_SIZE) if(H5O_alloc_shrink_chunk(f, dxpl_id, oh, curr_msg->chunkno) < 0) diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c index 65f6aa2..bea99a2 100644 --- a/src/H5Ocache_image.c +++ b/src/H5Ocache_image.c @@ -33,6 +33,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Opkg.h" /* Object headers */ #include "H5MFprivate.h" /* File space management */ @@ -290,9 +291,50 @@ H5O__mdci_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg) HDassert(mesg); /* Free file space for cache image */ - if(H5F_addr_defined(mesg->addr)) - if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block") + if(H5F_addr_defined(mesg->addr)) { + /* The space for the cache image block was allocated directly + * from the VFD layer at the end of file. As this was the + * last file space allocation before shutdown, the cache image + * should still be the last item in the file. + * + * If the hack to work around the self referential free space + * manager issue is in use, file space for the non-empty self + * referential free space managers was also allocated from VFD + * layer at the end of file. Since these allocations directly + * preceeded the cache image allocation they should be directly + * adjacent to the cache image block at the end of file. + * + * In this case, just call H5MF_tidy_self_referential_fsm_hack(). + * + * That routine will float the self referential free space + * managers, and reduce the eoa to its value just prior to + * allocation of space for same. Since the cache image appears + * just after the self referential free space managers, this + * will release the file space for the cache image as well. + * + * Note that in this case, there must not have been any file + * space allocations / deallocations prior to the free of the + * cache image. Verify this to the extent possible. + * + * If the hack to work around the persistant self referential + * free space manager issue is NOT in use, just call H5MF_xfree() + * to release the cache iamge. In principle, we should be able + * to just reduce the EOA to the base address of the cache + * image block, as there shouldn't be any file space allocation + * before the first metadata cache access. However, given + * time constraints, I don't want to go there now. + */ + if(H5F_FIRST_ALLOC_DEALLOC(f)) { + HDassert(HADDR_UNDEF !=H5F_EOA_PRE_FSM_FSALLOC(f)); + HDassert(H5F_addr_ge(mesg->addr, H5F_EOA_PRE_FSM_FSALLOC(f))); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end if */ + else { + if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block") + } /* end else */ + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c index 938d319..37ec627 100644 --- a/src/H5Ofsinfo.c +++ b/src/H5Ofsinfo.c @@ -23,12 +23,13 @@ * *------------------------------------------------------------------------- */ - +#define H5F_FRIEND /*suppress error about including H5Fpkg */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ #include "H5FLprivate.h" /* Free lists */ #include "H5Opkg.h" /* Object headers */ @@ -66,7 +67,8 @@ const H5O_msg_class_t H5O_MSG_FSINFO[1] = {{ }}; /* Current version of free-space manager info information */ -#define H5O_FSINFO_VERSION 0 +#define H5O_FSINFO_VERSION_0 0 +#define H5O_FSINFO_VERSION_1 1 /* Declare a free list to manage the H5O_fsinfo_t struct */ H5FL_DEFINE_STATIC(H5O_fsinfo_t); @@ -85,12 +87,13 @@ H5FL_DEFINE_STATIC(H5O_fsinfo_t); *------------------------------------------------------------------------- */ static void * -H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh, +H5O_fsinfo_decode(H5F_t *f, hid_t dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, const uint8_t *p) { - H5O_fsinfo_t *fsinfo = NULL; /* free-space manager info */ - H5FD_mem_t type; /* Memory type for iteration */ - void *ret_value = NULL; /* Return value */ + H5O_fsinfo_t *fsinfo = NULL; /* File space info message */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + unsigned vers; /* message version */ + void *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -98,26 +101,83 @@ H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED * HDassert(f); HDassert(p); - /* Version of message */ - if(*p++ != H5O_FSINFO_VERSION) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") - /* Allocate space for message */ if(NULL == (fsinfo = H5FL_CALLOC(H5O_fsinfo_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - fsinfo->strategy = (H5F_file_space_type_t)*p++; /* file space strategy */ - H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* free space section size threshold */ + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo->fs_addr[ptype - 1] = HADDR_UNDEF; - /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */ - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1])); - } /* end if */ - else { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo->fs_addr[type-1] = HADDR_UNDEF; - } /* end else */ + /* Version of message */ + vers = *p++; + + if(vers == H5O_FSINFO_VERSION_0) { + H5F_file_space_type_t strategy; /* Strategy */ + hsize_t threshold; /* Threshold */ + H5FD_mem_t type; /* Memory type for iteration */ + + fsinfo->persist = H5F_FREE_SPACE_PERSIST_DEF; + fsinfo->threshold = H5F_FREE_SPACE_THRESHOLD_DEF; + fsinfo->page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF; + fsinfo->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; + fsinfo->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + strategy = (H5F_file_space_type_t)*p++; /* File space strategy */ + H5F_DECODE_LENGTH(f, p, threshold); /* Free-space section threshold */ + + /* Map version 0 (deprecated) to version 1 message */ + switch(strategy) { + + case H5F_FILE_SPACE_ALL_PERSIST: + fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + fsinfo->persist = TRUE; + fsinfo->threshold = threshold; + if(HADDR_UNDEF == (fsinfo->eoa_pre_fsm_fsalloc = H5F_get_eoa(f, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file size") + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1])); + break; + + case H5F_FILE_SPACE_ALL: + fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + fsinfo->threshold = threshold; + break; + + case H5F_FILE_SPACE_AGGR_VFD: + fsinfo->strategy = H5F_FSPACE_STRATEGY_AGGR; + break; + + case H5F_FILE_SPACE_VFD: + fsinfo->strategy = H5F_FSPACE_STRATEGY_NONE; + break; + + case H5F_FILE_SPACE_NTYPES: + case H5F_FILE_SPACE_DEFAULT: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file space strategy") + } /* end switch */ + + fsinfo->mapped = TRUE; + + } else { + HDassert(vers == H5O_FSINFO_VERSION_1); + + fsinfo->strategy = (H5F_fspace_strategy_t)*p++; /* File space strategy */ + fsinfo->persist = *p++; /* Free-space persist or not */ + H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section threshold */ + + H5F_DECODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ + UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metdata threshold */ + H5F_addr_decode(f, &p, &(fsinfo->eoa_pre_fsm_fsalloc)); /* EOA before free-space header and section info */ + + /* Decode addresses of free space managers, if persisting */ + if(fsinfo->persist) { + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + H5F_addr_decode(f, &p, &(fsinfo->fs_addr[ptype - 1])); + } /* end if */ + + fsinfo->mapped = FALSE; + } /* Set return value */ ret_value = fsinfo; @@ -145,7 +205,7 @@ static herr_t H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, const void *_mesg) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg; - H5FD_mem_t type; /* Memory type for iteration */ + H5F_mem_page_t ptype; /* Memory type for iteration */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -154,14 +214,20 @@ H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, c HDassert(p); HDassert(fsinfo); - *p++ = H5O_FSINFO_VERSION; /* message version */ - *p++ = fsinfo->strategy; /* file space strategy */ - H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* free-space section size threshold */ + *p++ = H5O_FSINFO_VERSION_1; /* message version */ + *p++ = fsinfo->strategy; /* File space strategy */ + *p++ = (unsigned char)fsinfo->persist; /* Free-space persist or not */ + H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section size threshold */ + + H5F_ENCODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ + UINT16ENCODE(p, fsinfo->pgend_meta_thres); /* Page end metadata threshold */ + H5F_addr_encode(f, &p, fsinfo->eoa_pre_fsm_fsalloc); /* EOA before free-space header and section info */ - /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */ - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - H5F_addr_encode(f, &p, fsinfo->fs_addr[type-1]); + /* Store addresses of free-space managers, if persisting */ + if(fsinfo->persist) { + /* Addresses of free-space managers */ + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + H5F_addr_encode(f, &p, fsinfo->fs_addr[ptype - 1]); } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) @@ -224,19 +290,19 @@ static size_t H5O_fsinfo_size(const H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, const void *_mesg) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg; - size_t fs_addr_size = 0; size_t ret_value = 0; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Addresses of free-space managers exist only for H5F_FILE_SPACE_ALL_PERSIST type */ - if(H5F_FILE_SPACE_ALL_PERSIST == fsinfo->strategy) - fs_addr_size = (H5FD_MEM_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f); - - ret_value = 2 /* Version & strategy */ - + (size_t)H5F_SIZEOF_SIZE(f) /* Threshold */ - + fs_addr_size; /* Addresses of free-space managers */ + ret_value = 3 /* Version, strategy & persist */ + + (size_t)H5F_SIZEOF_SIZE(f) /* Free-space section threshold */ + + (size_t)H5F_SIZEOF_SIZE(f) /* File space page size */ + + 2 /* Page end meta threshold */ + + (size_t)H5F_SIZEOF_ADDR(f); + + /* Free-space manager addresses */ + if(fsinfo->persist) + ret_value += (H5F_MEM_PAGE_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_fsinfo_size() */ @@ -282,7 +348,7 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo int indent, int fwidth) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *) _mesg; - H5FD_mem_t type; /* Memory type for iteration */ + H5F_mem_page_t ptype; /* Free-space types for iteration */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -293,16 +359,48 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo HDassert(indent >= 0); HDassert(fwidth >= 0); - HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, - "File space strategy:", fsinfo->strategy); + HDfprintf(stream, "%*s%-*s ", indent, "", fwidth, "File space strategy:"); + switch(fsinfo->strategy) { + case H5F_FSPACE_STRATEGY_FSM_AGGR: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_FSM_AGGR"); + break; + + case H5F_FSPACE_STRATEGY_PAGE: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_PAGE"); + break; + + case H5F_FSPACE_STRATEGY_AGGR: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_AGGR"); + break; + + case H5F_FSPACE_STRATEGY_NONE: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_NONE"); + break; + + case H5F_FSPACE_STRATEGY_NTYPES: + default: + HDfprintf(stream, "%s\n", "unknown"); + } /* end switch */ + + HDfprintf(stream, "%*s%-*s %t\n", indent, "", fwidth, + "Free-space persist:", fsinfo->persist); HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth, - "Free space section threshold:", fsinfo->threshold); + "Free-space section threshold:", fsinfo->threshold); + + HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth, + "File space page size:", fsinfo->page_size); + + HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, + "Page end metadata threshold:", fsinfo->pgend_meta_thres); + + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "eoa_pre_fsm_fsalloc:", fsinfo->eoa_pre_fsm_fsalloc); - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, - "Free space manager address:", fsinfo->fs_addr[type-1]); + if(fsinfo->persist) { + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "Free space manager address:", fsinfo->fs_addr[ptype-1]); } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) diff --git a/src/H5Omessage.c b/src/H5Omessage.c index 376c888..7e6463a 100644 --- a/src/H5Omessage.c +++ b/src/H5Omessage.c @@ -2253,3 +2253,55 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_flush_msgs() */ + +/*------------------------------------------------------------------------- + * Function: H5O_msg_get_flags + * + * Purpose: Queries a message's message flags in the object header + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin; Jan 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags) +{ + H5O_t *oh = NULL; /* Object header to use */ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + H5O_mesg_t *idx_msg; /* Pointer to message to modify */ + unsigned idx; /* Index of message to modify */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(loc); + HDassert(loc->file); + HDassert(H5F_addr_defined(loc->addr)); + HDassert(type_id < NELMTS(H5O_msg_class_g)); + type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ + HDassert(type); + + /* Get the object header */ + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Locate message of correct type */ + for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++) + if(type == idx_msg->type) + break; + + if(idx == oh->nmesgs) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found") + + /* Set return value */ + *flags = idx_msg->flags; + +done: + if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_msg_get_flags() */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 71c512c..f0fbe72 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -791,9 +791,16 @@ typedef unsigned H5O_unknown_t; /* Original message type ID */ * (Data structure in memory) */ typedef struct H5O_fsinfo_t { - H5F_file_space_type_t strategy; /* File space strategy */ - hsize_t threshold; /* Free space section threshold */ - haddr_t fs_addr[H5FD_MEM_NTYPES-1]; /* Addresses of free space managers */ + H5F_fspace_strategy_t strategy; /* File space strategy */ + hbool_t persist; /* Persisting free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + hsize_t page_size; /* For paged aggregation: file space page size */ + size_t pgend_meta_thres; /* For paged aggregation: page end metadata threshold */ + haddr_t eoa_pre_fsm_fsalloc; /* For paged aggregation: the eoa before free-space headers & sinfo */ + haddr_t fs_addr[H5F_MEM_PAGE_NTYPES - 1]; /* 13 addresses of free-space managers */ + /* For non-paged aggregation: only 6 addresses are used */ + hbool_t mapped; /* Not stored */ + /* Indicate the message is mapped from version 0 to version 1 */ } H5O_fsinfo_t; /* @@ -920,6 +927,7 @@ H5_DLL void* H5O_msg_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, const unsigned char *buf); H5_DLL herr_t H5O_msg_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, void *mesg); +H5_DLL herr_t H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags); /* Object metadata flush/refresh routines */ H5_DLL herr_t H5O_flush_common(H5O_loc_t *oloc, hid_t obj_id, hid_t dxpl_id); diff --git a/src/H5PB.c b/src/H5PB.c new file mode 100644 index 0000000..c67ae59 --- /dev/null +++ b/src/H5PB.c @@ -0,0 +1,1533 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5PB.c + * + * Purpose: Page Buffer routines. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#include "H5PBmodule.h" /* This source code file is part of the H5PB module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5PBpkg.h" /* File access */ +#include "H5SLprivate.h" /* Skip List */ + + +/****************/ +/* Local Macros */ +/****************/ +#define H5PB__PREPEND(page_ptr, head_ptr, tail_ptr, len) { \ + if((head_ptr) == NULL) { \ + (head_ptr) = (page_ptr); \ + (tail_ptr) = (page_ptr); \ + } /* end if */ \ + else { \ + (head_ptr)->prev = (page_ptr); \ + (page_ptr)->next = (head_ptr); \ + (head_ptr) = (page_ptr); \ + } /* end else */ \ + (len)++; \ +} /* H5PB__PREPEND() */ + +#define H5PB__REMOVE(page_ptr, head_ptr, tail_ptr, len) { \ + if((head_ptr) == (page_ptr)) { \ + (head_ptr) = (page_ptr)->next; \ + if((head_ptr) != NULL) \ + (head_ptr)->prev = NULL; \ + } /* end if */ \ + else \ + (page_ptr)->prev->next = (page_ptr)->next; \ + if((tail_ptr) == (page_ptr)) { \ + (tail_ptr) = (page_ptr)->prev; \ + if((tail_ptr) != NULL) \ + (tail_ptr)->next = NULL; \ + } /* end if */ \ + else \ + (page_ptr)->next->prev = (page_ptr)->prev; \ + page_ptr->next = NULL; \ + page_ptr->prev = NULL; \ + (len)--; \ +} + +#define H5PB__INSERT_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* insert the entry at the head of the list. */ \ + H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + +#define H5PB__REMOVE_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* remove the entry from the list. */ \ + H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + +#define H5PB__MOVE_TO_TOP_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* Remove entry and insert at the head of the list. */ \ + H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ + H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + + +/******************/ +/* Local Typedefs */ +/******************/ + +/* Iteration context for destroying page buffer */ +typedef struct { + H5PB_t *page_buf; + hbool_t actual_slist; +} H5PB_ud1_t; + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ +static herr_t H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry); +static htri_t H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf, H5FD_mem_t inserted_type); +static herr_t H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ +/* Declare a free list to manage the H5PB_t struct */ +H5FL_DEFINE_STATIC(H5PB_t); + +/* Declare a free list to manage the H5PB_entry_t struct */ +H5FL_DEFINE_STATIC(H5PB_entry_t); + + + +/*------------------------------------------------------------------------- + * Function: H5PB_reset_stats + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Reset statistics collected for the page buffer layer. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_reset_stats(H5PB_t *page_buf) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + + page_buf->accesses[0] = 0; + page_buf->accesses[1] = 0; + page_buf->hits[0] = 0; + page_buf->hits[1] = 0; + page_buf->misses[0] = 0; + page_buf->misses[1] = 0; + page_buf->evictions[0] = 0; + page_buf->evictions[1] = 0; + page_buf->bypasses[0] = 0; + page_buf->bypasses[1] = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_reset_stats() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_get_stats + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Retrieve statistics collected about page accesses for the page buffer layer. + * --accesses: the number of metadata and raw data accesses to the page buffer layer + * --hits: the number of metadata and raw data hits in the page buffer layer + * --misses: the number of metadata and raw data misses in the page buffer layer + * --evictions: the number of metadata and raw data evictions from the page buffer layer + * --bypasses: the number of metadata and raw data accesses that bypass the page buffer layer + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2], unsigned hits[2], + unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + + accesses[0] = page_buf->accesses[0]; + accesses[1] = page_buf->accesses[1]; + hits[0] = page_buf->hits[0]; + hits[1] = page_buf->hits[1]; + misses[0] = page_buf->misses[0]; + misses[1] = page_buf->misses[1]; + evictions[0] = page_buf->evictions[0]; + evictions[1] = page_buf->evictions[1]; + bypasses[0] = page_buf->bypasses[0]; + bypasses[1] = page_buf->bypasses[1]; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_get_stats */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_print_stats() + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Print out statistics collected for the page buffer layer. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_print_stats(const H5PB_t *page_buf) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(page_buf); + + printf("PAGE BUFFER STATISTICS:\n"); + + printf("******* METADATA\n"); + printf("\t Total Accesses: %u\n", page_buf->accesses[0]); + printf("\t Hits: %u\n", page_buf->hits[0]); + printf("\t Misses: %u\n", page_buf->misses[0]); + printf("\t Evictions: %u\n", page_buf->evictions[0]); + printf("\t Bypasses: %u\n", page_buf->bypasses[0]); + printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[0]/(page_buf->accesses[0] - page_buf->bypasses[0]))*100); + printf("*****************\n\n"); + + printf("******* RAWDATA\n"); + printf("\t Total Accesses: %u\n", page_buf->accesses[1]); + printf("\t Hits: %u\n", page_buf->hits[1]); + printf("\t Misses: %u\n", page_buf->misses[1]); + printf("\t Evictions: %u\n", page_buf->evictions[1]); + printf("\t Bypasses: %u\n", page_buf->bypasses[1]); + printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[1]/(page_buf->accesses[1]-page_buf->bypasses[0]))*100); + printf("*****************\n\n"); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_print_stats */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_create + * + * Purpose: Create and setup the PB on the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc) +{ + H5PB_t *page_buf = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + + /* Check args */ + if(f->shared->fs_strategy != H5F_FSPACE_STRATEGY_PAGE) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "Enabling Page Buffering requires PAGE file space strategy") + /* round down the size if it is larger than the page size */ + else if(size > f->shared->fs_page_size) { + hsize_t temp_size; + + temp_size = (size / f->shared->fs_page_size) * f->shared->fs_page_size; + H5_CHECKED_ASSIGN(size, size_t, temp_size, hsize_t); + } /* end if */ + else if(0 != size % f->shared->fs_page_size) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "Page Buffer size must be >= to the page size") + + /* Allocate the new page buffering structure */ + if(NULL == (page_buf = H5FL_CALLOC(H5PB_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + page_buf->max_size = size; + H5_CHECKED_ASSIGN(page_buf->page_size, size_t, f->shared->fs_page_size, hsize_t); + page_buf->min_meta_perc = page_buf_min_meta_perc; + page_buf->min_raw_perc = page_buf_min_raw_perc; + + /* Calculate the minimum page count for metadata and raw data + * based on the fractions provided + */ + page_buf->min_meta_count = (unsigned)((size * page_buf_min_meta_perc) / (f->shared->fs_page_size * 100)); + page_buf->min_raw_count = (unsigned)((size * page_buf_min_raw_perc) / (f->shared->fs_page_size * 100)); + + if(NULL == (page_buf->slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list") + if(NULL == (page_buf->mf_slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list") + + if(NULL == (page_buf->page_fac = H5FL_fac_init(page_buf->page_size))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "can't create page factory") + + f->shared->page_buf = page_buf; + +done: + if(ret_value < 0) { + if(page_buf != NULL) { + if(page_buf->slist_ptr != NULL) + H5SL_close(page_buf->slist_ptr); + if(page_buf->mf_slist_ptr != NULL) + H5SL_close(page_buf->mf_slist_ptr); + if(page_buf->page_fac != NULL) + H5FL_fac_term(page_buf->page_fac); + page_buf = H5FL_FREE(H5PB_t, page_buf); + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_create */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__flush_cb + * + * Purpose: Callback to flush PB skiplist entries. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__flush_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) +{ + H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */ + const H5F_io_info2_t *fio_info = (const H5F_io_info2_t *)_op_data; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(page_entry); + HDassert(fio_info); + + /* Flush the page if it's dirty */ + if(page_entry->is_dirty) + if(H5PB__write_entry(fio_info, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB__flush_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_flush + * + * Purpose: Flush/Free all the PB entries to the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_flush(const H5F_io_info2_t *fio_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(fio_info); + HDassert(fio_info->f); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); + + /* Flush all the entries in the PB skiplist, if we have write access on the file */ + if(fio_info->f->shared->page_buf && (H5F_ACC_RDWR & H5F_INTENT(fio_info->f))) { + H5PB_t *page_buf = fio_info->f->shared->page_buf; + + /* Iterate over all entries in page buffer skip list */ + if(H5SL_iterate(page_buf->slist_ptr, H5PB__flush_cb, (void *)fio_info)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADITER, FAIL, "can't flush page buffer skip list") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_flush */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__dest_cb + * + * Purpose: Callback to free PB skiplist entries. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) +{ + H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */ + H5PB_ud1_t *op_data = (H5PB_ud1_t *)_op_data; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checking */ + HDassert(page_entry); + HDassert(op_data); + HDassert(op_data->page_buf); + + /* Remove entry from LRU list */ + if(op_data->actual_slist) { + H5PB__REMOVE_LRU(op_data->page_buf, page_entry) + page_entry->page_buf_ptr = H5FL_FAC_FREE(op_data->page_buf->page_fac, page_entry->page_buf_ptr); + } /* end if */ + + /* Free page entry */ + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB__dest_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_dest + * + * Purpose: destroy the PB on the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_dest(H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + + /* Destroy page buffer info, if there is any */ + if(f->shared->page_buf) { + H5PB_t *page_buf = f->shared->page_buf; + H5PB_ud1_t op_data; /* Iteration context */ + + /* Set up context info */ + op_data.page_buf = page_buf; + + /* Destroy the skip list containing all the entries in the PB */ + op_data.actual_slist = TRUE; + if(H5SL_destroy(page_buf->slist_ptr, H5PB__dest_cb, &op_data)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list") + + /* Destroy the skip list containing the new entries */ + op_data.actual_slist = FALSE; + if(H5SL_destroy(page_buf->mf_slist_ptr, H5PB__dest_cb, &op_data)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list") + + /* Destroy the page factory */ + if(H5FL_fac_term(page_buf->page_fac) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTRELEASE, FAIL, "can't destroy page buffer page factory") + +#ifdef QAK +H5PB_print_stats(page_buf); +#endif /* QAK */ + + f->shared->page_buf = H5FL_FREE(H5PB_t, page_buf); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_dest */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_add_new_page + * + * Purpose: Add a new page to the new page skip list. This is called + * from the MF layer when a new page is allocated to + * indicate to the page buffer layer that a read of the page + * from the file is not necessary since it's an empty page. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr) +{ + H5PB_t *page_buf = f->shared->page_buf; + H5PB_entry_t *page_entry = NULL; /* pointer to the corresponding page entry */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(page_buf); + + /* If there is an existing page, this means that at some point the + * file free space manager freed and re-allocated a page at the same + * address. No need to do anything here then... + */ + /* MSC - to be safe, might want to dig in the MF layer and remove + * the page when it is freed from this list if it still exists and + * remove this check + */ + if(NULL == H5SL_search(page_buf->mf_slist_ptr, &(page_addr))) { + /* Create the new PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Initialize page fields */ + page_entry->addr = page_addr; + page_entry->type = (H5F_mem_page_t)type; + page_entry->is_dirty = FALSE; + + /* Insert entry in skip list */ + if(H5SL_insert(page_buf->mf_slist_ptr, page_entry, &(page_entry->addr)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Can't insert entry in skip list") + } /* end if */ + +done: + if(ret_value < 0) + if(page_entry) + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_add_new_page */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_update_entry + * + * Purpose: In PHDF5, entries that are written by other processes and just + * marked clean by this process have to have their corresponding + * pages updated if they exist in the page buffer. + * This routine checks and update the pages. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf) +{ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + haddr_t page_addr; + + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + HDassert(size <= page_buf->page_size); + HDassert(buf); + + /* calculate the aligned address of the first page */ + page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* search for the page and update if found */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&page_addr)); + if(page_entry) { + haddr_t offset; + + HDassert(addr + size <= page_addr + page_buf->page_size); + offset = addr - page_addr; + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, size); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_update_entry */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_remove_entry + * + * Purpose: Remove possible metadata entry with ADDR from the PB cache. + * This is in response to the data corruption bug from fheap.c + * with page buffering + page strategy. + * Note: Large metadata page bypasses the PB cache. + * Note: Update of raw data page (large or small sized) is handled by the PB cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Feb 2017 + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_remove_entry(const H5F_t *f, haddr_t addr) +{ + H5PB_t *page_buf = f->shared->page_buf; + H5PB_entry_t *page_entry = NULL; /* pointer to the page entry being searched */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(page_buf); + + /* Search for address in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&addr)); + + /* If found, remove the entry from the PB cache */ + if(page_entry) { + HDassert(page_entry->type != H5F_MEM_PAGE_DRAW); + if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr))) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Page Entry is not in skip list") + + /* Remove from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len); + + page_buf->meta_count--; + + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_remove_entry */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_read + * + * Purpose: Reads in the data from the page containing it if it exists + * in the PB cache; otherwise reads in the page through the VFD. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, + size_t size, void *buf/*out*/) +{ + H5PB_t *page_buf; /* Page buffering info for this file */ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ + haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */ + haddr_t offset; + haddr_t search_addr; /* Address of current page */ + hsize_t num_touched_pages; /* Number of pages accessed */ + size_t access_size; + hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */ + hsize_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(fio_info); + + /* Get pointer to page buffer info for this file */ + page_buf = fio_info->f->shared->page_buf; + +#ifdef H5_HAVE_PARALLEL + if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) { +#if 1 + bypass_pb = TRUE; +#else + /* MSC - why this stopped working ? */ + int mpi_size; + + if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") + if(1 != mpi_size) + bypass_pb = TRUE; +#endif + } /* end if */ +#endif + + /* If page buffering is disabled, or the I/O size is larger than that of a + * single page, or if this is a parallel raw data access, bypass page + * buffering. + */ + if(NULL == page_buf || size >= page_buf->page_size || + (bypass_pb && H5FD_MEM_DRAW == type)) { + if(H5F__accum_read(fio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "read through metadata accumulator failed") + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->bypasses[1] ++; + else + page_buf->bypasses[0] ++; + } /* end if */ + + /* If page buffering is disabled, or if this is a large metadata access, + * or if this is parallel raw data access, we are done here + */ + if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) || + (bypass_pb && H5FD_MEM_DRAW == type)) + HGOTO_DONE(SUCCEED) + } /* end if */ + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->accesses[1]++; + else + page_buf->accesses[0]++; + } /* end if */ + + /* Calculate the aligned address of the first page */ + first_page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* For Raw data calculate the aligned address of the last page and + * the number of pages accessed if more than 1 page is accessed + */ + if(H5FD_MEM_DRAW == type) { + last_page_addr = ((addr + size - 1) / page_buf->page_size) * page_buf->page_size; + + /* How many pages does this write span */ + num_touched_pages = (last_page_addr / page_buf->page_size + 1) - + (first_page_addr / page_buf->page_size); + if(first_page_addr == last_page_addr) { + HDassert(1 == num_touched_pages); + last_page_addr = HADDR_UNDEF; + } /* end if */ + } /* end if */ + /* Otherwise set last page addr to HADDR_UNDEF */ + else { + num_touched_pages = 1; + last_page_addr = HADDR_UNDEF; + } /* end else */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + /* Copy raw data from dirty pages into the read buffer if the read + request spans pages in the page buffer*/ + if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) { + H5SL_node_t *node; + + /* For each touched page in the page buffer, check if it + * exists in the page Buffer and is dirty. If it does, we + * update the buffer with what's in the page so we get the up + * to date data into the buffer after the big read from the file. + */ + node = H5SL_find(page_buf->slist_ptr, (void *)(&first_page_addr)); + for(i = 0; i < num_touched_pages; i++) { + search_addr = i*page_buf->page_size + first_page_addr; + + /* if we still haven't located a starting page, search again */ + if(!node && i!=0) + node = H5SL_find(page_buf->slist_ptr, (void *)(&search_addr)); + + /* if the current page is in the Page Buffer, do the updates */ + if(node) { + page_entry = (H5PB_entry_t *)H5SL_item(node); + + HDassert(page_entry); + + /* If the current page address falls out of the access + block, then there are no more pages to go over */ + if(page_entry->addr >= addr + size) + break; + + HDassert(page_entry->addr == search_addr); + + if(page_entry->is_dirty) { + /* special handling for the first page if it is not a full page access */ + if(i == 0 && first_page_addr != addr) { + offset = addr - first_page_addr; + HDassert(page_buf->page_size > offset); + + HDmemcpy(buf, (uint8_t *)page_entry->page_buf_ptr + offset, + page_buf->page_size - (size_t)offset); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + /* special handling for the last page if it is not a full page access */ + else if(num_touched_pages > 1 && i == num_touched_pages-1 && search_addr < addr+size) { + offset = (num_touched_pages-2)*page_buf->page_size + + (page_buf->page_size - (addr - first_page_addr)); + + HDmemcpy((uint8_t *)buf + offset, page_entry->page_buf_ptr, + (size_t)((addr + size) - last_page_addr)); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end else-if */ + /* copy the entire fully accessed pages */ + else { + offset = i*page_buf->page_size; + + HDmemcpy((uint8_t *)buf+(i*page_buf->page_size) , page_entry->page_buf_ptr, + page_buf->page_size); + } /* end else */ + } /* end if */ + node = H5SL_next(node); + } /* end if */ + } /* end for */ + } /* end if */ + else { + /* A raw data access could span 1 or 2 PB entries at this point so + we need to handle that */ + HDassert(1 == num_touched_pages || 2 == num_touched_pages); + for(i = 0 ; i < num_touched_pages; i++) { + haddr_t buf_offset; + + /* Calculate the aligned address of the page to search for it in the skip list */ + search_addr = (0==i ? first_page_addr : last_page_addr); + + /* Calculate the access size if the access spans more than 1 page */ + if(1 == num_touched_pages) + access_size = size; + else + access_size = (0 == i ? (size_t)((first_page_addr + page_buf->page_size) - addr) : (size - access_size)); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + + /* if found */ + if(page_entry) { + offset = (0 == i ? addr - page_entry->addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* copy the requested data from the page into the input buffer */ + HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)page_entry->page_buf_ptr + offset, access_size); + + /* Update LRU */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* if not found */ + else { + void *new_page_buf = NULL; + size_t page_size = page_buf->page_size; + haddr_t eoa; + + /* make space for new entry */ + if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) { + htri_t can_make_space; + + /* check if we can make space in page buffer */ + if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") + + /* if make_space returns 0, then we can't use the page + buffer for this I/O and we need to bypass */ + if(0 == can_make_space) { + /* make space can't return FALSE on second touched page since the first is of the same type */ + HDassert(0 == i); + + /* read entire block from VFD and return */ + if(H5FD_read(&fdio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Break out of loop */ + break; + } /* end if */ + } /* end if */ + + /* Read page from VFD */ + if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + + /* Read page through the VFD layer, but make sure we don't read past the EOA. */ + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the entire page falls outside the EOA, then fail */ + if(search_addr > eoa) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "reading an entire page that is outside the file EOA") + + /* Adjust the read size to not go beyond the EOA */ + if(search_addr + page_size > eoa) + page_size = (size_t)(eoa - search_addr); + + /* Read page from VFD */ + if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Copy the requested data from the page into the input buffer */ + offset = (0 == i ? addr - search_addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)new_page_buf + offset, access_size); + + /* Create the new PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + page_entry->page_buf_ptr = new_page_buf; + page_entry->addr = search_addr; + page_entry->type = (H5F_mem_page_t)type; + page_entry->is_dirty = FALSE; + + /* Insert page into PB */ + if(H5PB__insert_entry(page_buf, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer") + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->misses[1]++; + else + page_buf->misses[0]++; + } /* end else */ + } /* end for */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_write + * + * Purpose: Write data into the Page Buffer. If the page exists in the + * cache, update it; otherwise read it from disk, update it, and + * insert into cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_write(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, + size_t size, const void *buf) +{ + H5PB_t *page_buf; /* Page buffering info for this file */ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ + haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */ + haddr_t offset; + haddr_t search_addr; /* Address of current page */ + hsize_t num_touched_pages; /* Number of pages accessed */ + size_t access_size; + hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */ + hsize_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(fio_info); + HDassert(fio_info->f); + + /* Get pointer to page buffer info for this file */ + page_buf = fio_info->f->shared->page_buf; + +#ifdef H5_HAVE_PARALLEL + if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) { +#if 1 + bypass_pb = TRUE; +#else + /* MSC - why this stopped working ? */ + int mpi_size; + + if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") + if(1 != mpi_size) + bypass_pb = TRUE; +#endif + } /* end if */ +#endif + + /* If page buffering is disabled, or the I/O size is larger than that of a + * single page, or if this is a parallel raw data access, bypass page + * buffering. + */ + if(NULL == page_buf || size >= page_buf->page_size || bypass_pb) { + if(H5F__accum_write(fio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->bypasses[1]++; + else + page_buf->bypasses[0]++; + } /* end if */ + + /* If page buffering is disabled, or if this is a large metadata access, + * or if this is a parallel raw data access, we are done here + */ + if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) || + (bypass_pb && H5FD_MEM_DRAW == type)) + HGOTO_DONE(SUCCEED) + +#ifdef H5_HAVE_PARALLEL + if(bypass_pb) { + if(H5PB_update_entry(page_buf, addr, size, buf) > 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTUPDATE, FAIL, "failed to update PB with metadata cache") + HGOTO_DONE(SUCCEED) + } /* end if */ +#endif + } /* end if */ + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->accesses[1]++; + else + page_buf->accesses[0]++; + } /* end if */ + + /* Calculate the aligned address of the first page */ + first_page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* For raw data calculate the aligned address of the last page and + * the number of pages accessed if more than 1 page is accessed + */ + if(H5FD_MEM_DRAW == type) { + last_page_addr = (addr + size - 1) / page_buf->page_size * page_buf->page_size; + + /* how many pages does this write span */ + num_touched_pages = (last_page_addr/page_buf->page_size + 1) - + (first_page_addr / page_buf->page_size); + if(first_page_addr == last_page_addr) { + HDassert(1 == num_touched_pages); + last_page_addr = HADDR_UNDEF; + } /* end if */ + } /* end if */ + /* Otherwise set last page addr to HADDR_UNDEF */ + else { + num_touched_pages = 1; + last_page_addr = HADDR_UNDEF; + } /* end else */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + /* Check if existing pages for raw data need to be updated since raw data access is not atomic */ + if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) { + /* For each touched page, check if it exists in the page buffer, and + * update it with the data in the buffer to keep it up to date + */ + for(i = 0; i < num_touched_pages; i++) { + search_addr = i * page_buf->page_size + first_page_addr; + + /* Special handling for the first page if it is not a full page update */ + if(i == 0 && first_page_addr != addr) { + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + offset = addr - first_page_addr; + HDassert(page_buf->page_size > offset); + + /* Update page's data */ + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, page_buf->page_size - (size_t)offset); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + } /* end if */ + /* Special handling for the last page if it is not a full page update */ + else if(num_touched_pages > 1 && i == (num_touched_pages - 1) && + (search_addr + page_buf->page_size) != (addr + size)) { + HDassert(search_addr+page_buf->page_size > addr+size); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + offset = (num_touched_pages - 2) * page_buf->page_size + + (page_buf->page_size - (addr - first_page_addr)); + + /* Update page's data */ + HDmemcpy(page_entry->page_buf_ptr, (const uint8_t *)buf + offset, + (size_t)((addr + size) - last_page_addr)); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + } /* end else-if */ + /* Discard all fully written pages from the page buffer */ + else { + page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + /* Remove from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + + /* Decrement page count of appropriate type */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count--; + else + page_buf->meta_count--; + + /* Free page info */ + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + } /* end if */ + } /* end else */ + } /* end for */ + } /* end if */ + else { + /* An access could span 1 or 2 PBs at this point so we need to handle that */ + HDassert(1 == num_touched_pages || 2 == num_touched_pages); + for(i = 0; i < num_touched_pages; i++) { + haddr_t buf_offset; + + /* Calculate the aligned address of the page to search for it in the skip list */ + search_addr = (0 == i ? first_page_addr : last_page_addr); + + /* Calculate the access size if the access spans more than 1 page */ + if(1 == num_touched_pages) + access_size = size; + else + access_size = (0 == i ? (size_t)(first_page_addr + page_buf->page_size - addr) : (size - access_size)); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + + /* If found */ + if(page_entry) { + offset = (0 == i ? addr - page_entry->addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* Copy the requested data from the input buffer into the page */ + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, (const uint8_t *)buf + buf_offset, access_size); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* If not found */ + else { + void *new_page_buf; + size_t page_size = page_buf->page_size; + + /* Make space for new entry */ + if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) { + htri_t can_make_space; + + /* Check if we can make space in page buffer */ + if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") + + /* If make_space returns 0, then we can't use the page + * buffer for this I/O and we need to bypass + */ + if(0 == can_make_space) { + HDassert(0 == i); + + /* Write to VFD and return */ + if(H5FD_write(&fdio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "driver write request failed") + + /* Break out of loop */ + break; + } /* end if */ + } /* end if */ + + /* Don't bother searching if there is no write access */ + if(H5F_ACC_RDWR & H5F_INTENT(fio_info->f)) + /* Lookup & remove the page from the new skip list page if + * it exists to see if this is a new page from the MF layer + */ + page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->mf_slist_ptr, (void *)(&search_addr)); + + /* Calculate offset into the buffer of the page and the user buffer */ + offset = (0 == i ? addr - search_addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* If found, then just update the buffer pointer to the newly allocate buffer */ + if(page_entry) { + /* Allocate space for the page buffer */ + if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + HDmemset(new_page_buf, 0, (size_t)offset); + HDmemset((uint8_t *)new_page_buf + offset + access_size, 0, page_size - ((size_t)offset + access_size)); + + page_entry->page_buf_ptr = new_page_buf; + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* Otherwise read page through the VFD layer, but make sure we don't read past the EOA. */ + else { + haddr_t eoa, eof = HADDR_UNDEF; + + /* Allocate space for the page buffer */ + if(NULL == (new_page_buf = H5FL_FAC_CALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + + /* Create the new loaded PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed") + + page_entry->page_buf_ptr = new_page_buf; + page_entry->addr = search_addr; + page_entry->type = (H5F_mem_page_t)type; + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the entire page falls outside the EOA, then fail */ + if(search_addr > eoa) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "writing to a page that is outside the file EOA") + + /* Retrieve the 'eof' for the file - The MPI-VFD EOF + * returned will most likely be HADDR_UNDEF, so skip + * that check. + */ + if(!H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) + if(HADDR_UNDEF == (eof = H5FD_get_eof(fio_info->f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eof request failed") + + /* Adjust the read size to not go beyond the EOA */ + if(search_addr + page_size > eoa) + page_size = (size_t)(eoa - search_addr); + + if(search_addr < eof) { + if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->misses[1]++; + else + page_buf->misses[0]++; + } /* end if */ + } /* end else */ + + /* Copy the requested data from the page into the input buffer */ + HDmemcpy((uint8_t *)new_page_buf + offset, (const uint8_t *)buf+buf_offset, access_size); + + /* Page is dirty now */ + page_entry->is_dirty = TRUE; + + /* Insert page into PB, evicting other pages as necessary */ + if(H5PB__insert_entry(page_buf, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer") + } /* end else */ + } /* end for */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__insert_entry() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Insert the supplied page into the page buffer, both the + * skip list and the LRU. + * + * As best I can tell, this function imposes no limit on the + * number of entries in the page buffer beyond an assertion + * failure it the page count exceeds the limit. + * + * JRM -- 12/22/16 + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Insert entry in skip list */ + if(H5SL_insert(page_buf->slist_ptr, page_entry, &(page_entry->addr)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINSERT, FAIL, "can't insert entry in skip list") + HDassert(H5SL_count(page_buf->slist_ptr) * page_buf->page_size <= page_buf->max_size); + + /* Increment appropriate page count */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count++; + else + page_buf->meta_count++; + + /* Insert entry in LRU */ + H5PB__INSERT_LRU(page_buf, page_entry) + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__insert_entry() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__make_space() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * If necessary and if possible, evict a page from the page + * buffer to make space for the supplied page. Depending on + * the page buffer configuration and contents, and the page + * supplied this may or may not be possible. + * + * JRM -- 12/22/16 + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static htri_t +H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf, + H5FD_mem_t inserted_type) +{ + H5PB_entry_t *page_entry; /* Pointer to page eviction candidate */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(fio_info); + HDassert(page_buf); + + /* Get oldest entry */ + page_entry = page_buf->LRU_tail_ptr; + + if(H5FD_MEM_DRAW == inserted_type) { + /* If threshould is 100% metadata and page buffer is full of + metadata, then we can't make space for raw data */ + if(0 == page_buf->raw_count && page_buf->min_meta_count == page_buf->meta_count) { + HDassert(page_buf->meta_count * page_buf->page_size == page_buf->max_size); + HGOTO_DONE(FALSE) + } /* end if */ + + /* check the metadata threshold before evicting metadata items */ + while(1) { + if(page_entry->prev && H5F_MEM_PAGE_META == page_entry->type && + page_buf->min_meta_count >= page_buf->meta_count) + page_entry = page_entry->prev; + else + break; + } /* end while */ + } /* end if */ + else { + /* If threshould is 100% raw data and page buffer is full of + raw data, then we can't make space for meta data */ + if(0 == page_buf->meta_count && page_buf->min_raw_count == page_buf->raw_count) { + HDassert(page_buf->raw_count * page_buf->page_size == page_buf->max_size); + HGOTO_DONE(FALSE) + } /* end if */ + + /* check the raw data threshold before evicting raw data items */ + while(1) { + if(page_entry->prev && (H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) && + page_buf->min_raw_count >= page_buf->raw_count) + page_entry = page_entry->prev; + else + break; + } /* end while */ + } /* end else */ + + /* Remove from page index */ + if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Tail Page Entry is not in skip list") + + /* Remove entry from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len); + + /* Decrement appropriate page type counter */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count--; + else + page_buf->meta_count--; + + /* Flush page if dirty */ + if(page_entry->is_dirty) + if(H5PB__write_entry(fio_info, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + + /* Update statistics */ + if(page_entry->type == H5F_MEM_PAGE_DRAW || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->evictions[1]++; + else + page_buf->evictions[0]++; + + /* Release page */ + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__make_space() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__write_entry() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry) +{ + haddr_t eoa; /* Current EOA for the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(fio_info); + HDassert(fio_info->f); + HDassert(page_entry); + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, page_entry->type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the starting address of the page is larger than + * the EOA, then the entire page is discarded without writing. + */ + if(page_entry->addr <= eoa) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ + size_t page_size = fio_info->f->shared->page_buf->page_size; + + /* Adjust the page length if it exceeds the EOA */ + if((page_entry->addr + page_size) > eoa) + page_size = (size_t)(eoa - page_entry->addr); + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + if(H5FD_write(&fdio_info, page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + } /* end if */ + + page_entry->is_dirty = FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__write_entry() */ + diff --git a/src/H5PBmodule.h b/src/H5PBmodule.h new file mode 100644 index 0000000..35da3f6 --- /dev/null +++ b/src/H5PBmodule.h @@ -0,0 +1,34 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Saturday, September 12, 2015 + * + * Purpose: This file contains declarations which define macros for the + * H5PB package. Including this header means that the source file + * is part of the H5PB package. + */ +#ifndef _H5PBmodule_H +#define _H5PBmodule_H + +/* Define the proper control macros for the generic FUNC_ENTER/LEAVE and error + * reporting macros. + */ +#define H5PB_MODULE +#define H5_MY_PKG H5PB +#define H5_MY_PKG_ERR H5E_RESOURCE +#define H5_MY_PKG_INIT NO + +#endif /* _H5PBmodule_H */ diff --git a/src/H5PBpkg.h b/src/H5PBpkg.h new file mode 100644 index 0000000..976f18d --- /dev/null +++ b/src/H5PBpkg.h @@ -0,0 +1,60 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if !(defined H5PB_FRIEND || defined H5PB_MODULE) +#error "Do not include this file outside the H5PB package!" +#endif + +#ifndef _H5PBpkg_H +#define _H5PBpkg_H + +/* Get package's private header */ +#include "H5PBprivate.h" + +/* Other private headers needed by this file */ + +/**************************/ +/* Package Private Macros */ +/**************************/ + + +/****************************/ +/* Package Private Typedefs */ +/****************************/ + +typedef struct H5PB_entry_t { + void *page_buf_ptr; /* Pointer to the buffer containing the data */ + haddr_t addr; /* Address of the page in the file */ + H5F_mem_page_t type; /* Type of the page entry (H5F_MEM_PAGE_RAW/META) */ + hbool_t is_dirty; /* Flag indicating whether the page has dirty data or not */ + + /* Fields supporting replacement policies */ + struct H5PB_entry_t *next; /* next pointer in the LRU list */ + struct H5PB_entry_t *prev; /* previous pointer in the LRU list */ +} H5PB_entry_t; + + +/*****************************/ +/* Package Private Variables */ +/*****************************/ + + +/******************************/ +/* Package Private Prototypes */ +/******************************/ + + +#endif /* _H5PBpkg_H */ + diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h new file mode 100644 index 0000000..3b5dcae --- /dev/null +++ b/src/H5PBprivate.h @@ -0,0 +1,108 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5PBprivate.h + * June 2014 + * Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ + +#ifndef _H5PBprivate_H +#define _H5PBprivate_H + +/* Include package's public header */ +#ifdef NOT_YET +#include "H5PBpublic.h" +#endif /* NOT_YET */ + +/* Private headers needed by this header */ +#include "H5private.h" /* Generic Functions */ +#include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5SLprivate.h" /* Skip List */ + + +/**************************/ +/* Library Private Macros */ +/**************************/ + + +/****************************/ +/* Library Private Typedefs */ +/****************************/ + +/* Forward declaration for a page buffer entry */ +struct H5PB_entry_t; + +/* Typedef for the main structure for the page buffer */ +typedef struct H5PB_t { + size_t max_size; /* The total page buffer size */ + size_t page_size; /* Size of a single page */ + unsigned min_meta_perc; /* Minimum ratio of metadata entries required before evicting meta entries */ + unsigned min_raw_perc; /* Minimum ratio of raw data entries required before evicting raw entries */ + unsigned meta_count; /* Number of entries for metadata */ + unsigned raw_count; /* Number of entries for raw data */ + unsigned min_meta_count; /* Minimum # of entries for metadata */ + unsigned min_raw_count; /* Minimum # of entries for raw data */ + + H5SL_t *slist_ptr; /* Skip list with all the active page entries */ + H5SL_t *mf_slist_ptr; /* Skip list containing newly allocated page entries inserted from the MF layer */ + + size_t LRU_list_len; /* Number of entries in the LRU (identical to slist_ptr count) */ + struct H5PB_entry_t *LRU_head_ptr; /* Head pointer of the LRU */ + struct H5PB_entry_t *LRU_tail_ptr; /* Tail pointer of the LRU */ + + H5FL_fac_head_t *page_fac; /* Factory for allocating pages */ + + /* Statistics */ + unsigned accesses[2]; + unsigned hits[2]; + unsigned misses[2]; + unsigned evictions[2]; + unsigned bypasses[2]; +} H5PB_t; + +/*****************************/ +/* Library-private Variables */ +/*****************************/ + + +/***************************************/ +/* Library-private Function Prototypes */ +/***************************************/ + +/* General routines */ +H5_DLL herr_t H5PB_create(H5F_t *file, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc); +H5_DLL herr_t H5PB_flush(const H5F_io_info2_t *fio_info); +H5_DLL herr_t H5PB_dest(H5F_t *file); +H5_DLL herr_t H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr); +H5_DLL herr_t H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf); +H5_DLL herr_t H5PB_remove_entry(const H5F_t *f, haddr_t addr); +H5_DLL herr_t H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, + haddr_t addr, size_t size, void *buf/*out*/); +H5_DLL herr_t H5PB_write(const H5F_io_info2_t *f, H5FD_mem_t type, haddr_t addr, + size_t size, const void *buf); + +/* Statistics routines */ +H5_DLL herr_t H5PB_reset_stats(H5PB_t *page_buf); +H5_DLL herr_t H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2], + unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); +H5_DLL herr_t H5PB_print_stats(const H5PB_t *page_buf); + +#endif /* !_H5PBprivate_H */ + diff --git a/src/H5Pdeprec.c b/src/H5Pdeprec.c index ecf2bea..d225dfa 100644 --- a/src/H5Pdeprec.c +++ b/src/H5Pdeprec.c @@ -488,5 +488,135 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_version() */ +/*------------------------------------------------------------------------- + * Function: H5Pset_file_space + * + * Purpose: It is mapped to H5Pset_file_space_strategy(). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Jan 2017 + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold) +{ + + H5F_fspace_strategy_t new_strategy; /* File space strategy type */ + hbool_t new_persist = H5F_FREE_SPACE_PERSIST_DEF; /* Persisting free-space or not */ + hsize_t new_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; /* Free-space section threshold */ + H5F_file_space_type_t in_strategy = strategy; /* Input strategy */ + hsize_t in_threshold = threshold; /* Input threshold */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "iFfh", plist_id, strategy, threshold); + + if((unsigned)in_strategy >= H5F_FILE_SPACE_NTYPES) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy") + /* + * For 1.10.0 H5Pset_file_space: + * If strategy is zero, the property is not changed; + * the existing strategy is retained. + * If threshold is zero, the property is not changed; + * the existing threshold is retained. + */ + if(!in_strategy) + H5Pget_file_space(plist_id, &in_strategy, NULL); + if(!in_threshold) + H5Pget_file_space(plist_id, NULL, &in_threshold); + + switch(in_strategy) { + case H5F_FILE_SPACE_ALL_PERSIST: + new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + new_persist = TRUE; + new_threshold = in_threshold; + break; + + case H5F_FILE_SPACE_ALL: + new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + new_threshold = in_threshold; + break; + + case H5F_FILE_SPACE_AGGR_VFD: + new_strategy = H5F_FSPACE_STRATEGY_AGGR; + break; + + case H5F_FILE_SPACE_VFD: + new_strategy = H5F_FSPACE_STRATEGY_NONE; + break; + + case H5F_FILE_SPACE_NTYPES: + case H5F_FILE_SPACE_DEFAULT: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy") + } + + if(H5Pset_file_space_strategy(plist_id, new_strategy, new_persist, new_threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_file_space() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_space + * + * Purpose: It is mapped to H5Pget_file_space_strategy(). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Jan 2017 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold) +{ + H5F_fspace_strategy_t new_strategy; /* File space strategy type */ + hbool_t new_persist; /* Persisting free-space or not */ + hsize_t new_threshold; /* Free-space section threshold */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold); + + /* Get current file space info */ + if(H5Pget_file_space_strategy(plist_id, &new_strategy, &new_persist, &new_threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy") + + /* Get value(s) */ + if(strategy) { + switch(new_strategy) { + + case H5F_FSPACE_STRATEGY_FSM_AGGR: + if(new_persist) + *strategy = H5F_FILE_SPACE_ALL_PERSIST; + else + *strategy = H5F_FILE_SPACE_ALL; + break; + + case H5F_FSPACE_STRATEGY_AGGR: + *strategy = H5F_FILE_SPACE_AGGR_VFD; + break; + + case H5F_FSPACE_STRATEGY_NONE: + *strategy = H5F_FILE_SPACE_VFD; + break; + + case H5F_FSPACE_STRATEGY_PAGE: + case H5F_FSPACE_STRATEGY_NTYPES: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy") + } + } + + if(threshold) + *threshold = new_threshold; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_file_space() */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 7ea0fe0..7865fdf 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -80,18 +80,18 @@ #define H5F_ACS_PREEMPT_READ_CHUNKS_DEC H5P__decode_double /* Definition for threshold for alignment */ #define H5F_ACS_ALIGN_THRHD_SIZE sizeof(hsize_t) -#define H5F_ACS_ALIGN_THRHD_DEF 1 +#define H5F_ACS_ALIGN_THRHD_DEF H5F_ALIGN_THRHD_DEF #define H5F_ACS_ALIGN_THRHD_ENC H5P__encode_hsize_t #define H5F_ACS_ALIGN_THRHD_DEC H5P__decode_hsize_t /* Definition for alignment */ #define H5F_ACS_ALIGN_SIZE sizeof(hsize_t) -#define H5F_ACS_ALIGN_DEF 1 +#define H5F_ACS_ALIGN_DEF H5F_ALIGN_DEF #define H5F_ACS_ALIGN_ENC H5P__encode_hsize_t #define H5F_ACS_ALIGN_DEC H5P__decode_hsize_t /* Definition for minimum metadata allocation block size (when aggregating metadata allocations. */ #define H5F_ACS_META_BLOCK_SIZE_SIZE sizeof(hsize_t) -#define H5F_ACS_META_BLOCK_SIZE_DEF 2048 +#define H5F_ACS_META_BLOCK_SIZE_DEF H5F_META_BLOCK_SIZE_DEF #define H5F_ACS_META_BLOCK_SIZE_ENC H5P__encode_hsize_t #define H5F_ACS_META_BLOCK_SIZE_DEC H5P__decode_hsize_t /* Definition for maximum sieve buffer size (when data sieving @@ -103,7 +103,7 @@ /* Definition for minimum "small data" allocation block size (when aggregating "small" raw data allocations. */ #define H5F_ACS_SDATA_BLOCK_SIZE_SIZE sizeof(hsize_t) -#define H5F_ACS_SDATA_BLOCK_SIZE_DEF 2048 +#define H5F_ACS_SDATA_BLOCK_SIZE_DEF H5F_SDATA_BLOCK_SIZE_DEF #define H5F_ACS_SDATA_BLOCK_SIZE_ENC H5P__encode_hsize_t #define H5F_ACS_SDATA_BLOCK_SIZE_DEC H5P__decode_hsize_t /* Definition for garbage-collect references */ @@ -231,6 +231,21 @@ #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_ENC H5P__facc_cache_image_config_enc #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEC H5P__facc_cache_image_config_dec #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP H5P__facc_cache_image_config_cmp +/* Definition for total size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_SIZE_SIZE sizeof(size_t) +#define H5F_ACS_PAGE_BUFFER_SIZE_DEF 0 +#define H5F_ACS_PAGE_BUFFER_SIZE_ENC H5P__encode_size_t +#define H5F_ACS_PAGE_BUFFER_SIZE_DEC H5P__decode_size_t +/* Definition for minimum metadata size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE sizeof(unsigned) +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF 0 +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC H5P__encode_unsigned +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC H5P__decode_unsigned +/* Definition for minimum raw data size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE sizeof(unsigned) +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF 0 +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC H5P__encode_unsigned +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC H5P__decode_unsigned /******************/ @@ -359,6 +374,9 @@ static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ #endif /* H5_HAVE_PARALLEL */ static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEF; /* Default metadata cache image settings */ +static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */ +static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ +static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer minumum raw data size */ /*------------------------------------------------------------------------- @@ -574,6 +592,22 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_SIZE_NAME, H5F_ACS_PAGE_BUFFER_SIZE_SIZE, &H5F_def_page_buf_size_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_SIZE_ENC, H5F_ACS_PAGE_BUFFER_SIZE_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum metadata size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE, &H5F_def_page_buf_min_meta_perc_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum raw data size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE, &H5F_def_page_buf_min_raw_perc_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4681,3 +4715,93 @@ done: } /* end H5Pget_coll_metadata_write() */ #endif /* H5_HAVE_PARALLEL */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_page_buffer_size + * + * Purpose: Set the maximum page buffering size. This has to be a + * multiple of the page allocation size which must be enabled; + * otherwise file create/open will fail. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * June 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_perc, unsigned min_raw_perc) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "izIuIu", plist_id, buf_size, min_meta_perc, min_raw_perc); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(min_meta_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum metadata fractions must be between 0 and 100 inclusive") + if(min_raw_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum rawdata fractions must be between 0 and 100 inclusive") + + if(min_meta_perc + min_raw_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Sum of minimum metadata and raw data fractions can't be bigger than 100"); + + /* Set size */ + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &buf_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set page buffer size") + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &min_meta_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min metadata entries") + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &min_raw_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min rawdata entries") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_page_buffer_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_page_buffer_size + * + * Purpose: Retrieves the maximum page buffer size. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * June 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_perc, unsigned *min_raw_perc) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "i*z*Iu*Iu", plist_id, buf_size, min_meta_perc, min_raw_perc); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get size */ + + if(buf_size) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, buf_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer size") + if(min_meta_perc) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, min_meta_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum metadata percent") + if(min_raw_perc) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, min_raw_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum raw data percent") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_page_buffer_size() */ + diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c index d451982..c778801 100644 --- a/src/H5Pfcpl.c +++ b/src/H5Pfcpl.c @@ -97,14 +97,23 @@ #define H5F_CRT_SHMSG_BTREE_MIN_ENC H5P__encode_unsigned #define H5F_CRT_SHMSG_BTREE_MIN_DEC H5P__decode_unsigned /* Definitions for file space handling strategy */ -#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(unsigned) -#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF -#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__encode_unsigned -#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__decode_unsigned -#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t) -#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF -#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t -#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t +#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(H5F_fspace_strategy_t) +#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF +#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__fcrt_fspace_strategy_enc +#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__fcrt_fspace_strategy_dec +#define H5F_CRT_FREE_SPACE_PERSIST_SIZE sizeof(hbool_t) +#define H5F_CRT_FREE_SPACE_PERSIST_DEF H5F_FREE_SPACE_PERSIST_DEF +#define H5F_CRT_FREE_SPACE_PERSIST_ENC H5P__encode_hbool_t +#define H5F_CRT_FREE_SPACE_PERSIST_DEC H5P__decode_hbool_t +#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t) +#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF +#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t +#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t +/* Definitions for file space page size in support of level-2 page caching */ +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE sizeof(hsize_t) +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF H5F_FILE_SPACE_PAGE_SIZE_DEF +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC H5P__encode_hsize_t +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC H5P__decode_hsize_t /******************/ @@ -131,6 +140,8 @@ static herr_t H5P__fcrt_shmsg_index_types_enc(const void *value, void **_pp, siz static herr_t H5P__fcrt_shmsg_index_types_dec(const void **_pp, void *value); static herr_t H5P__fcrt_shmsg_index_minsize_enc(const void *value, void **_pp, size_t *size); static herr_t H5P__fcrt_shmsg_index_minsize_dec(const void **_pp, void *value); +static herr_t H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size); +static herr_t H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value); /*********************/ @@ -178,8 +189,10 @@ static const unsigned H5F_def_sohm_index_flags_g[H5O_SHMESG_MAX_NINDEXES] = H static const unsigned H5F_def_sohm_index_minsizes_g[H5O_SHMESG_MAX_NINDEXES] = H5F_CRT_SHMSG_INDEX_MINSIZE_DEF; static const unsigned H5F_def_sohm_list_max_g = H5F_CRT_SHMSG_LIST_MAX_DEF; static const unsigned H5F_def_sohm_btree_min_g = H5F_CRT_SHMSG_BTREE_MIN_DEF; -static const unsigned H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF; +static const H5F_fspace_strategy_t H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF; +static const hbool_t H5F_def_free_space_persist_g = H5F_CRT_FREE_SPACE_PERSIST_DEF; static const hsize_t H5F_def_free_space_threshold_g = H5F_CRT_FREE_SPACE_THRESHOLD_DEF; +static const hsize_t H5F_def_file_space_page_size_g = H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF; @@ -267,12 +280,24 @@ H5P_fcrt_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the free-space persist flag */ + if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_PERSIST_NAME, H5F_CRT_FREE_SPACE_PERSIST_SIZE, &H5F_def_free_space_persist_g, + NULL, NULL, NULL, H5F_CRT_FREE_SPACE_PERSIST_ENC, H5F_CRT_FREE_SPACE_PERSIST_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the free space section threshold */ if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, H5F_CRT_FREE_SPACE_THRESHOLD_SIZE, &H5F_def_free_space_threshold_g, NULL, NULL, NULL, H5F_CRT_FREE_SPACE_THRESHOLD_ENC, H5F_CRT_FREE_SPACE_THRESHOLD_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the file space page size */ + if(H5P_register_real(pclass, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE, &H5F_def_file_space_page_size_g, + NULL, NULL, NULL, H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC, H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_fcrt_reg_prop() */ @@ -1246,15 +1271,13 @@ H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *mi if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); - /* Get value */ - if (max_list) { + /* Get value(s) */ + if(max_list) if(H5P_get(plist, H5F_CRT_SHMSG_LIST_MAX_NAME, max_list) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get list maximum"); - } - if (min_btree) { + if(min_btree) if(H5P_get(plist, H5F_CRT_SHMSG_BTREE_MIN_NAME, min_btree) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get SOHM information"); - } done: FUNC_LEAVE_API(ret_value) @@ -1262,15 +1285,12 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pset_file_space + * Function: H5Pset_file_space_strategy * - * Purpose: Sets the strategy that the library employs in managing file space. - * If strategy is zero, the property is not changed; the existing - * strategy is retained. - * Sets the threshold value that the file's free space - * manager(s) will use to track free space sections. - * If threshold is zero, the property is not changed; the existing - * threshold is retained. + * Purpose: Sets the "strategy" that the library employs in managing file space + * Sets the "persist" value as to persist free-space or not + * Sets the "threshold" value that the free space manager(s) will use to track free space sections. + * Ignore "persist" and "threshold" for strategies that do not use free-space managers * * Return: Non-negative on success/Negative on failure * @@ -1279,15 +1299,16 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold) +H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("e", "iFfh", plist_id, strategy, threshold); + H5TRACE4("e", "iFfbh", plist_id, strategy, persist, threshold); - if((unsigned)strategy >= H5F_FILE_SPACE_NTYPES) + /* Check arguments */ + if(strategy >= H5F_FSPACE_STRATEGY_NTYPES) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy") /* Get the plist structure */ @@ -1295,24 +1316,28 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") /* Set value(s), if non-zero */ - if(strategy) - if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space strategy") - if(threshold) - if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set free-space threshold") + if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy") + + /* Ignore persist and threshold settings for strategies that do not use FSM */ + if(strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || strategy == H5F_FSPACE_STRATEGY_PAGE) { + if(H5P_set(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space persisting status") + + if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space threshold") + } /* end if */ done: FUNC_LEAVE_API(ret_value) -} /* H5Pset_file_space() */ +} /* H5Pset_file_space_strategy() */ /*------------------------------------------------------------------------- - * Function: H5Pget_file_space + * Function: H5Pget_file_space_strategy * - * Purpose: Retrieves the strategy that the library uses in managing file space. - * Retrieves the threshold value that the file's free space - * managers use to track free space sections. + * Purpose: Retrieves the strategy, persist, and threshold that the library + * uses in managing file space. * * Return: Non-negative on success/Negative on failure * @@ -1321,13 +1346,13 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold) +H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold); + H5TRACE4("e", "i*Ff*b*h", plist_id, strategy, persist, threshold); /* Get the plist structure */ if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) @@ -1337,11 +1362,158 @@ H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *thre if(strategy) if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, strategy) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy") + if(persist) + if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space persisting status") if(threshold) if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, threshold) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space threshold") done: FUNC_LEAVE_API(ret_value) -} /* H5Pget_file_space() */ +} /* H5Pget_file_space_strategy() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__fcrt_fspace_strategy_enc + * + * Purpose: Callback routine which is called whenever the free-space + * strategy property in the file creation property list + * is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Friday, December 27, 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size) +{ + const H5F_fspace_strategy_t *strategy = (const H5F_fspace_strategy_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(strategy); + HDassert(size); + + if(NULL != *pp) + /* Encode free-space strategy */ + *(*pp)++ = (uint8_t)*strategy; + + /* Size of free-space strategy */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__fcrt_fspace_strategy_enc() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__fcrt_fspace_strategy_dec + * + * Purpose: Callback routine which is called whenever the free-space + * strategy property in the file creation property list + * is decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Friday, December 27, 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value) +{ + H5F_fspace_strategy_t *strategy = (H5F_fspace_strategy_t *)_value; /* Free-space strategy */ + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(strategy); + + /* Decode free-space strategy */ + *strategy = (H5F_fspace_strategy_t)*(*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__fcrt_fspace_strategy_dec() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_file_space_page_size + * + * Purpose: Sets the file space page size for paged aggregation. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; August 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ih", plist_id, fsp_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512") + + /* Set the value*/ + if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_file_space_page_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_space_page_size + * + * Purpose: Retrieves the file space page size for aggregating small metadata + * or raw data in the parameter "fsp_size". + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; August 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*h", plist_id, fsp_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get value */ + if(fsp_size) + if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, fsp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space page size") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_file_space_page_size() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 5aa8301..50b9eb0 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -300,8 +300,10 @@ H5_DLL herr_t H5Pset_shared_mesg_index(hid_t plist_id, unsigned index_num, unsig H5_DLL herr_t H5Pget_shared_mesg_index(hid_t plist_id, unsigned index_num, unsigned *mesg_type_flags, unsigned *min_mesg_size); H5_DLL herr_t H5Pset_shared_mesg_phase_change(hid_t plist_id, unsigned max_list, unsigned min_btree); H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *min_btree); -H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); -H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); +H5_DLL herr_t H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold); +H5_DLL herr_t H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold); +H5_DLL herr_t H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size); +H5_DLL herr_t H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size); /* File access property list (FAPL) routines */ H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold, @@ -367,6 +369,8 @@ H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective) #endif /* H5_HAVE_PARALLEL */ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr); H5_DLL herr_t H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr /*out*/); +H5_DLL herr_t H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_per, unsigned min_raw_per); +H5_DLL herr_t H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_per, unsigned *min_raw_per); /* Dataset creation property list (DCPL) routines */ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); @@ -534,6 +538,8 @@ H5_DLL herr_t H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, H5_DLL herr_t H5Pget_version(hid_t plist_id, unsigned *boot/*out*/, unsigned *freelist/*out*/, unsigned *stab/*out*/, unsigned *shhdr/*out*/); +H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); +H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); #endif /* H5_NO_DEPRECATED_SYMBOLS */ #ifdef __cplusplus @@ -626,7 +626,7 @@ H5Z__flush_file_cb(void *obj_ptr, hid_t H5_ATTR_UNUSED obj_id, void H5_ATTR_UNUS /* Call the flush routine for mounted file hierarchies. Do a global flush * if the file is opened for write */ if(H5F_ACC_RDWR & H5F_INTENT((H5F_t *)obj_ptr)) { - if(H5F_flush_mounts((H5F_t *)obj_ptr, H5AC_ind_read_dxpl_id) < 0) + if(H5F_flush_mounts((H5F_t *)obj_ptr, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFLUSH, FAIL, "unable to flush file hierarchy") } /* end if */ diff --git a/src/H5err.txt b/src/H5err.txt index 48069a5..b2ff3bd 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -77,6 +77,7 @@ MAJOR, H5E_SOHM, Shared Object Header Messages MAJOR, H5E_EARRAY, Extensible Array MAJOR, H5E_FARRAY, Fixed Array MAJOR, H5E_PLUGIN, Plugin for dynamically loaded library +MAJOR, H5E_PAGEBUF, Page Buffering MAJOR, H5E_NONE_MAJOR, No error # Sections (for grouping minor errors) diff --git a/src/H5private.h b/src/H5private.h index 00de96c..61ecc5b 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -1386,7 +1386,13 @@ typedef off_t h5_stat_size_t; #ifndef HDstrtol #define HDstrtol(S,R,N) strtol(S,R,N) #endif /* HDstrtol */ -H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base); +#ifndef HDstrtoll + #ifdef H5_HAVE_STRTOLL + #define HDstrtoll(S,R,N) strtoll(S,R,N) + #else + H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base); + #endif /* H5_HAVE_STRTOLL */ +#endif /* HDstrtoll */ #ifndef HDstrtoul #define HDstrtoul(S,R,N) strtoul(S,R,N) #endif /* HDstrtoul */ diff --git a/src/H5system.c b/src/H5system.c index ac323c0..1f92e19 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -32,10 +32,10 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* File access */ -#include "H5MMprivate.h" /* Memory management */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5MMprivate.h" /* Memory management */ /****************/ @@ -474,6 +474,7 @@ HDfprintf(FILE *stream, const char *fmt, ...) * *------------------------------------------------------------------------- */ +#ifndef HDstrtoll int64_t HDstrtoll(const char *s, const char **rest, int base) { @@ -549,7 +550,7 @@ HDstrtoll(const char *s, const char **rest, int base) *rest = s; return acc; } /* end HDstrtoll() */ - +#endif /*------------------------------------------------------------------------- * Function: HDrand/HDsrand @@ -604,7 +605,7 @@ void HDsrand(unsigned int seed) #ifdef H5_HAVE_FCNTL int Pflock(int fd, int operation) { - + struct flock flk; /* Set the lock type */ @@ -649,18 +650,18 @@ Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) { /*------------------------------------------------------------------------- - * Function: H5_make_time + * Function: H5_make_time * - * Purpose: Portability routine to abstract converting a 'tm' struct into - * a time_t value. + * Purpose: Portability routine to abstract converting a 'tm' struct into + * a time_t value. * - * Note: This is a little problematic because mktime() operates on - * local times. We convert to local time and then figure out the - * adjustment based on the local time zone and daylight savings - * setting. + * Note: This is a little problematic because mktime() operates on + * local times. We convert to local time and then figure out the + * adjustment based on the local time zone and daylight savings + * setting. * - * Return: Success: The value of timezone - * Failure: -1 + * Return: Success: The value of timezone + * Failure: -1 * * Programmer: Quincey Koziol * November 18, 2015 @@ -1138,7 +1139,7 @@ H5_combine_path(const char* path1, const char* path2, char **full_name /*out*/) if(NULL == (*full_name = (char *)H5MM_strdup(path2))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - } /* end if */ + } /* end if */ else if(H5_CHECK_ABS_PATH(path2)) { /* On windows path2 is a path absolute name */ diff --git a/src/H5trace.c b/src/H5trace.c index 44b2ed5..48c357a 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -895,32 +895,28 @@ H5_trace(const double *returning, const char *func, const char *type, ...) fprintf(out, "NULL"); } /* end if */ else { - H5F_file_space_type_t fs_type = (H5F_file_space_type_t)va_arg(ap, int); + H5F_fspace_strategy_t fs_strategy = (H5F_fspace_strategy_t)va_arg(ap, int); - switch(fs_type) { - case H5F_FILE_SPACE_DEFAULT: - fprintf(out, "H5F_FILE_SPACE_DEFAULT"); + switch(fs_strategy) { + case H5F_FSPACE_STRATEGY_FSM_AGGR: + fprintf(out, "H5F_FSPACE_STRATEGY_FSM_AGGR"); break; - case H5F_FILE_SPACE_ALL_PERSIST: - fprintf(out, "H5F_FILE_SPACE_ALL_PERSIST"); + case H5F_FSPACE_STRATEGY_PAGE: + fprintf(out, "H5F_FSPACE_STRATEGY_PAGE"); break; - case H5F_FILE_SPACE_ALL: - fprintf(out, "H5F_FILE_SPACE_ALL"); + case H5F_FSPACE_STRATEGY_AGGR: + fprintf(out, "H5F_FSPACE_STRATEGY_AGGR"); break; - case H5F_FILE_SPACE_AGGR_VFD: - fprintf(out, "H5F_FILE_SPACE_AGGR_VFD"); + case H5F_FSPACE_STRATEGY_NONE: + fprintf(out, "H5F_FSPACE_STRATEGY_NONE"); break; - case H5F_FILE_SPACE_VFD: - fprintf(out, "H5F_FILE_SPACE_VFD"); - break; - - case H5F_FILE_SPACE_NTYPES: + case H5F_FSPACE_STRATEGY_NTYPES: default: - fprintf(out, "%ld", (long)fs_type); + fprintf(out, "%ld", (long)fs_strategy); break; } /* end switch */ } /* end else */ @@ -1003,6 +999,15 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end else */ break; + case 't': + if(ptr) { + if(vp) + fprintf(out, "0x%lx", (unsigned long)vp); + else + fprintf(out, "NULL"); + } /* end if */ + break; + case 'v': if(ptr) { if(vp) diff --git a/src/H5win32defs.h b/src/H5win32defs.h index b419f06..e005b51 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -23,6 +23,11 @@ * */ +/* + * _MSC_VER = 1900 VS2015 + * _MSC_VER = 1800 VS2013 + * _MSC_VER = 1700 VS2012 + */ #ifdef H5_HAVE_WIN32_API typedef struct _stati64 h5_stat_t; @@ -54,13 +59,22 @@ typedef __int64 h5_stat_size_t; #define HDsleep(S) Sleep(S*1000) #define HDstat(S,B) _stati64(S,B) #define HDstrcasecmp(A,B) _stricmp(A,B) -#define HDstrtoull(S,R,N) _strtoui64(S,R,N) #define HDstrdup(S) _strdup(S) #define HDtzset() _tzset() #define HDunlink(S) _unlink(S) #define HDwrite(F,M,Z) _write(F,M,Z) #ifdef H5_HAVE_VISUAL_STUDIO + +#if (_MSC_VER < 1800) + #ifndef H5_HAVE_STRTOLL + #define HDstrtoll(S,R,N) _strtoi64(S,R,N) + #endif /* H5_HAVE_STRTOLL */ + #ifndef H5_HAVE_STRTOULL + #define HDstrtoull(S,R,N) _strtoui64(S,R,N) + #endif /* H5_HAVE_STRTOULL */ +#endif /* MSC_VER < 1800 */ + /* * The (void*) cast just avoids a compiler warning in H5_HAVE_VISUAL_STUDIO */ diff --git a/src/Makefile.am b/src/Makefile.am index 69b54b4..59df4c2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -60,7 +60,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5F.c H5Fint.c H5Faccum.c H5Fcwfs.c \ H5Fdbg.c H5Fdeprec.c H5Fefc.c H5Ffake.c H5Fio.c \ H5Fmount.c H5Fquery.c \ - H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ + H5Fsfile.c H5Fspace.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c \ @@ -98,6 +98,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Pfapl.c H5Pfcpl.c H5Pfmpl.c \ H5Pgcpl.c H5Pint.c \ H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \ + H5PB.c \ H5PL.c \ H5R.c H5Rdeprec.c \ H5UC.c \ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9657971..d7965cb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -246,6 +246,7 @@ set (H5_TESTS cross_read freespace mf + page_buffer vds file_image unregister diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index fc8587b..f372acf 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -170,6 +170,7 @@ endforeach () #-- Copy all the HDF5 files from the test directory into the source directory # -------------------------------------------------------------------- set (HDF5_REFERENCE_TEST_FILES + aggr.h5 bad_compound.h5 be_data.h5 be_extlink1.h5 @@ -187,7 +188,15 @@ set (HDF5_REFERENCE_TEST_FILES filespace_1_8.h5 fill_old.h5 filter_error.h5 + fsm_aggr_nopersist.h5 + fsm_aggr_persist.h5 group_old.h5 + h5fc_ext1_i.h5 + h5fc_ext1_f.h5 + h5fc_ext2_if.h5 + h5fc_ext2_sf.h5 + h5fc_ext3_isf.h5 + h5fc_ext_none.h5 le_data.h5 le_extlink1.h5 le_extlink2.h5 @@ -195,6 +204,9 @@ set (HDF5_REFERENCE_TEST_FILES multi_file_v16-r.h5 multi_file_v16-s.h5 noencoder.h5 + none.h5 + paged_nopersist.h5 + paged_persist.h5 specmetaread.h5 tarrold.h5 tbad_msg_count.h5 @@ -618,6 +630,7 @@ set (H5TEST_TESTS cross_read freespace mf + page_buffer vds file_image unregister @@ -1288,7 +1301,6 @@ if (HDF5_BUILD_GENERATORS) gen_cross gen_deflate gen_filters - gen_idx gen_new_array gen_new_fill gen_new_group diff --git a/test/Makefile.am b/test/Makefile.am index 98cfc42..a64cd0e 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -54,7 +54,7 @@ check_SCRIPTS = $(TEST_SCRIPT) TEST_PROG= testhdf5 \ cache cache_api cache_image cache_tagging lheap ohdr stab gheap \ evict_on_close farray earray btree2 fheap \ - pool accum hyperslab istore bittests dt_arith \ + pool accum hyperslab istore bittests dt_arith page_buffer \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ twriteorder big mtime fillval mount flush1 flush2 app_ref enum \ set_extent ttsafe enc_dec_plist enc_dec_plist_cross_platform\ @@ -91,7 +91,7 @@ endif # --enable-build-all at configure time. # The gen_old_* files can only be compiled with older versions of the library # so do not appear in this list. -BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_idx gen_new_array \ +BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \ gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \ gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread \ gen_sizes_lheap gen_file_image gen_plist diff --git a/test/accum.c b/test/accum.c index 1fcd051..d134bbc 100644 --- a/test/accum.c +++ b/test/accum.c @@ -45,18 +45,18 @@ H5F_t * f = NULL; /* Function Prototypes */ -unsigned test_write_read(const H5F_io_info_t *fio_info); -unsigned test_write_read_nonacc_front(const H5F_io_info_t *fio_info); -unsigned test_write_read_nonacc_end(const H5F_io_info_t *fio_info); -unsigned test_accum_overlap(const H5F_io_info_t *fio_info); -unsigned test_accum_overlap_clean(const H5F_io_info_t *fio_info); -unsigned test_accum_overlap_size(const H5F_io_info_t *fio_info); -unsigned test_accum_non_overlap_size(const H5F_io_info_t *fio_info); -unsigned test_accum_adjust(const H5F_io_info_t *fio_info); -unsigned test_read_after(const H5F_io_info_t *fio_info); -unsigned test_free(const H5F_io_info_t *fio_info); -unsigned test_big(const H5F_io_info_t *fio_info); -unsigned test_random_write(const H5F_io_info_t *fio_info); +unsigned test_write_read(const H5F_io_info2_t *fio_info); +unsigned test_write_read_nonacc_front(const H5F_io_info2_t *fio_info); +unsigned test_write_read_nonacc_end(const H5F_io_info2_t *fio_info); +unsigned test_accum_overlap(const H5F_io_info2_t *fio_info); +unsigned test_accum_overlap_clean(const H5F_io_info2_t *fio_info); +unsigned test_accum_overlap_size(const H5F_io_info2_t *fio_info); +unsigned test_accum_non_overlap_size(const H5F_io_info2_t *fio_info); +unsigned test_accum_adjust(const H5F_io_info2_t *fio_info); +unsigned test_read_after(const H5F_io_info2_t *fio_info); +unsigned test_free(const H5F_io_info2_t *fio_info); +unsigned test_big(const H5F_io_info2_t *fio_info); +unsigned test_random_write(const H5F_io_info2_t *fio_info); unsigned test_swmr_write_big(hbool_t newest_format); /* Helper Function Prototypes */ @@ -90,7 +90,7 @@ void accum_printf(void); int main(void) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ unsigned nerrors = 0; /* track errors */ hid_t fid = -1; @@ -109,7 +109,8 @@ main(void) /* Set up I/O info for operation */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) FAIL_STACK_ERROR + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) FAIL_STACK_ERROR + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) FAIL_STACK_ERROR /* Reset metadata accumulator for the file */ if(accum_reset(&fio_info) < 0) FAIL_STACK_ERROR @@ -166,7 +167,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_write_read(const H5F_io_info_t *fio_info) +test_write_read(const H5F_io_info2_t *fio_info) { int i = 0; int *write_buf, *read_buf; @@ -222,7 +223,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_write_read_nonacc_front(const H5F_io_info_t *fio_info) +test_write_read_nonacc_front(const H5F_io_info2_t *fio_info) { int i = 0; int *write_buf, *read_buf; @@ -281,7 +282,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_write_read_nonacc_end(const H5F_io_info_t *fio_info) +test_write_read_nonacc_end(const H5F_io_info2_t *fio_info) { int i = 0; int *write_buf, *read_buf; @@ -340,7 +341,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_free(const H5F_io_info_t *fio_info) +test_free(const H5F_io_info2_t *fio_info) { int i = 0; int32_t *wbuf = NULL; @@ -527,7 +528,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_accum_overlap(const H5F_io_info_t *fio_info) +test_accum_overlap(const H5F_io_info2_t *fio_info) { int i = 0; int32_t *wbuf, *rbuf; @@ -699,7 +700,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_accum_overlap_clean(const H5F_io_info_t *fio_info) +test_accum_overlap_clean(const H5F_io_info2_t *fio_info) { int i = 0; int32_t *wbuf, *rbuf; @@ -878,7 +879,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_accum_non_overlap_size(const H5F_io_info_t *fio_info) +test_accum_non_overlap_size(const H5F_io_info2_t *fio_info) { int i = 0; int32_t *wbuf, *rbuf; @@ -945,7 +946,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_accum_overlap_size(const H5F_io_info_t *fio_info) +test_accum_overlap_size(const H5F_io_info2_t *fio_info) { int i = 0; int32_t *wbuf, *rbuf; @@ -1023,7 +1024,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_accum_adjust(const H5F_io_info_t *fio_info) +test_accum_adjust(const H5F_io_info2_t *fio_info) { int i = 0; int s = 1048576; /* size of buffer */ @@ -1279,7 +1280,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_read_after(const H5F_io_info_t *fio_info) +test_read_after(const H5F_io_info2_t *fio_info) { int i = 0; int s = 128; /* size of buffer */ @@ -1358,7 +1359,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_big(const H5F_io_info_t *fio_info) +test_big(const H5F_io_info2_t *fio_info) { uint8_t *wbuf, *wbuf2, *rbuf, *zbuf; /* Buffers for reading & writing, etc */ unsigned u; /* Local index variable */ @@ -1666,7 +1667,7 @@ error: *------------------------------------------------------------------------- */ unsigned -test_random_write(const H5F_io_info_t *fio_info) +test_random_write(const H5F_io_info2_t *fio_info) { uint8_t *wbuf, *rbuf; /* Buffers for reading & writing */ unsigned seed = 0; /* Random # seed */ @@ -1820,7 +1821,7 @@ test_swmr_write_big(hbool_t newest_format) pid_t pid; /* Process ID */ #endif /* H5_HAVE_UNISTD_H */ int status; /* Status returned from child process */ - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ char *new_argv[] = {NULL}; char *driver = NULL; /* VFD string (from env variable) */ @@ -1877,7 +1878,9 @@ test_swmr_write_big(hbool_t newest_format) /* Set up I/O info for operation */ fio_info.f = rf; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + FAIL_STACK_ERROR + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) FAIL_STACK_ERROR /* We'll be writing lots of garbage data, so extend the diff --git a/tools/test/h5diff/testfiles/h5diff_dset_idx2.h5 b/test/aggr.h5 Binary files differindex db7584d..b24365d 100644 --- a/tools/test/h5diff/testfiles/h5diff_dset_idx2.h5 +++ b/test/aggr.h5 diff --git a/test/cache.c b/test/cache.c index 87b1272..ba6fb0e 100644 --- a/test/cache.c +++ b/test/cache.c @@ -97,19 +97,19 @@ struct move_entry_test_spec /* private function declarations: */ -static unsigned smoke_check_1(int express_test); -static unsigned smoke_check_2(int express_test); -static unsigned smoke_check_3(int express_test); -static unsigned smoke_check_4(int express_test); -static unsigned smoke_check_5(int express_test); -static unsigned smoke_check_6(int express_test); -static unsigned smoke_check_7(int express_test); -static unsigned smoke_check_8(int express_test); -static unsigned smoke_check_9(int express_test); -static unsigned smoke_check_10(int express_test); -static unsigned write_permitted_check(int express_test); -static unsigned check_insert_entry(void); -static unsigned check_flush_cache(void); +static unsigned smoke_check_1(int express_test, unsigned paged); +static unsigned smoke_check_2(int express_test, unsigned paged); +static unsigned smoke_check_3(int express_test, unsigned paged); +static unsigned smoke_check_4(int express_test, unsigned paged); +static unsigned smoke_check_5(int express_test, unsigned paged); +static unsigned smoke_check_6(int express_test, unsigned paged); +static unsigned smoke_check_7(int express_test, unsigned paged); +static unsigned smoke_check_8(int express_test, unsigned paged); +static unsigned smoke_check_9(int express_test, unsigned paged); +static unsigned smoke_check_10(int express_test, unsigned paged); +static unsigned write_permitted_check(int express_test, unsigned paged); +static unsigned check_insert_entry(unsigned paged); +static unsigned check_flush_cache(unsigned paged); static void check_flush_cache__empty_cache(H5F_t * file_ptr); static void check_flush_cache__multi_entry(H5F_t * file_ptr); static void check_flush_cache__multi_entry_test(H5F_t * file_ptr, @@ -159,48 +159,49 @@ static void check_flush_cache__flush_op_test(H5F_t * file_ptr, int check_size, struct fo_flush_entry_check check[]); static void check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr); -static unsigned check_get_entry_status(void); -static unsigned check_expunge_entry(void); -static unsigned check_multiple_read_protect(void); -static unsigned check_move_entry(void); +static unsigned check_get_entry_status(unsigned paged); +static unsigned check_expunge_entry(unsigned paged); +static unsigned check_multiple_read_protect(unsigned paged); +static unsigned check_move_entry(unsigned paged); static void check_move_entry__run_test(H5F_t * file_ptr, unsigned test_num, struct move_entry_test_spec * spec_ptr); -static unsigned check_pin_protected_entry(void); -static unsigned check_resize_entry(void); -static unsigned check_evictions_enabled(void); -static unsigned check_flush_protected_err(void); -static unsigned check_destroy_pinned_err(void); -static unsigned check_destroy_protected_err(void); -static unsigned check_duplicate_insert_err(void); -static unsigned check_double_pin_err(void); -static unsigned check_double_unpin_err(void); -static unsigned check_pin_entry_errs(void); -static unsigned check_double_protect_err(void); -static unsigned check_double_unprotect_err(void); -static unsigned check_mark_entry_dirty_errs(void); -static unsigned check_expunge_entry_errs(void); -static unsigned check_move_entry_errs(void); -static unsigned check_resize_entry_errs(void); -static unsigned check_unprotect_ro_dirty_err(void); -static unsigned check_protect_ro_rw_err(void); -static unsigned check_check_evictions_enabled_err(void); -static unsigned check_auto_cache_resize(hbool_t cork_ageout); -static unsigned check_auto_cache_resize_disable(void); -static unsigned check_auto_cache_resize_epoch_markers(void); -static unsigned check_auto_cache_resize_input_errs(void); -static unsigned check_auto_cache_resize_aux_fcns(void); -static unsigned check_metadata_blizzard_absence(hbool_t fill_via_insertion); -static unsigned check_flush_deps(void); -static unsigned check_flush_deps_err(void); -static unsigned check_flush_deps_order(void); -static unsigned check_notify_cb(void); -static unsigned check_metadata_cork(hbool_t fill_via_insertion); -static unsigned check_entry_deletions_during_scans(void); +static unsigned check_pin_protected_entry(unsigned paged); +static unsigned check_resize_entry(unsigned paged); +static unsigned check_evictions_enabled(unsigned paged); +static unsigned check_flush_protected_err(unsigned paged); +static unsigned check_destroy_pinned_err(unsigned paged); +static unsigned check_destroy_protected_err(unsigned paged); +static unsigned check_duplicate_insert_err(unsigned paged); +static unsigned check_double_pin_err(unsigned paged); +static unsigned check_double_unpin_err(unsigned paged); +static unsigned check_pin_entry_errs(unsigned paged); +static unsigned check_double_protect_err(unsigned paged); +static unsigned check_double_unprotect_err(unsigned paged); +static unsigned check_mark_entry_dirty_errs(unsigned paged); +static unsigned check_expunge_entry_errs(unsigned paged); +static unsigned check_move_entry_errs(unsigned paged); +static unsigned check_resize_entry_errs(unsigned paged); +static unsigned check_unprotect_ro_dirty_err(unsigned paged); +static unsigned check_protect_ro_rw_err(unsigned paged); +static unsigned check_protect_retries(unsigned paged); +static unsigned check_check_evictions_enabled_err(unsigned paged); +static unsigned check_auto_cache_resize(hbool_t cork_ageout, unsigned paged); +static unsigned check_auto_cache_resize_disable(unsigned paged); +static unsigned check_auto_cache_resize_epoch_markers(unsigned paged); +static unsigned check_auto_cache_resize_input_errs(unsigned paged); +static unsigned check_auto_cache_resize_aux_fcns(unsigned paged); +static unsigned check_metadata_blizzard_absence(hbool_t fill_via_insertion, unsigned paged); +static unsigned check_flush_deps(unsigned paged); +static unsigned check_flush_deps_err(unsigned paged); +static unsigned check_flush_deps_order(unsigned paged); +static unsigned check_notify_cb(unsigned paged); +static unsigned check_metadata_cork(hbool_t fill_via_insertion, unsigned paged); +static unsigned check_entry_deletions_during_scans(unsigned paged); static void cedds__expunge_dirty_entry_in_flush_test(H5F_t * file_ptr); static void cedds__H5C_make_space_in_cache(H5F_t * file_ptr); static void cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * file_ptr); static void cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr); -static unsigned check_stats(void); +static unsigned check_stats(unsigned paged); #if H5C_COLLECT_CACHE_STATS static void check_stats__smoke_check_1(H5F_t * file_ptr); #endif /* H5C_COLLECT_CACHE_STATS */ @@ -228,7 +229,7 @@ static void check_stats__smoke_check_1(H5F_t * file_ptr); */ static unsigned -smoke_check_1(int express_test) +smoke_check_1(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = FALSE; @@ -239,7 +240,10 @@ smoke_check_1(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #1 -- all clean, ins, dest, ren, 4/2 MB cache"); + if(paged) + TESTING("smoke check #1 -- all clean, ins, dest, ren, 4/2 MB cache, paged aggregation") + else + TESTING("smoke check #1 -- all clean, ins, dest, ren, 4/2 MB cache") switch (express_test) { @@ -274,8 +278,7 @@ smoke_check_1(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -427,7 +430,7 @@ smoke_check_1(int express_test) */ static unsigned -smoke_check_2(int express_test) +smoke_check_2(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = TRUE; @@ -438,7 +441,10 @@ smoke_check_2(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #2 -- ~1/2 dirty, ins, dest, ren, 4/2 MB cache"); + if(paged) + TESTING("smoke check #2 -- ~1/2 dirty, ins, dest, ren, 4/2 MB cache, paged aggregation") + else + TESTING("smoke check #2 -- ~1/2 dirty, ins, dest, ren, 4/2 MB cache") switch (express_test) { @@ -473,8 +479,7 @@ smoke_check_2(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -625,7 +630,7 @@ smoke_check_2(int express_test) */ static unsigned -smoke_check_3(int express_test) +smoke_check_3(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = FALSE; @@ -636,7 +641,10 @@ smoke_check_3(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #3 -- all clean, ins, dest, ren, 2/1 KB cache"); + if(paged) + TESTING("smoke check #3 -- all clean, ins, dest, ren, 2/1 KB cache, paged aggregation") + else + TESTING("smoke check #3 -- all clean, ins, dest, ren, 2/1 KB cache") switch (express_test) { @@ -671,8 +679,7 @@ smoke_check_3(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -824,7 +831,7 @@ smoke_check_3(int express_test) */ static unsigned -smoke_check_4(int express_test) +smoke_check_4(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = TRUE; @@ -835,7 +842,10 @@ smoke_check_4(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #4 -- ~1/2 dirty, ins, dest, ren, 2/1 KB cache"); + if(paged) + TESTING("smoke check #4 -- ~1/2 dirty, ins, dest, ren, 2/1 KB cache, paged aggregation") + else + TESTING("smoke check #4 -- ~1/2 dirty, ins, dest, ren, 2/1 KB cache") switch (express_test) { @@ -870,8 +880,7 @@ smoke_check_4(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -1023,7 +1032,7 @@ smoke_check_4(int express_test) */ static unsigned -smoke_check_5(int express_test) +smoke_check_5(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1082,7 +1091,10 @@ smoke_check_5(int express_test) /* double empty_reserve = */ 0.5f }; - TESTING("smoke check #5 -- all clean, ins, prot, unprot, AR cache 1"); + if(paged) + TESTING("smoke check #5 -- all clean, ins, prot, unprot, AR cache 1, paged aggregation") + else + TESTING("smoke check #5 -- all clean, ins, prot, unprot, AR cache 1") switch (express_test) { @@ -1117,8 +1129,7 @@ smoke_check_5(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1259,7 +1270,7 @@ smoke_check_5(int express_test) */ static unsigned -smoke_check_6(int express_test) +smoke_check_6(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1318,7 +1329,10 @@ smoke_check_6(int express_test) /* double empty_reserve = */ 0.05f }; - TESTING("smoke check #6 -- ~1/2 dirty, ins, prot, unprot, AR cache 1"); + if(paged) + TESTING("smoke check #6 -- ~1/2 dirty, ins, prot, unprot, AR cache 1, paged aggregation") + else + TESTING("smoke check #6 -- ~1/2 dirty, ins, prot, unprot, AR cache 1") pass = TRUE; @@ -1353,8 +1367,7 @@ smoke_check_6(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1495,7 +1508,7 @@ smoke_check_6(int express_test) */ static unsigned -smoke_check_7(int express_test) +smoke_check_7(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1555,7 +1568,10 @@ smoke_check_7(int express_test) /* double empty_reserve = */ 0.1f }; - TESTING("smoke check #7 -- all clean, ins, prot, unprot, AR cache 2"); + if(paged) + TESTING("smoke check #7 -- all clean, ins, prot, unprot, AR cache 2, paged aggregation") + else + TESTING("smoke check #7 -- all clean, ins, prot, unprot, AR cache 2") switch (express_test) { @@ -1590,8 +1606,7 @@ smoke_check_7(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1732,7 +1747,7 @@ smoke_check_7(int express_test) */ static unsigned -smoke_check_8(int express_test) +smoke_check_8(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1792,7 +1807,10 @@ smoke_check_8(int express_test) /* double empty_reserve = */ 0.1f }; - TESTING("smoke check #8 -- ~1/2 dirty, ins, prot, unprot, AR cache 2"); + if(paged) + TESTING("smoke check #8 -- ~1/2 dirty, ins, prot, unprot, AR cache 2, paged aggregation") + else + TESTING("smoke check #8 -- ~1/2 dirty, ins, prot, unprot, AR cache 2") switch (express_test) { @@ -1827,7 +1845,7 @@ smoke_check_8(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1975,7 +1993,7 @@ smoke_check_8(int express_test) */ static unsigned -smoke_check_9(int express_test) +smoke_check_9(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1989,7 +2007,10 @@ smoke_check_9(int express_test) H5F_t * file_ptr = NULL; H5C_t * cache_ptr = NULL; - TESTING("smoke check #9 -- all clean, ins, dest, ren, 4/2 MB, corked"); + if(paged) + TESTING("smoke check #9 -- all clean, ins, dest, ren, 4/2 MB, corked, paged aggregation") + else + TESTING("smoke check #9 -- all clean, ins, dest, ren, 4/2 MB, corked") switch (express_test) { @@ -2024,8 +2045,7 @@ smoke_check_9(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); cache_ptr = file_ptr->shared->cache; /* disable evictions */ @@ -2295,7 +2315,7 @@ smoke_check_9(int express_test) */ static unsigned -smoke_check_10(int express_test) +smoke_check_10(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -2309,7 +2329,10 @@ smoke_check_10(int express_test) H5F_t * file_ptr = NULL; H5C_t * cache_ptr = NULL; - TESTING("smoke check #10 -- ~1/2 dirty, ins, dest, ren, 4/2 MB, corked"); + if(paged) + TESTING("smoke check #10 -- ~1/2 dirty, ins, dest, ren, 4/2 MB, corked, paged aggregation") + else + TESTING("smoke check #10 -- ~1/2 dirty, ins, dest, ren, 4/2 MB, corked") switch (express_test) { @@ -2344,8 +2367,7 @@ smoke_check_10(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(show_progress) /* 3 */ @@ -2609,7 +2631,7 @@ write_permitted_check(int #if !H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS H5_ATTR_UNUSED #endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ -express_test) +express_test, unsigned paged) { #if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS @@ -2623,7 +2645,10 @@ express_test) #endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ - TESTING("write permitted check -- 1/0 MB cache"); + if(paged) + TESTING("write permitted check -- 1/0 MB cache, paged aggregation") + else + TESTING("write permitted check -- 1/0 MB cache") #if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS @@ -2660,8 +2685,7 @@ express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(1 * 1024 * 1024), - (size_t)(0)); + file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)0, paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -2823,7 +2847,7 @@ express_test) */ static unsigned -check_insert_entry(void) +check_insert_entry(unsigned paged) { int entry_type = PICO_ENTRY_TYPE; int i; @@ -2839,8 +2863,10 @@ check_insert_entry(void) test_entry_t * entry_ptr; struct H5C_cache_entry_t * search_ptr; - - TESTING("H5C_insert_entry() functionality"); + if(paged) + TESTING("H5C_insert_entry() functionality (paged aggregation)") + else + TESTING("H5C_insert_entry() functionality") pass = TRUE; @@ -2858,8 +2884,7 @@ check_insert_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -3115,11 +3140,14 @@ check_insert_entry(void) */ static unsigned -check_flush_cache(void) +check_flush_cache(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("H5C_flush_cache() functionality"); + if(paged) + TESTING("H5C_flush_cache() functionality (paged aggregation)") + else + TESTING("H5C_flush_cache() functionality") pass = TRUE; @@ -3132,8 +3160,7 @@ check_flush_cache(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); } /* first test behaviour on an empty cache. Can't do much sanity @@ -9825,7 +9852,6 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) } } - if(pass) { /* Now load a large entry. This should result in the eviction @@ -10370,14 +10396,28 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) * However, (VET 9)'s serialize function needs to modify (VET, 8), * which is currently not in cache. Thus it calls H5C_protect(VET, 8) * to gain access to it. H5C_protect(VET, 8) loads (VET, 8), and - * then attempts to evict entries to make space for it. While (VET, 9) - * is still at the bottom of the LRU, it is marked flush in progress - * and this is skipped. Thus the next entries on the LRU are (MET, 0) - * thru (MET, 30) and (LET, 0) thru (LET, 10) -- all of which are dirty, + * then attempts to evict entries to make space for it. + * + * However, H5C_make_space_in_cache() now exits without taking + * any action on re-entrant calls. Thus H5C_protect(VET, 8) simply + * loads the entry into the cache -- resulting in a cache that is + * 10 KB oversize. The subsequent unprotect puts (VET, 8) at the + * head of the LRU and marks it dirty. + * + * After (VET, 9) is serialized, it is flushed, and moved to the + * head of the LRU. + * + * At this point, the H5C_make_space_in_cache() call made by + * H5C_protect(LET, 11) now has 14 KB of space to make. + * + * The next entries on the LRU are (MET, 0) thru (MET, 30), + * (LET, 0) thru (LET, 10), and (VET, 8) -- all of which are dirty, * and are therefore flushed and moved to the head of the LRU list. * * The next entry on the bottom of the LRU list is (VET, 0), which - * is clean, and is therefore evicted to make space for (VET, 8). + * is clean, and is therefore evicted, leaving H5C_make_space_in_cache() + * with 4 KB of space to create. + * * This space is sufficient, so H5C_protect(VET, 8) inserts * (VET, 8) into the cache's index, marks it as protected, and * returns to the serialize function for (VET, 9). @@ -10386,22 +10426,10 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) * calls H5C_unprotect(VET, 8), which markes (VET, 8) as dirty and * unprotected, and places it at the head of the LRU. * - * The serialize function for (VET, 9) then returns, and (VET, 9) is - * is written to disk, marked clean, and moved to the head of the LRU. + * (VET, 0) is the next item on the LRU -- it is clean and is therefore + * evicted -- leaving 6 KB of free space after (LET, 11) is inserted + * into the cache. * - * At this point, the cache is still full (since (VET, 8) took the - * space created by the eviction of (VET, 0)). Thus - * H5C_protect(LET, 11) continues to look for space. While - * (MET, 0) was the next item on the LRU list when it called the - * serialize function for (VET, 9), the function notices that the - * LRU has been modified, and restarts its search for candidates - * for eviction at the bottom of the LRU. - * - * (MET, 0) is now at the bottom of the LRU, and is clean. Thus - * it is evicted. This makes sufficient space for (LET, 11), so - * H5C_protect(LET, 11) inserts it into the cache, marks it as - * protected, and returns. - * * H5C_unprotect(LET, 11) marks (LET, 11) as unprotected, and then * returns as well. * @@ -10427,9 +10455,9 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) * * (VET, 7) N 5 KB N N - - * - * (VET, 8) Y 10 KB Y N - - + * (VET, 8) Y 10 KB N N - - * - * (VET, 9) Y 10 KB N N - - + * (VET, 9) N 10 KB N N - - * * Start by updating the expected table for the expected changes in * entry status: @@ -10448,25 +10476,22 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) expected[0].serialized = TRUE; expected[0].destroyed = TRUE; expected[8].in_cache = TRUE; - expected[8].is_dirty = TRUE; + expected[8].is_dirty = FALSE; expected[8].deserialized = TRUE; - expected[8].serialized = FALSE; + expected[8].serialized = TRUE; expected[8].destroyed = FALSE; - expected[9].in_cache = TRUE; + expected[9].in_cache = FALSE; expected[9].is_dirty = FALSE; expected[9].serialized = TRUE; - expected[9].destroyed = FALSE; + expected[9].destroyed = TRUE; - expected[10].in_cache = FALSE; + expected[10].in_cache = TRUE; expected[10].is_dirty = FALSE; expected[10].serialized = TRUE; - expected[10].destroyed = TRUE; + expected[10].destroyed = FALSE; num_large_entries = 12; - /* a newly loaded entry is not inserted in the cache until after - * space has been made for it. Thus (LET, 11) will not be flushed. - */ for (i = num_variable_entries; i < num_variable_entries + num_monster_entries + num_large_entries - 1; i++) @@ -10484,10 +10509,10 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) /* verify cache size */ if((cache_ptr->index_len != 44) || (cache_ptr->index_size != (2 * 1024 * 1024) - - (2 * VARIABLE_ENTRY_SIZE) - - (10 * LARGE_ENTRY_SIZE)) || - (cache_ptr->index_size != ((2 * VARIABLE_ENTRY_SIZE) + - (30 * MONSTER_ENTRY_SIZE) + + (2 * 1024) - + (1 * LARGE_ENTRY_SIZE)) || + (cache_ptr->index_size != ((1 * VARIABLE_ENTRY_SIZE) + + (31 * MONSTER_ENTRY_SIZE) + (12 * LARGE_ENTRY_SIZE)))) { pass = FALSE; @@ -10497,15 +10522,27 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) /* verify entry status */ verify_entry_status(cache_ptr, 9, - (num_variable_entries + num_monster_entries + num_large_entries), + (num_variable_entries + num_monster_entries + + num_large_entries), expected); } if(pass) { - /* protect and unprotect VET 8 to move it to the top of the LRU */ + /* protect and unprotect VET 9 to evict MET 0 */ + protect_entry(file_ptr, VARIABLE_ENTRY_TYPE, 9); + unprotect_entry(file_ptr, VARIABLE_ENTRY_TYPE, 9, H5C__NO_FLAGS_SET); + + /* protect and unprotect VET 8 to dirty it and move it to the + * top of the LRU. Since we are dirtying it again, reset its + * serialized flag. + */ + base_addr = entries[VARIABLE_ENTRY_TYPE]; + entry_ptr = &(base_addr[8]); + entry_ptr->serialized = FALSE; + protect_entry(file_ptr, VARIABLE_ENTRY_TYPE, 8); - unprotect_entry(file_ptr, VARIABLE_ENTRY_TYPE, 8, H5C__NO_FLAGS_SET); + unprotect_entry(file_ptr, VARIABLE_ENTRY_TYPE, 8, H5C__DIRTIED_FLAG); /* Again, touch all the non VARIABLE_ENTRY_TYPE entries in the @@ -10517,7 +10554,13 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) /* skip MET 0 in first pass so that we evict VET 9 when we * reload MET 0 + * + * Since we are reloading MET 0, reset its destroyed flag. */ + base_addr = entries[MONSTER_ENTRY_TYPE]; + entry_ptr = &(base_addr[0]); + entry_ptr->destroyed = FALSE; + for (i = 1; i < num_monster_entries; i++) { protect_entry(file_ptr, MONSTER_ENTRY_TYPE, i); @@ -10550,7 +10593,9 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) expected[i].is_dirty = TRUE; } - /* update MET 0 to set its in cache flag, and reset the its destroyed flag */ + /* update MET 0 to set its in cache flag, and reset + * its destroyed flag + */ expected[10].in_cache = TRUE; /* pass through non variable entries will flush VET 8, and evict VET 9. @@ -10719,8 +10764,8 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) if((cache_ptr->insertions[VARIABLE_ENTRY_TYPE] != 0) || (cache_ptr->pinned_insertions[VARIABLE_ENTRY_TYPE] != 0) || (cache_ptr->clears[VARIABLE_ENTRY_TYPE] != 0) || - (cache_ptr->flushes[VARIABLE_ENTRY_TYPE] != 8) || - (cache_ptr->evictions[VARIABLE_ENTRY_TYPE] != 11) || + (cache_ptr->flushes[VARIABLE_ENTRY_TYPE] != 9) || + (cache_ptr->evictions[VARIABLE_ENTRY_TYPE] != 12) || (cache_ptr->take_ownerships[VARIABLE_ENTRY_TYPE] != 0) || (cache_ptr->moves[VARIABLE_ENTRY_TYPE] != 1) || (cache_ptr->entry_flush_moves[VARIABLE_ENTRY_TYPE] != 0) || @@ -12733,7 +12778,7 @@ check_flush_cache__pinned_single_entry_test(H5F_t * file_ptr, */ static unsigned -check_get_entry_status(void) +check_get_entry_status(unsigned paged) { static char msg[128]; herr_t result; @@ -12746,7 +12791,10 @@ check_get_entry_status(void) test_entry_t * base_addr = NULL; test_entry_t * entry_ptr = NULL; - TESTING("H5C_get_entry_status() functionality"); + if(paged) + TESTING("H5C_get_entry_status() functionality (paged aggregation)") + else + TESTING("H5C_get_entry_status() functionality") pass = TRUE; @@ -12754,8 +12802,7 @@ check_get_entry_status(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); if(file_ptr == NULL) { @@ -12962,7 +13009,7 @@ check_get_entry_status(void) */ static unsigned -check_expunge_entry(void) +check_expunge_entry(unsigned paged) { static char msg[128]; herr_t result; @@ -12975,7 +13022,10 @@ check_expunge_entry(void) test_entry_t * base_addr; test_entry_t * entry_ptr; - TESTING("H5C_expunge_entry() functionality"); + if(paged) + TESTING("H5C_expunge_entry() functionality (paged aggregation)") + else + TESTING("H5C_expunge_entry() functionality") pass = TRUE; @@ -12983,8 +13033,7 @@ check_expunge_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); base_addr = entries[0]; entry_ptr = &(base_addr[0]); @@ -13271,7 +13320,7 @@ check_expunge_entry(void) *------------------------------------------------------------------------- */ static unsigned -check_multiple_read_protect(void) +check_multiple_read_protect(unsigned paged) { H5F_t * file_ptr = NULL; #if H5C_COLLECT_CACHE_STATS @@ -13279,7 +13328,10 @@ check_multiple_read_protect(void) #endif /* H5C_COLLECT_CACHE_STATS */ test_entry_t * entry_ptr; - TESTING("multiple read only protects on a single entry"); + if(paged) + TESTING("multiple read only protects on a single entry (paged aggregation)") + else + TESTING("multiple read only protects on a single entry") pass = TRUE; @@ -13308,8 +13360,7 @@ check_multiple_read_protect(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); #if H5C_COLLECT_CACHE_STATS cache_ptr = file_ptr->shared->cache; #endif /* H5C_COLLECT_CACHE_STATS */ @@ -13693,7 +13744,7 @@ check_multiple_read_protect(void) */ static unsigned -check_move_entry(void) +check_move_entry(unsigned paged) { unsigned u; H5F_t * file_ptr = NULL; @@ -13725,7 +13776,10 @@ check_move_entry(void) }, }; - TESTING("H5C_move_entry() functionality"); + if(paged) + TESTING("H5C_move_entry() functionality (paged aggregation)") + else + TESTING("H5C_move_entry() functionality") pass = TRUE; @@ -13758,8 +13812,7 @@ check_move_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); } u = 0; @@ -13985,7 +14038,7 @@ check_move_entry__run_test(H5F_t * file_ptr, */ static unsigned -check_pin_protected_entry(void) +check_pin_protected_entry(unsigned paged) { static char msg[128]; herr_t result; @@ -13993,7 +14046,10 @@ check_pin_protected_entry(void) test_entry_t * base_addr; test_entry_t * entry_ptr; - TESTING("H5C_pin_protected_entry() functionality"); + if(paged) + TESTING("H5C_pin_protected_entry() functionality (paged aggregation)") + else + TESTING("H5C_pin_protected_entry() functionality") pass = TRUE; @@ -14006,8 +14062,7 @@ check_pin_protected_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); if(file_ptr == NULL) { @@ -14084,7 +14139,7 @@ check_pin_protected_entry(void) */ static unsigned -check_resize_entry(void) +check_resize_entry(unsigned paged) { static char msg[128]; herr_t result; @@ -14099,7 +14154,10 @@ check_resize_entry(void) test_entry_t * base_addr; test_entry_t * entry_ptr = NULL; - TESTING("entry resize functionality"); + if(paged) + TESTING("entry resize functionality (paged aggregation)") + else + TESTING("entry resize functionality") /* Setup a cache and verify that it is empty. * @@ -14138,8 +14196,8 @@ check_resize_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); + if(file_ptr == NULL) { pass = FALSE; @@ -15079,7 +15137,7 @@ check_resize_entry(void) */ static unsigned -check_evictions_enabled(void) +check_evictions_enabled(unsigned paged) { static char msg[128]; herr_t result; @@ -15093,7 +15151,10 @@ check_evictions_enabled(void) test_entry_t * base_addr = NULL; test_entry_t * entry_ptr; - TESTING("evictions enabled/disabled functionality"); + if(paged) + TESTING("evictions enabled/disabled functionality (paged aggregation)") + else + TESTING("evictions enabled/disabled functionality") /* Setup a cache and verify that it is empty. * @@ -15141,8 +15202,8 @@ check_evictions_enabled(void) reset_entries(); - file_ptr = setup_cache((size_t)(1 * 1024 * 1024), - (size_t)( 512 * 1024)); + file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)(512 * 1024), paged); + if(file_ptr == NULL) { pass = FALSE; @@ -15812,11 +15873,14 @@ check_evictions_enabled(void) */ static unsigned -check_flush_protected_err(void) +check_flush_protected_err(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("flush cache with protected entry error"); + if(paged) + TESTING("flush cache with protected entry error (paged aggregation)") + else + TESTING("flush cache with protected entry error") pass = TRUE; @@ -15829,8 +15893,7 @@ check_flush_protected_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -15886,11 +15949,14 @@ check_flush_protected_err(void) *------------------------------------------------------------------------- */ static unsigned -check_destroy_pinned_err(void) +check_destroy_pinned_err(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("destroy cache with permanently pinned entry error"); + if(paged) + TESTING("destroy cache with permanently pinned entry error (paged aggregation)") + else + TESTING("destroy cache with permanently pinned entry error") pass = TRUE; @@ -15902,8 +15968,7 @@ check_destroy_pinned_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); unprotect_entry(file_ptr, 0, 0, H5C__PIN_ENTRY_FLAG); @@ -15967,11 +16032,14 @@ check_destroy_pinned_err(void) */ static unsigned -check_destroy_protected_err(void) +check_destroy_protected_err(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("destroy cache with protected entry error"); + if(paged) + TESTING("destroy cache with protected entry error (paged aggregation)") + else + TESTING("destroy cache with protected entry error") pass = TRUE; @@ -15984,8 +16052,7 @@ check_destroy_protected_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); /* Note: normally this call would go just before the series of * flushes prior to file close -- in particular, all entries @@ -16057,14 +16124,17 @@ check_destroy_protected_err(void) */ static unsigned -check_duplicate_insert_err(void) +check_duplicate_insert_err(unsigned paged) { herr_t result = -1; H5F_t * file_ptr = NULL; test_entry_t * base_addr; test_entry_t * entry_ptr; - TESTING("duplicate entry insertion error"); + if(paged) + TESTING("duplicate entry insertion error (paged aggregation)") + else + TESTING("duplicate entry insertion error") pass = TRUE; @@ -16077,8 +16147,7 @@ check_duplicate_insert_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16137,13 +16206,16 @@ check_duplicate_insert_err(void) */ static unsigned -check_double_pin_err(void) +check_double_pin_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("pin a pinned entry error"); + if(paged) + TESTING("pin a pinned entry error (paged aggregation)") + else + TESTING("pin a pinned entry error") pass = TRUE; @@ -16157,8 +16229,7 @@ check_double_pin_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16223,13 +16294,16 @@ check_double_pin_err(void) */ static unsigned -check_double_unpin_err(void) +check_double_unpin_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("unpin an unpinned entry error"); + if(paged) + TESTING("unpin an unpinned entry error (paged aggregation)") + else + TESTING("unpin an unpinned entry error") pass = TRUE; @@ -16245,8 +16319,7 @@ check_double_unpin_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16320,13 +16393,16 @@ check_double_unpin_err(void) */ static unsigned -check_pin_entry_errs(void) +check_pin_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("pin entry related errors"); + if(paged) + TESTING("pin entry related errors (paged aggregation)") + else + TESTING("pin entry related errors") pass = TRUE; @@ -16346,8 +16422,7 @@ check_pin_entry_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16426,13 +16501,16 @@ check_pin_entry_errs(void) */ static unsigned -check_double_protect_err(void) +check_double_protect_err(unsigned paged) { H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; H5C_cache_entry_t * cache_entry_ptr; - TESTING("protect a protected entry error"); + if(paged) + TESTING("protect a protected entry error (paged aggregation)") + else + TESTING("protect a protected entry error") pass = TRUE; @@ -16445,8 +16523,7 @@ check_double_protect_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16504,13 +16581,16 @@ check_double_protect_err(void) */ static unsigned -check_double_unprotect_err(void) +check_double_unprotect_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("unprotect an unprotected entry error"); + if(paged) + TESTING("unprotect an unprotected entry error (paged aggregation)") + else + TESTING("unprotect an unprotected entry error") pass = TRUE; @@ -16523,8 +16603,7 @@ check_double_unprotect_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16582,13 +16661,16 @@ check_double_unprotect_err(void) */ static unsigned -check_mark_entry_dirty_errs(void) +check_mark_entry_dirty_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("mark entry dirty related errors"); + if(paged) + TESTING("mark entry dirty related errors (paged aggregation)") + else + TESTING("mark entry dirty related errors") pass = TRUE; @@ -16602,8 +16684,7 @@ check_mark_entry_dirty_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16662,13 +16743,16 @@ check_mark_entry_dirty_errs(void) */ static unsigned -check_expunge_entry_errs(void) +check_expunge_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("expunge entry related errors"); + if(paged) + TESTING("expunge entry related errors (paged aggregation)") + else + TESTING("expunge entry related errors") pass = TRUE; @@ -16688,8 +16772,7 @@ check_expunge_entry_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); entry_ptr = &((entries[0])[0]); @@ -16780,7 +16863,7 @@ check_expunge_entry_errs(void) *------------------------------------------------------------------------- */ static unsigned -check_move_entry_errs(void) +check_move_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; @@ -16790,7 +16873,10 @@ check_move_entry_errs(void) test_entry_t * entry_0_1_ptr; test_entry_t * entry_1_0_ptr; - TESTING("move entry related errors"); + if(paged) + TESTING("move entry related errors (paged aggregation)") + else + TESTING("move entry related errors") pass = TRUE; @@ -16802,7 +16888,7 @@ check_move_entry_errs(void) if(pass) { reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; insert_entry(file_ptr, 0, 0, H5C__NO_FLAGS_SET); @@ -16845,8 +16931,7 @@ check_move_entry_errs(void) if(pass) { reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); - + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; insert_entry(file_ptr, 0, 0, H5C__NO_FLAGS_SET); @@ -16896,13 +16981,16 @@ check_move_entry_errs(void) */ static unsigned -check_resize_entry_errs(void) +check_resize_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("resize entry related errors"); + if(paged) + TESTING("resize entry related errors (paged aggregation)") + else + TESTING("resize entry related errors") pass = TRUE; @@ -16921,8 +17009,7 @@ check_resize_entry_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); entry_ptr = &((entries[0])[0]); @@ -16997,13 +17084,16 @@ check_resize_entry_errs(void) */ static unsigned -check_unprotect_ro_dirty_err(void) +check_unprotect_ro_dirty_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("unprotect a read only entry dirty error"); + if(paged) + TESTING("unprotect a read only entry dirty error (paged aggregation)") + else + TESTING("unprotect a read only entry dirty error") pass = TRUE; @@ -17016,8 +17106,7 @@ check_unprotect_ro_dirty_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry_ro(file_ptr, 0, 0); @@ -17058,8 +17147,7 @@ check_unprotect_ro_dirty_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry_ro(file_ptr, 0, 0); protect_entry_ro(file_ptr, 0, 0); @@ -17124,13 +17212,16 @@ check_unprotect_ro_dirty_err(void) */ static unsigned -check_protect_ro_rw_err(void) +check_protect_ro_rw_err(unsigned paged) { H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; void * thing_ptr = NULL; - TESTING("protect a read only entry rw error"); + if(paged) + TESTING("protect a read only entry rw error (paged aggregation)") + else + TESTING("protect a read only entry rw error") pass = TRUE; @@ -17144,8 +17235,7 @@ check_protect_ro_rw_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry_ro(file_ptr, 0, 0); @@ -17201,7 +17291,7 @@ check_protect_ro_rw_err(void) *------------------------------------------------------------------------- */ static unsigned -check_protect_retries(void) +check_protect_retries(unsigned paged) { H5F_t * file_ptr = NULL; H5C_t *cache_ptr = NULL; @@ -17211,7 +17301,10 @@ check_protect_retries(void) int32_t type; int32_t idx; - TESTING("protect an entry to verify retries"); + if(paged) + TESTING("protect an entry to verify retries (paged aggregation)") + else + TESTING("protect an entry to verify retries") pass = TRUE; @@ -17220,8 +17313,7 @@ check_protect_retries(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); /* Set up read attempts for verifying checksum */ file_ptr->shared->read_attempts = 10; @@ -17336,14 +17428,17 @@ check_protect_retries(void) */ static unsigned -check_check_evictions_enabled_err(void) +check_check_evictions_enabled_err(unsigned paged) { herr_t result; hbool_t evictions_enabled; H5F_t * file_ptr = NULL; H5C_t * cache_ptr = NULL; - TESTING("get/set evictions enabled errors"); + if(paged) + TESTING("get/set evictions enabled errors (paged aggregation)") + else + TESTING("get/set evictions enabled errors") pass = TRUE; @@ -17364,8 +17459,7 @@ check_check_evictions_enabled_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -17488,7 +17582,7 @@ static void test_rpt_fcn(H5_ATTR_UNUSED H5C_t * cache_ptr, } static unsigned -check_auto_cache_resize(hbool_t cork_ageout) +check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) { hbool_t show_progress = FALSE; herr_t result; @@ -17542,7 +17636,10 @@ check_auto_cache_resize(hbool_t cork_ageout) /* double empty_reserve = */ 0.05f }; - TESTING("automatic cache resizing"); + if(paged) + TESTING("automatic cache resizing (paged aggregation)") + else + TESTING("automatic cache resizing") pass = TRUE; @@ -17556,7 +17653,7 @@ check_auto_cache_resize(hbool_t cork_ageout) if(pass) { reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -21558,7 +21655,7 @@ check_auto_cache_resize(hbool_t cork_ageout) */ static unsigned -check_auto_cache_resize_disable(void) +check_auto_cache_resize_disable(unsigned paged) { hbool_t show_progress = FALSE; herr_t result; @@ -21612,7 +21709,10 @@ check_auto_cache_resize_disable(void) /* double empty_reserve = */ 0.05f }; - TESTING("automatic cache resize disable"); + if(paged) + TESTING("automatic cache resize disable (paged aggregation)") + else + TESTING("automatic cache resize disable") pass = TRUE; @@ -21627,8 +21727,7 @@ check_auto_cache_resize_disable(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(file_ptr == NULL) { @@ -24283,7 +24382,7 @@ check_auto_cache_resize_disable(void) */ static unsigned -check_auto_cache_resize_epoch_markers(void) +check_auto_cache_resize_epoch_markers(unsigned paged) { hbool_t show_progress = FALSE; herr_t result; @@ -24338,7 +24437,10 @@ check_auto_cache_resize_epoch_markers(void) /* double empty_reserve = */ 0.05f }; - TESTING("automatic cache resize epoch marker management"); + if(paged) + TESTING("automatic cache resize epoch marker management (paged aggregation)") + else + TESTING("automatic cache resize epoch marker management") pass = TRUE; @@ -24348,8 +24450,7 @@ check_auto_cache_resize_epoch_markers(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -24992,7 +25093,7 @@ check_auto_cache_resize_epoch_markers(void) */ static unsigned -check_auto_cache_resize_input_errs(void) +check_auto_cache_resize_input_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; @@ -25046,7 +25147,10 @@ check_auto_cache_resize_input_errs(void) H5C_auto_size_ctl_t invalid_auto_size_ctl; H5C_auto_size_ctl_t test_auto_size_ctl; - TESTING("automatic cache resize input errors"); + if(paged) + TESTING("automatic cache resize input errors (paged aggregation)") + else + TESTING("automatic cache resize input errors") pass = TRUE; @@ -25060,8 +25164,7 @@ check_auto_cache_resize_input_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -27399,7 +27502,7 @@ check_auto_cache_resize_input_errs(void) */ static unsigned -check_auto_cache_resize_aux_fcns(void) +check_auto_cache_resize_aux_fcns(unsigned paged) { herr_t result; int32_t i; @@ -27460,7 +27563,10 @@ check_auto_cache_resize_aux_fcns(void) }; - TESTING("automatic cache resize auxilary functions"); + if(paged) + TESTING("automatic cache resize auxilary functions (paged aggregation)") + else + TESTING("automatic cache resize auxilary functions") pass = TRUE; @@ -27471,8 +27577,7 @@ check_auto_cache_resize_aux_fcns(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -27944,7 +28049,7 @@ check_auto_cache_resize_aux_fcns(void) */ static unsigned -check_metadata_blizzard_absence(hbool_t fill_via_insertion) +check_metadata_blizzard_absence(hbool_t fill_via_insertion, unsigned paged) { int entry_type = HUGE_ENTRY_TYPE; size_t entry_size = HUGE_ENTRY_SIZE; /* 16 KB */ @@ -28124,13 +28229,17 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion) reset_entries(); if(fill_via_insertion) { - - TESTING("to ensure metadata blizzard absence when inserting"); - - } else { - - TESTING("to ensure metadata blizzard absence on protect/unprotect"); - } + if(paged) + TESTING("to ensure metadata blizzard absence when inserting (paged aggregation)") + else + TESTING("to ensure metadata blizzard absence when inserting") + } /* end if */ + else { + if(paged) + TESTING("to ensure metadata blizzard absence on protect/unprotect (paged aggregation)") + else + TESTING("to ensure metadata blizzard absence on protect/unprotect") + } /* end else */ if(show_progress) /* 0 */ HDfprintf(stdout, "\n%s: check point %d -- pass %d\n", FUNC, checkpoint++, pass); @@ -28142,8 +28251,7 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion) * The max_cache_size should have room for 50 entries. * The min_clean_size is half of that, or 25 entries. */ - file_ptr = setup_cache((size_t)(50 * entry_size), /* max_cache_size */ - (size_t)(25 * entry_size)); /* min_clean_size */ + file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged); if(file_ptr == NULL) { @@ -28787,7 +28895,7 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion) */ static unsigned -check_flush_deps(void) +check_flush_deps(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ H5C_t * cache_ptr = NULL; /* Metadata cache for this test */ @@ -28805,7 +28913,10 @@ check_flush_deps(void) { PICO_ENTRY_TYPE, 4, PICO_ENTRY_SIZE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, {0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0}, 0, 0, 0, -1, FALSE} }; - TESTING("flush dependencies"); + if(paged) + TESTING("flush dependencies (paged aggregation)") + else + TESTING("flush dependencies") pass = TRUE; @@ -28814,7 +28925,7 @@ check_flush_deps(void) */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; base_addr = entries[entry_type]; @@ -30523,13 +30634,16 @@ done: */ static unsigned -check_flush_deps_err(void) +check_flush_deps_err(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ int entry_type = PICO_ENTRY_TYPE; /* Use very small entry size (size of entries doesn't matter) */ unsigned test_count; /* Test iteration variable */ - TESTING("flush dependency errors"); + if(paged) + TESTING("flush dependency errors (paged aggregation)") + else + TESTING("flush dependency errors") pass = TRUE; @@ -30542,7 +30656,7 @@ check_flush_deps_err(void) /* Allocate a cache */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(!pass) CACHE_ERROR("setup_cache failed") /* Insert entries to work with into the cache */ @@ -30744,7 +30858,7 @@ done: */ static unsigned -check_flush_deps_order(void) +check_flush_deps_order(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ H5C_t * cache_ptr = NULL; /* Metadata cache for this test */ @@ -30762,7 +30876,10 @@ check_flush_deps_order(void) }; unsigned flush_order; /* Index for tracking flush order */ - TESTING("flush dependencies flush order"); + if(paged) + TESTING("flush dependencies flush order (paged aggregation)") + else + TESTING("flush dependencies flush order") pass = TRUE; @@ -30771,7 +30888,7 @@ check_flush_deps_order(void) */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(!pass) CACHE_ERROR("setup_cache failed") @@ -33312,7 +33429,7 @@ done: */ static unsigned -check_notify_cb(void) +check_notify_cb(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ H5C_t * cache_ptr = NULL; /* Metadata cache for this test */ @@ -33331,7 +33448,10 @@ check_notify_cb(void) { NOTIFY_ENTRY_TYPE, 4, NOTIFY_ENTRY_SIZE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, {0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0}, 0, 0, 0, -1, FALSE} }; - TESTING("'notify' callback"); + if(paged) + TESTING("'notify' callback (paged)") + else + TESTING("'notify' callback") pass = TRUE; @@ -33340,7 +33460,7 @@ check_notify_cb(void) */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(!file_ptr) CACHE_ERROR("setup_cache returned NULL") cache_ptr = file_ptr->shared->cache; base_addr = entries[entry_type]; @@ -33513,7 +33633,7 @@ done: *------------------------------------------------------------------------- */ static unsigned -check_metadata_cork(hbool_t fill_via_insertion) +check_metadata_cork(hbool_t fill_via_insertion, unsigned paged) { const char * fcn_name = "check_metadata_cork"; int entry_type = HUGE_ENTRY_TYPE; @@ -33709,8 +33829,7 @@ check_metadata_cork(hbool_t fill_via_insertion) * The max_cache_size should have room for 50 entries. * The min_clean_size is half of that, or 25 entries. */ - file_ptr = setup_cache((size_t)(50 * entry_size), /* max_cache_size */ - (size_t)(25 * entry_size)); /* min_clean_size */ + file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged); if(file_ptr == NULL) { @@ -34259,11 +34378,14 @@ check_metadata_cork(hbool_t fill_via_insertion) *------------------------------------------------------------------------- */ static unsigned -check_entry_deletions_during_scans(void) +check_entry_deletions_during_scans(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("entry deletion during list scan detection and adaption"); + if(paged) + TESTING("entry deletion during list scan detection and adaption (paged aggregation)") + else + TESTING("entry deletion during list scan detection and adaption") pass = TRUE; @@ -34276,8 +34398,7 @@ check_entry_deletions_during_scans(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); } /* run the tests. This set of tests is somewhat eclectic, as @@ -35751,7 +35872,7 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr) */ static unsigned -check_stats(void) +check_stats(unsigned paged) { #if H5C_COLLECT_CACHE_STATS @@ -35760,7 +35881,10 @@ check_stats(void) #endif /* H5C_COLLECT_CACHE_STATS */ - TESTING("metadata cache statistics collection"); + if(paged) + TESTING("metadata cache statistics collection (paged aggregation)") + else + TESTING("metadata cache statistics collection") #if H5C_COLLECT_CACHE_STATS @@ -35768,8 +35892,7 @@ check_stats(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); if(pass) { @@ -36250,6 +36373,7 @@ int main(void) { unsigned nerrs = 0; + unsigned paged; int express_test; H5open(); @@ -36267,61 +36391,63 @@ main(void) return EXIT_FAILURE; } /* end if */ - nerrs += smoke_check_1(express_test); - nerrs += smoke_check_2(express_test); - nerrs += smoke_check_3(express_test); - nerrs += smoke_check_4(express_test); - nerrs += smoke_check_5(express_test); - nerrs += smoke_check_6(express_test); - nerrs += smoke_check_7(express_test); - nerrs += smoke_check_8(express_test); - nerrs += smoke_check_9(express_test); - nerrs += smoke_check_10(express_test); - - nerrs += write_permitted_check(express_test); - - nerrs += check_insert_entry(); - nerrs += check_flush_cache(); - nerrs += check_get_entry_status(); - nerrs += check_expunge_entry(); - nerrs += check_multiple_read_protect(); - nerrs += check_move_entry(); - nerrs += check_pin_protected_entry(); - nerrs += check_resize_entry(); - nerrs += check_evictions_enabled(); - nerrs += check_flush_protected_err(); - nerrs += check_destroy_pinned_err(); - nerrs += check_destroy_protected_err(); - nerrs += check_duplicate_insert_err(); - nerrs += check_double_pin_err(); - nerrs += check_double_unpin_err(); - nerrs += check_pin_entry_errs(); - nerrs += check_double_protect_err(); - nerrs += check_double_unprotect_err(); - nerrs += check_mark_entry_dirty_errs(); - nerrs += check_expunge_entry_errs(); - nerrs += check_move_entry_errs(); - nerrs += check_resize_entry_errs(); - nerrs += check_unprotect_ro_dirty_err(); - nerrs += check_protect_ro_rw_err(); - nerrs += check_protect_retries(); - nerrs += check_check_evictions_enabled_err(); - nerrs += check_auto_cache_resize(FALSE); - nerrs += check_auto_cache_resize(TRUE); - nerrs += check_auto_cache_resize_disable(); - nerrs += check_auto_cache_resize_epoch_markers(); - nerrs += check_auto_cache_resize_input_errs(); - nerrs += check_auto_cache_resize_aux_fcns(); - nerrs += check_metadata_blizzard_absence(TRUE); - nerrs += check_metadata_blizzard_absence(FALSE); - nerrs += check_flush_deps(); - nerrs += check_flush_deps_err(); - nerrs += check_flush_deps_order(); - nerrs += check_notify_cb(); - nerrs += check_metadata_cork(TRUE); - nerrs += check_metadata_cork(FALSE); - nerrs += check_entry_deletions_during_scans(); - nerrs += check_stats(); + /* Test with paged aggregation enabled or not */ + /* Each test will call setup_cache() which set up the file space strategy according to "paged" */ + for(paged = FALSE; paged <= TRUE; paged++) { + nerrs += smoke_check_1(express_test, paged); + nerrs += smoke_check_2(express_test, paged); + nerrs += smoke_check_3(express_test, paged); + nerrs += smoke_check_4(express_test, paged); + nerrs += smoke_check_5(express_test, paged); + nerrs += smoke_check_6(express_test, paged); + nerrs += smoke_check_7(express_test, paged); + nerrs += smoke_check_8(express_test, paged); + nerrs += smoke_check_9(express_test, paged); + nerrs += smoke_check_10(express_test, paged); + nerrs += write_permitted_check(express_test, paged); + nerrs += check_insert_entry(paged); + nerrs += check_flush_cache(paged); + nerrs += check_get_entry_status(paged); + nerrs += check_expunge_entry(paged); + nerrs += check_multiple_read_protect(paged); + nerrs += check_move_entry(paged); + nerrs += check_pin_protected_entry(paged); + nerrs += check_resize_entry(paged); + nerrs += check_evictions_enabled(paged); + nerrs += check_flush_protected_err(paged); + nerrs += check_destroy_pinned_err(paged); + nerrs += check_destroy_protected_err(paged); + nerrs += check_duplicate_insert_err(paged); + nerrs += check_double_pin_err(paged); + nerrs += check_double_unpin_err(paged); + nerrs += check_pin_entry_errs(paged); + nerrs += check_double_protect_err(paged); + nerrs += check_double_unprotect_err(paged); + nerrs += check_mark_entry_dirty_errs(paged); + nerrs += check_expunge_entry_errs(paged); + nerrs += check_move_entry_errs(paged); + nerrs += check_resize_entry_errs(paged); + nerrs += check_unprotect_ro_dirty_err(paged); + nerrs += check_protect_ro_rw_err(paged); + nerrs += check_protect_retries(paged); + nerrs += check_check_evictions_enabled_err(paged); + nerrs += check_auto_cache_resize(FALSE, paged); + nerrs += check_auto_cache_resize(TRUE, paged); + nerrs += check_auto_cache_resize_disable(paged); + nerrs += check_auto_cache_resize_epoch_markers(paged); + nerrs += check_auto_cache_resize_input_errs(paged); + nerrs += check_auto_cache_resize_aux_fcns(paged); + nerrs += check_metadata_blizzard_absence(TRUE, paged); + nerrs += check_metadata_blizzard_absence(FALSE, paged); + nerrs += check_flush_deps(paged); + nerrs += check_flush_deps_err(paged); + nerrs += check_flush_deps_order(paged); + nerrs += check_notify_cb(paged); + nerrs += check_metadata_cork(TRUE, paged); + nerrs += check_metadata_cork(FALSE, paged); + nerrs += check_entry_deletions_during_scans(paged); + nerrs += check_stats(paged); + } /* end for */ /* can't fail, returns void */ free_entry_arrays(); diff --git a/test/cache_api.c b/test/cache_api.c index 710e38f..867f055 100644 --- a/test/cache_api.c +++ b/test/cache_api.c @@ -30,12 +30,12 @@ /* private function declarations: */ -static hbool_t check_fapl_mdc_api_calls(void); -static hbool_t check_file_mdc_api_calls(void); -static hbool_t mdc_api_call_smoke_check(int express_test); +static hbool_t check_fapl_mdc_api_calls(unsigned paged, hid_t fcpl_id); +static hbool_t check_file_mdc_api_calls(unsigned paged, hid_t fcpl_id); +static hbool_t mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id); static H5AC_cache_config_t * init_invalid_configs(void); static hbool_t check_fapl_mdc_api_errs(void); -static hbool_t check_file_mdc_api_errs(void); +static hbool_t check_file_mdc_api_errs(unsigned paged, hid_t fcpl_id); @@ -64,7 +64,7 @@ static hbool_t check_file_mdc_api_errs(void); *------------------------------------------------------------------------- */ static hbool_t -check_fapl_mdc_api_calls(void) +check_fapl_mdc_api_calls(unsigned paged, hid_t fcpl_id) { char filename[512]; herr_t result; @@ -113,7 +113,10 @@ check_fapl_mdc_api_calls(void) H5C_auto_size_ctl_t default_auto_size_ctl; H5C_auto_size_ctl_t mod_auto_size_ctl; - TESTING("MDC/FAPL related API calls"); + if(paged) + TESTING("MDC/FAPL related API calls for paged aggregation strategy") + else + TESTING("MDC/FAPL related API calls") pass = TRUE; @@ -217,7 +220,7 @@ check_fapl_mdc_api_calls(void) /* create the file using the default FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -365,7 +368,7 @@ check_fapl_mdc_api_calls(void) /* create the file using the modified FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); if ( file_id < 0 ) { @@ -517,7 +520,7 @@ check_fapl_mdc_api_calls(void) *------------------------------------------------------------------------- */ static hbool_t -check_file_mdc_api_calls(void) +check_file_mdc_api_calls(unsigned paged, hid_t fcpl_id) { char filename[512]; hid_t file_id = -1; @@ -669,7 +672,10 @@ check_file_mdc_api_calls(void) H5AC__DEFAULT_METADATA_WRITE_STRATEGY }; - TESTING("MDC/FILE related API calls"); + if(paged) + TESTING("MDC/FILE related API calls for paged aggregation strategy") + else + TESTING("MDC/FILE related API calls") pass = TRUE; @@ -695,7 +701,7 @@ check_file_mdc_api_calls(void) /* create the file using the default FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -874,7 +880,7 @@ check_file_mdc_api_calls(void) #define NUM_RANDOM_ACCESSES 200000 static hbool_t -mdc_api_call_smoke_check(int express_test) +mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id) { char filename[512]; hbool_t valid_chunk; @@ -1003,7 +1009,10 @@ mdc_api_call_smoke_check(int express_test) H5AC__DEFAULT_METADATA_WRITE_STRATEGY }; - TESTING("MDC API smoke check"); + if(paged) + TESTING("MDC API smoke check for paged aggregation strategy") + else + TESTING("MDC API smoke check") pass = TRUE; @@ -1038,7 +1047,7 @@ mdc_api_call_smoke_check(int express_test) /* create the file using the default FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -1921,7 +1930,7 @@ check_fapl_mdc_api_errs(void) *------------------------------------------------------------------------- */ static hbool_t -check_file_mdc_api_errs(void) +check_file_mdc_api_errs(unsigned paged, hid_t fcpl_id) { char filename[512]; static char msg[128]; @@ -1937,7 +1946,10 @@ check_file_mdc_api_errs(void) H5AC_cache_config_t default_config = H5AC__DEFAULT_CACHE_CONFIG; H5AC_cache_config_t scratch; - TESTING("MDC/FILE related API input errors"); + if(paged) + TESTING("MDC/FILE related API input errors for paged aggregation strategy") + else + TESTING("MDC/FILE related API input errors") pass = TRUE; @@ -1968,7 +1980,7 @@ check_file_mdc_api_errs(void) HDfprintf(stdout, "%s: calling H5Fcreate().\n", FUNC); } - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -2267,6 +2279,9 @@ main(void) { unsigned nerrs = 0; int express_test; + hid_t fcpl_id = -1; + hid_t fcpl2_id = -1; + unsigned paged; H5open(); @@ -2281,42 +2296,64 @@ main(void) /* Initialize invalid configurations. */ invalid_configs = init_invalid_configs(); - if ( NULL == invalid_configs ) { - failure_mssg = "Unable to allocate memory for invalid configs."; HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); return EXIT_FAILURE; - } + } /* end if */ - if ( !check_fapl_mdc_api_calls() ) { + if((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0) { + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ - nerrs += 1; - } + /* Set file space strategy to default or paged aggregation strategy */ + if((fcpl2_id = H5Pcopy(fcpl_id)) < 0) { + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ - if ( !check_file_mdc_api_calls() ) { + if(H5Pset_file_space_strategy(fcpl2_id, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)1) < 0) { + failure_mssg = "H5Pset_file_space_strategy() failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ - nerrs += 1; - } + /* Test with paged aggregation enabled or not */ + /* The "my_fcpl" passed to each test has the paged or non-paged strategy set up accordinly */ + for(paged = FALSE; paged <= TRUE; paged++) { + hid_t my_fcpl = fcpl_id; - if ( !mdc_api_call_smoke_check(express_test) ) { + if(paged) + my_fcpl = fcpl2_id; - nerrs += 1; - } + if(!check_fapl_mdc_api_calls(paged, my_fcpl)) + nerrs += 1; - if ( !check_fapl_mdc_api_errs() ) { + if(!check_file_mdc_api_calls(paged, my_fcpl)) + nerrs += 1; - nerrs += 1; - } + if(!mdc_api_call_smoke_check(express_test, paged, my_fcpl)) + nerrs += 1; - if ( !check_file_mdc_api_errs() ) { + if(!check_file_mdc_api_errs(paged, my_fcpl)) + nerrs += 1; + } /* end for paged */ + if(!check_fapl_mdc_api_errs()) nerrs += 1; - } if(invalid_configs) HDfree(invalid_configs); + if(H5Pclose(fcpl_id) < 0 ) { + failure_mssg = "H5Pclose() failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ + if(nerrs > 0) return EXIT_FAILURE; else diff --git a/test/cache_common.c b/test/cache_common.c index d1bbf10..52dbb52 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -39,6 +39,12 @@ hid_t saved_fapl_id = H5P_DEFAULT; /* store the fapl id here between * close. */ +hid_t saved_fcpl_id = H5P_DEFAULT; /* store the fcpl id here between + * cache setup and takedown. Note + * that if saved_fcpl_id == H5P_DEFAULT, + * we assume that there is no fcpl to + * close. + */ hid_t saved_fid = -1; /* store the file id here between cache setup * and takedown. */ @@ -1632,9 +1638,12 @@ free_icr(test_entry_t *entry, int32_t entry_type) HDassert(entry->cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert((entry->header.destroy_in_progress) || (entry->header.addr == entry->addr)); + HDassert(entry->header.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(entry->header.size == entry->size); HDassert((entry->type == VARIABLE_ENTRY_TYPE) || (entry->size == entry_sizes[entry->type])); + HDassert(entry->header.tl_next == NULL); + HDassert(entry->header.tl_prev == NULL); if(entry->num_pins > 0) { int i; @@ -3165,7 +3174,8 @@ verify_unprotected(void) H5F_t * setup_cache(size_t max_cache_size, - size_t min_clean_size) + size_t min_clean_size, + unsigned paged) { char filename[512]; hbool_t show_progress = FALSE; @@ -3177,6 +3187,7 @@ setup_cache(size_t max_cache_size, H5F_t * ret_val = NULL; haddr_t actual_base_addr; hid_t fapl_id = H5P_DEFAULT; + hid_t fcpl_id = H5P_DEFAULT; if(show_progress) /* 1 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -3184,6 +3195,36 @@ setup_cache(size_t max_cache_size, saved_fid = -1; + if(pass) { + if((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) == FAIL) { + pass = FALSE; + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed.\n"; + } + } + + if(pass && paged) { + /* Set up paged aggregation strategy */ + if(H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)1) == FAIL) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_strategy() failed.\n"; + H5Pclose(fcpl_id); + fcpl_id = H5P_DEFAULT; + } + } + + if(pass && paged) { + /* Set up file space page size to BASE_ADDR */ + if(H5Pset_file_space_page_size(fcpl_id, (hsize_t)BASE_ADDR) == FAIL) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_page_size() failed.\n"; + H5Pclose(fcpl_id); + fcpl_id = H5P_DEFAULT; + } + } + + if(pass) + saved_fcpl_id = fcpl_id; + /* setup the file name */ if(pass) { if(NULL == h5_fixname(FILENAME[0], H5P_DEFAULT, filename, sizeof(filename))) { @@ -3207,7 +3248,7 @@ setup_cache(size_t max_cache_size, pass = FALSE; failure_mssg = "H5P_set_fapl_core() failed.\n"; } - else if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) { + else if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id)) < 0) { core_file_driver_failed = TRUE; if(verbose) @@ -3226,8 +3267,8 @@ setup_cache(size_t max_cache_size, * If this fails, we are cooked. */ if(pass && fid < 0) { - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); - saved_fid = fid; + fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); + saved_fid = fid; if(fid < 0) { pass = FALSE; @@ -3438,6 +3479,11 @@ takedown_cache(H5F_t * file_ptr, saved_fapl_id = H5P_DEFAULT; } + if ( saved_fcpl_id != H5P_DEFAULT ) { + H5Pclose(saved_fcpl_id); + saved_fcpl_id = H5P_DEFAULT; + } + if ( saved_fid != -1 ) { if ( H5F_addr_defined(saved_actual_base_addr) ) { diff --git a/test/cache_common.h b/test/cache_common.h index f003189..f17c16b 100644 --- a/test/cache_common.h +++ b/test/cache_common.h @@ -643,7 +643,7 @@ H5TEST_DLL void resize_entry(H5F_t * file_ptr, size_t new_size, hbool_t in_cache); -H5TEST_DLL H5F_t *setup_cache(size_t max_cache_size, size_t min_clean_size); +H5TEST_DLL H5F_t *setup_cache(size_t max_cache_size, size_t min_clean_size, unsigned paged); H5TEST_DLL void row_major_scan_forward(H5F_t * file_ptr, int32_t max_index, diff --git a/test/cache_image.c b/test/cache_image.c index de0507b..f49f7b3 100644 --- a/test/cache_image.c +++ b/test/cache_image.c @@ -59,6 +59,9 @@ static unsigned cache_image_smoke_check_6(void); static unsigned cache_image_api_error_check_1(void); static unsigned cache_image_api_error_check_2(void); static unsigned cache_image_api_error_check_3(void); +static unsigned cache_image_api_error_check_4(void); + +static unsigned get_free_sections_test(void); /****************************************************************************/ @@ -664,12 +667,10 @@ open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, } if ( ( pass ) && ( config_fsm ) ) { - - if ( H5Pset_file_space(fcpl_id, H5F_FILE_SPACE_ALL_PERSIST, 1) < 0 ) { - - pass = FALSE; - failure_mssg = "H5Pset_file_space() failed."; - } + if(H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_strategy() failed."; + } } if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); @@ -4560,7 +4561,7 @@ cache_image_smoke_check_4(void) * If sufficient zoos have been created, continue to * 10). Otherwise goto 5) * - * 10) Open the file. + * 10) Open the file R/O. * * Verify that the file contains a metadata cache * image superblock extension message. @@ -4571,14 +4572,23 @@ cache_image_smoke_check_4(void) * * 13) Open the file. * - * Verify that the file doesn't contain a metadata cache + * Verify that the file contains a metadata cache * image superblock extension message. * * 14) Validate all the zoos. * * 15) Close the file. * - * 16) Delete the file. + * 16) Open the file. + * + * Verify that the file doesn't contain a metadata cache + * image superblock extension message. + * + * 17) Validate all the zoos. + * + * 18) Close the file. + * + * 19) Delete the file. * * Return: void * @@ -4821,9 +4831,56 @@ cache_image_smoke_check_5(void) } /* end while */ cp += 5; + /* 10) Open the file read only. + * + * Verify that the file contains a metadata cache image + * superblock extension message. + */ + if(pass) { + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ TRUE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ 0, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if(show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 11) Validate all the zoos. */ + i = min_group; + while(pass && i <= max_group) { + sprintf(process_group_name, "/process_%d", i); + validate_zoo(file_id, process_group_name, i++); + } - - /* 10) Open the file. +#if H5C_COLLECT_CACHE_STATS + if( pass) { + if(cache_ptr->images_loaded == 0) { + pass = FALSE; + failure_mssg = "metadata cache image block not loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if(show_progress) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 12) Close the file. */ + if(pass) { + if(H5Fclose(file_id) < 0) { + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + } + } + + /* 13) Open the file R/W. * * Verify that the file contains a metadata cache image * superblock extension message. @@ -4847,7 +4904,7 @@ cache_image_smoke_check_5(void) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 11) Validate all the zoos. */ + /* 14) Validate all the zoos. */ i = min_group; while ( ( pass ) && ( i <= max_group ) ) { @@ -4871,7 +4928,7 @@ cache_image_smoke_check_5(void) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 12) Close the file. */ + /* 15) Close the file. */ if ( pass ) { @@ -4883,7 +4940,7 @@ cache_image_smoke_check_5(void) } - /* 13) Open the file. + /* 16) Open the file. * * Verify that the file doesn't contain a metadata cache image * superblock extension message. @@ -4907,7 +4964,7 @@ cache_image_smoke_check_5(void) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 14) Validate all the zoos. + /* 17) Validate all the zoos. * * Verify that the metadata cache image superblock * extension message has been deleted. @@ -4935,7 +4992,7 @@ cache_image_smoke_check_5(void) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 15) Close the file. */ + /* 18) Close the file. */ if ( pass ) { @@ -4950,7 +5007,7 @@ cache_image_smoke_check_5(void) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 16) Delete the file */ + /* 19) Delete the file */ if ( pass ) { @@ -5035,11 +5092,14 @@ cache_image_smoke_check_5(void) * 13) Close the file. * * 14) Get the size of the file. Verify that it is less - * than 1 KB. Without deletions and persistant free + * than 20 KB. Without deletions and persistant free * space managers, size size is about 167 MB, so this * is sufficient to verify that the persistant free * space managers are more or less doing their job. * + * Note that in the absence of paged allocation, file + * size gets below 1 KB. + * * 15) Delete the file. * * Return: void @@ -5352,20 +5412,24 @@ cache_image_smoke_check_6(void) /* 14) Get the size of the file. Verify that it is less - * than 1 KB. Without deletions and persistant free + * than 20 KB. Without deletions and persistant free * space managers, size size is about 167 MB, so this * is sufficient to verify that the persistant free * space managers are more or less doing their job. + * + * Note that in the absence of paged allocation, file + * size gets below 1 KB, but since this test is run both + * with and without paged allocation, we must leave some + * extra space for the paged allocation case. */ - if((file_size = h5_get_file_size(filename, H5P_DEFAULT)) < 0) { - - pass = FALSE; - failure_mssg = "h5_get_file_size() failed.\n"; - - } else if ( file_size > 1024 ) { - - pass = FALSE; - failure_mssg = "unexpectedly large file size.\n"; + if(pass) { + if((file_size = h5_get_file_size(filename, H5P_DEFAULT)) < 0) { + pass = FALSE; + failure_mssg = "h5_get_file_size() failed.\n"; + } else if(file_size > 20 * 1024) { + pass = FALSE; + failure_mssg = "unexpectedly large file size.\n"; + } } if ( show_progress ) @@ -6444,6 +6508,1064 @@ cache_image_api_error_check_3(void) /*------------------------------------------------------------------------- + * Function: cache_image_api_error_check_4() + * + * Purpose: This test is one of a sequence of tests intended + * to verify correct management of API errors. + * + * The object of this test is to verify that a request for + * a cache image when a version 2 superblock is not available/ + * not requested is handled correctly. + * (the cache image request should be ignored silently). + * + * The test is set up as follows: + * + * 1) Create a FAPL requesting a cache image, but WITHOUT + * specifying the latest file format. + * + * 2) Create a HDF5 file using the above FAPL. + * + * 3) Create some datasets in the file. + * + * 4) Close the file. + * + * 5) Open the file read only. Verify that the file doesn't + * contain a cache image. + * + * 6) Verify that the datasets exist and contain the + * expected data + * + * Verify that the cache image was not loaded. + * + * 7) Close the file. + * + * 8) Open the file R/W using the FAPL defined in 1) above. + * Verify that the file does not contain a cache image. + * + * 9) Close the file. + * + * 10) Open the file R/W using the FAPL defined in 1) above. + * Verify that the file does not contain a cache image. + * + * 11) Verify that the data sets contain the expected data + * + * Verify that a cache image was not loaded. + * + * 12) Create several more data sets. + * + * 13) Close the file. + * + * 14) Open the file read write. + * + * Verify that the file does not contain a cache image. + * + * 15) Verify the data sets exist and contain the expected + * data. + * + * Verify that a cache image was not loaded. + * + * 16) Close the file. + * + * 17) Delete the file. + * + * Return: void + * + * Programmer: John Mainzer + * 9/25/15 + * + *------------------------------------------------------------------------- + */ + +static unsigned +cache_image_api_error_check_4(void) +{ + const char * fcn_name = "cache_image_api_error_check_4()"; + char filename[512]; + hbool_t show_progress = FALSE; + hid_t fapl_id = -1; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + int cp = 0; + H5AC_cache_image_config_t cache_image_config; + + TESTING("metadata cache image api error check 4"); + + pass = TRUE; + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + if ( h5_fixname(FILENAMES[0], H5P_DEFAULT, filename, sizeof(filename)) + == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 1) Create a FAPL requesting a cache image, but WITHOUT + * specifying the latest file format. + */ + if ( pass ) { + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + + if ( fapl_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pcreate() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + /* set cache image config fields to taste */ + cache_image_config.version = H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION; + cache_image_config.generate_image = TRUE; + cache_image_config.save_resize_status = FALSE; + cache_image_config.entry_ageout = H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE; + + if ( H5Pset_mdc_image_config(fapl_id, &cache_image_config) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_mdc_image_config() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Create a HDF5 file using the above FAPL. */ + + if ( pass ) { + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + + if ( file_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fcreate() failed.\n"; + + } else { + + file_ptr = (struct H5F_t *)H5I_object_verify(file_id, H5I_FILE); + + if ( file_ptr == NULL ) { + + pass = FALSE; + failure_mssg = "Can't get file_ptr."; + + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* get a pointer to the files internal data structure and then + * to the cache structure + */ + if ( pass ) { + + if ( file_ptr->shared->cache == NULL ) { + + pass = FALSE; + failure_mssg = "can't get cache pointer(1).\n"; + + } else { + + cache_ptr = file_ptr->shared->cache; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 3) Create some datasets in the file. */ + + if ( pass ) { + + create_datasets(file_id, 0, 5); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + HDassert(cache_ptr); + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(1)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 4) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 5) Open the file read only. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ TRUE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 6) Verify that the datasets exist and contain the + * expected data + */ + + if ( pass ) { + + verify_datasets(file_id, 0, 5); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 7) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 8) Open the file R/W using the FAPL defined in 1) above. + * + * Verify that the file does not contain a cache image. + */ + + if ( pass ) { + + file_id = H5Fopen(filename, H5F_ACC_RDWR, fapl_id); + + if ( file_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fopen() failed.\n"; + + } else { + + file_ptr = (struct H5F_t *)H5I_object_verify(file_id, H5I_FILE); + + if ( file_ptr == NULL ) { + + pass = FALSE; + failure_mssg = "Can't get file_ptr."; + + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* get a pointer to the files internal data structure and then + * to the cache structure + */ + if ( pass ) { + + if ( file_ptr->shared->cache == NULL ) { + + pass = FALSE; + failure_mssg = "can't get cache pointer(1).\n"; + + } else { + + cache_ptr = file_ptr->shared->cache; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + if ( ( cache_ptr->load_image == TRUE ) || + ( cache_ptr->delete_image == TRUE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message present?\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 9) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + + /* 10) Open the file R/W using the FAPL defined in 1) above. + * Verify that the file does not contain a cache image. + */ + + if ( pass ) { + + file_id = H5Fopen(filename, H5F_ACC_RDWR, fapl_id); + + if ( file_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fopen() failed.\n"; + + } else { + + file_ptr = (struct H5F_t *)H5I_object_verify(file_id, H5I_FILE); + + if ( file_ptr == NULL ) { + + pass = FALSE; + failure_mssg = "Can't get file_ptr."; + + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* get a pointer to the files internal data structure and then + * to the cache structure + */ + if ( pass ) { + + if ( file_ptr->shared->cache == NULL ) { + + pass = FALSE; + failure_mssg = "can't get cache pointer(1).\n"; + + } else { + + cache_ptr = file_ptr->shared->cache; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + if ( ( cache_ptr->load_image == TRUE ) || + ( cache_ptr->delete_image == TRUE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message present?\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 11) Verify that the data sets contain the expected data + * + * Verify that a cache image was not loaded. + */ + + if ( pass ) { + + verify_datasets(file_id, 0, 5); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 12) Create several more data sets. */ + + if ( pass ) { + + create_datasets(file_id, 6, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 13) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 14) Open the file read write. + * + * Verify that the file does not contain a cache image. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 15) Verify the data sets exist and contain the expected data. + * + * Verify that a cache image was not loaded. + */ + + if ( pass ) { + + verify_datasets(file_id, 0, 10); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 16) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 17) Delete the file */ + + if ( pass ) { + + if ( HDremove(filename) < 0 ) { + + pass = FALSE; + failure_mssg = "HDremove() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* tidy up */ + if ( fapl_id != -1 ) + H5Pclose(fapl_id); + + + if ( pass ) { PASSED(); } else { H5_FAILED(); } + + if ( ! pass ) + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", + FUNC, failure_mssg); + + return !pass; + +} /* cache_image_api_error_check_4() */ + + +/*------------------------------------------------------------------------- + * Function: get_free_sections_test() + * + * Purpose: It is possible that H5Fget_free_sections() to be + * called before any activity on the metadata cache. + * This is a potential problem, as satisfying the + * H5Fget_free_sections() call requires access to all + * free space managers. When persistant free space + * managers are enabled, this will require calling + * H5MF_tidy_self_referential_fsm_hack(). This is a + * non issue in the absence of a cache image. However, + * this is a problem if a cache image exists, as + * the call to H5MF_tidy_self_referential_fsm_hack() + * will free the file space allocated to the cache + * image. + * + * The objective of this test is to create a test file + * with both non-empty self referential presistant + * free space managers, and a cache image, and then + * verify that this situation is handled correctly if + * H5Fget_free_sections() is called before the metadata + * cache image is loaded. + * + * The test is set up as follows: + * + * 1) Create a HDF5 file with a cache image requested + * and persistant free space managers enabled. + * + * 2) Create some data sets, and then delete some of + * of those near the beginning of the file. + * + * 3) Close the file. + * + * 4) Open the file read only. + * + * 5) Verify that a cache image exists, and has not + * been loaded. + * + * 6) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + * + * 7) Call H5Fget_free_sections(). + * + * 8) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + * + * 9) Verify that the remaining data sets contain the + * expected data. + * + * 10) Close the file. + * + * 11) Open the file R/W. + * + * 12) Verify that a cache image exists, and has not + * been loaded. + * + * 13) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + * + * 14) Call H5Fget_free_sections(). + * + * 15) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + * + * 16) Verify that the remaining data sets contain the + * expected data. + * + * 17) Delete the remaining data sets. + * + * 18) Close the file. + * + * 19) Verify that file space has been reclaimed. + * + * 20) Discard the file. + * + * Return: void + * + * Programmer: John Mainzer + * 1/10/17 + * + *------------------------------------------------------------------------- + */ +static unsigned +get_free_sections_test(void) +{ + const char * fcn_name = "get_free_sections_test()"; + char filename[512]; + hbool_t show_progress = FALSE; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + h5_stat_size_t file_size; + int cp = 0; + + TESTING("Cache image / H5Fget_free_sections() interaction"); + + pass = TRUE; + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + if ( h5_fixname(FILENAMES[0], H5P_DEFAULT, filename, sizeof(filename)) + == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 1) Create a HDF5 file with a cache image requested + * and persistant free space managers enabled. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ TRUE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ TRUE, + /* config_fsm */ TRUE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Create some data sets, and then delete some of + * of those near the beginning of the file. + */ + + if ( pass ) { + + create_datasets(file_id, 1, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + verify_datasets(file_id, 1, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + delete_datasets(file_id, 1, 5); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 3) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (1).\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 4) Open the file read only. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ TRUE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 5) Verify that a cache image exists, and has not been loaded. */ + + if ( pass ) { + + if ( ( ! file_ptr->shared->cache->load_image ) || + ( file_ptr->shared->cache->image_loaded ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 6) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + */ + + if ( pass ) { + + /* file_ptr->shared->first_alloc_dealloc is set to FALSE if the + * file is opened R/O. + */ + if ( ( file_ptr->shared->first_alloc_dealloc ) || + ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) || + ( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) || + ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc, + file_ptr->shared->cache->image_addr) ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status (1).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 7) Call H5Fget_free_sections(). */ + + if ( pass ) { + + if ( H5Fget_free_sections(file_id, H5FD_MEM_DEFAULT, (size_t)0, NULL) + < 0 ){ + + pass = FALSE; + failure_mssg = "H5Fget_free_sections() failed (1).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 8) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + */ + if ( pass ) { + + if ( ! file_ptr->shared->cache->image_loaded ) { + + pass = FALSE; + failure_mssg = "cache image not loaded (1).\n"; + + } else if ( file_ptr->shared->first_alloc_dealloc ) { + + pass = FALSE; + failure_mssg = "self referential FSMs not floated (1).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 9) Verify that the remaining data sets contain the expected data. */ + + if ( pass ) { + + verify_datasets(file_id, 6, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 10) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 11) Open the file R/W. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 12) Verify that a cache image exists, and has not been loaded. */ + + if ( pass ) { + + if ( ( ! file_ptr->shared->cache->load_image ) || + ( file_ptr->shared->cache->image_loaded ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 13) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + */ + if ( pass ) { + + if ( ( ! file_ptr->shared->first_alloc_dealloc ) || + ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) || + ( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) || + ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc, + file_ptr->shared->cache->image_addr) ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 14) Call H5Fget_free_sections(). */ + + if ( pass ) { + + if ( H5Fget_free_sections(file_id, H5FD_MEM_DEFAULT, (size_t)0, NULL) + < 0 ){ + + pass = FALSE; + failure_mssg = "H5Fget_free_sections() failed (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 15) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + */ + if ( pass ) { + + if ( ! file_ptr->shared->cache->image_loaded ) { + + pass = FALSE; + failure_mssg = "cache image not loaded (2).\n"; + + } else if ( file_ptr->shared->first_alloc_dealloc ) { + + pass = FALSE; + failure_mssg = "self referential FSMs not floated (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 16) Verify that the remaining data sets contain the expected data. */ + + if ( pass ) { + + verify_datasets(file_id, 6, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 17) Delete the remaining data sets. */ + + if ( pass ) { + + delete_datasets(file_id, 6, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 18) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (3).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 19) Verify that file space has been reclaimed. */ + + if ( pass ) { + + if((file_size = h5_get_file_size(filename, H5P_DEFAULT)) < 0) { + + pass = FALSE; + failure_mssg = "h5_get_file_size() failed.\n"; + + } else if ( file_size > 20 * 1024 ) { + + pass = FALSE; + failure_mssg = "unexpectedly large file size.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 20) Discard the file. */ + + if ( pass ) { + + if ( HDremove(filename) < 0 ) { + + pass = FALSE; + failure_mssg = "HDremove() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { PASSED(); } else { H5_FAILED(); } + + if ( ! pass ) + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", + FUNC, failure_mssg); + + return !pass; + +} /* get_free_sections_test() */ + + +/*------------------------------------------------------------------------- * Function: main * * Purpose: Run tests on the cache code contained in H5C.c @@ -6489,6 +7611,9 @@ main(void) nerrs += cache_image_api_error_check_1(); nerrs += cache_image_api_error_check_2(); nerrs += cache_image_api_error_check_3(); + nerrs += cache_image_api_error_check_4(); + + nerrs += get_free_sections_test(); return(nerrs > 0); diff --git a/test/cache_tagging.c b/test/cache_tagging.c index 02ce19b..957187b 100644 --- a/test/cache_tagging.c +++ b/test/cache_tagging.c @@ -53,13 +53,16 @@ /* ===================== */ /* Helper Functions */ +#ifndef NDEBUG static int dump_cache(hid_t fid); +#endif /* NDEBUG */ /* end debugging functions */ static int verify_no_unknown_tags(hid_t fid); static int mark_all_entries_investigated(hid_t fid); static int reset_all_entries_investigated(hid_t fid); static int verify_tag(hid_t fid, int id, haddr_t tag); static int get_object_header_tag(hid_t loc_id, haddr_t *tag); static int get_sbe_tag(hid_t fid, haddr_t *tag); + /* Tests */ static unsigned check_file_creation_tags(hid_t fcpl_id, int type); static unsigned check_file_open_tags(hid_t fcpl, int type); @@ -71,7 +74,6 @@ static unsigned check_attribute_rename_tags(hid_t fcpl, int type); static unsigned check_dataset_creation_tags(hid_t fcpl, int type); static unsigned check_dataset_creation_earlyalloc_tags(hid_t fcpl, int type); static unsigned check_link_removal_tags(hid_t fcpl, int type); - static unsigned check_group_creation_tags(void); static unsigned check_multi_group_creation_tags(void); static unsigned check_group_open_tags(void); @@ -95,6 +97,7 @@ static unsigned check_invalid_tag_application(void); /* ================ */ +#ifndef NDEBUG /*------------------------------------------------------------------------- * Function: dump_cache() @@ -128,6 +131,7 @@ static int dump_cache(hid_t fid) error: return -1; } /* dump_cache */ +#endif /* NDEBUG */ /* end debugging functions */ /*------------------------------------------------------------------------- @@ -445,8 +449,10 @@ check_file_creation_tags(hid_t fcpl_id, int type) /* Create a test file with provided fcpl_t */ if ( (fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT)) < 0 ) TEST_ERROR; +#ifndef NDEBUG /* if verbose, print cache index to screen before verification . */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* verify there is a superblock entry with superblock tag */ if ( verify_tag(fid, H5AC_SUPERBLOCK_ID, H5AC__SUPERBLOCK_TAG) < 0 ) TEST_ERROR; @@ -551,8 +557,10 @@ check_file_open_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen before verification . */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* verify there is a superblock entry with superblock tag. */ if ( verify_tag(fid, H5AC_SUPERBLOCK_ID, H5AC__SUPERBLOCK_TAG) < 0 ) TEST_ERROR; @@ -653,8 +661,10 @@ check_group_creation_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group's tagged metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -756,8 +766,10 @@ check_multi_group_creation_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify there is an object header for each group */ for (i = 0; i < MULTIGROUPS; i++) { @@ -888,8 +900,10 @@ check_link_iteration_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group's tagged metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -1005,8 +1019,10 @@ check_dense_attribute_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify free space header and section info */ if ( verify_tag(fid, H5AC_FSPACE_SINFO_ID, d_tag) < 0 ) TEST_ERROR; @@ -1061,8 +1077,10 @@ check_dense_attribute_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* verify object header belonging to dataset */ if ( verify_tag(fid, H5AC_OHDR_ID, d_tag) < 0 ) TEST_ERROR; @@ -1169,8 +1187,10 @@ check_group_open_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -1273,8 +1293,10 @@ check_attribute_creation_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* verify object header belonging to group */ if ( verify_tag(fid, H5AC_OHDR_ID, g_tag) < 0 ) TEST_ERROR; @@ -1407,8 +1429,10 @@ check_attribute_open_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* verify object header belonging to group */ if ( verify_tag(fid, H5AC_OHDR_ID, g_tag) < 0 ) TEST_ERROR; @@ -1490,6 +1514,7 @@ check_attribute_rename_tags(hid_t fcpl, int type) haddr_t g_tag = 0; hsize_t dims1[2] = {DIMS, DIMS}; /* dimensions */ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* dimensions */ + hbool_t persistant_fsms = FALSE; /* Testing Macro */ TESTING("tag application during attribute renaming"); @@ -1498,6 +1523,10 @@ check_attribute_rename_tags(hid_t fcpl, int type) /* Setup */ /* ===== */ + /* check to see if the FCPL specified persistant free space managers */ + if(H5Pget_file_space_strategy(fcpl, NULL, &persistant_fsms, NULL) < 0) + TEST_ERROR; + /* Allocate array */ if ( (NULL == (data = (int *)HDcalloc(DIMS * DIMS, sizeof(int)))) ) TEST_ERROR; @@ -1550,8 +1579,10 @@ check_attribute_rename_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -1586,9 +1617,18 @@ check_attribute_rename_tags(hid_t fcpl, int type) * 3 calls to verify_tag() for verifying free space: * one freespace header tag for H5FD_MEM_DRAW manager, * one freespace header tag for H5FD_MEM_SUPER manager + * one freespace section info tag for H5FD_MEM_SUPER manager */ if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; - if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; + + /* If the free space managers are persistant, the + * H5MF_tidy_self_referential_fsm_hack() must have been run. + * Since this function floats all self referential free space + * managers, the H5FD_MEM_SUPER FSM will not be in the metadata + * cache. + */ + if(!persistant_fsms && verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR; + if(!persistant_fsms && verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR; /* verify btree header and leaf node belonging to group */ if ( verify_tag(fid, H5AC_BT2_HDR_ID, g_tag) < 0 ) TEST_ERROR; @@ -1653,6 +1693,7 @@ check_attribute_delete_tags(hid_t fcpl, int type) haddr_t g_tag = 0; hsize_t dims1[2] = {DIMS, DIMS}; /* dimensions */ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* dimensions */ + hbool_t persistant_fsms = FALSE; /* Testing Macro */ TESTING("tag application during attribute delete"); @@ -1661,6 +1702,10 @@ check_attribute_delete_tags(hid_t fcpl, int type) /* Setup */ /* ===== */ + /* check to see if the FCPL specified persistant free space managers */ + if ( H5Pget_file_space_strategy(fcpl, NULL, &persistant_fsms, NULL) < 0 ) + TEST_ERROR; + /* Allocate array */ if ( (NULL == (data = (int *)HDcalloc(DIMS * DIMS, sizeof(int)))) ) TEST_ERROR; @@ -1712,8 +1757,10 @@ check_attribute_delete_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* verify object header belonging to group */ if ( verify_tag(fid, H5AC_OHDR_ID, g_tag) < 0 ) TEST_ERROR; @@ -1724,12 +1771,26 @@ check_attribute_delete_tags(hid_t fcpl, int type) if ( verify_tag(fid, H5AC_SOHM_TABLE_ID, H5AC__SOHM_TAG) < 0 ) TEST_ERROR; /* - * 2 calls to verify_tag() for verifying free space: - * one freespace header tag for H5FD_MEM_DRAW manager, - * one freespace header tag for H5FD_MEM_SUPER manager - */ - if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; - if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; + * 2 calls to verify_tag() for verifying free space: + * one freespace header tag for free-space header, + * one freespace header tag for free-space section info + */ + if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) + TEST_ERROR; + if ( verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0 ) + TEST_ERROR; + +#if 0 + /* If the free space managers are persistant, the + * H5MF_tidy_self_referential_fsm_hack() must have been run. + * Since this function floats all self referential free space + * managers, the H5FD_MEM_SUPER FSM will not be in the metadata + * cache. + */ + if ( ( ! persistant_fsms ) && + ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) ) + TEST_ERROR; +#endif } /* end if */ @@ -1834,8 +1895,10 @@ check_dataset_creation_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -1959,8 +2022,10 @@ check_dataset_creation_earlyalloc_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -2097,8 +2162,10 @@ check_dataset_open_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -2228,8 +2295,10 @@ check_dataset_write_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify 10 b-tree nodes belonging to dataset */ for (i=0; i<10; i++) @@ -2351,8 +2420,10 @@ check_attribute_write_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify object header of group */ if ( verify_tag(fid, H5AC_OHDR_ID, g_tag) < 0 ) TEST_ERROR; @@ -2506,8 +2577,10 @@ check_dataset_read_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify 19 b-tree nodes belonging to dataset */ for (i=0; i<19; i++) @@ -2636,8 +2709,10 @@ check_dataset_size_retrieval(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify 19 b-tree nodes belonging to dataset */ for (i=0; i<19; i++) @@ -2769,8 +2844,10 @@ check_dataset_extend_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, d_tag) < 0 ) TEST_ERROR; @@ -2866,8 +2943,10 @@ check_object_info_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group's tagged metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -2972,8 +3051,10 @@ check_object_copy_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group's tagged metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -3122,8 +3203,10 @@ check_link_removal_tags(hid_t fcpl, int type) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group's tagged metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -3271,8 +3354,10 @@ check_link_getname_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group's tagged metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -3371,8 +3456,10 @@ check_external_link_creation_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* Verify root group metadata */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -3478,8 +3565,10 @@ check_external_link_open_tags(void) /* Verification of Metadata Tag Values */ /* =================================== */ +#ifndef NDEBUG /* if verbose, print cache index to screen for visual verification */ if ( verbose ) dump_cache(fid); +#endif /* NDEBUG */ /* end debugging functions */ /* verify tag value of first file's root group */ if ( verify_tag(fid, H5AC_OHDR_ID, root_tag) < 0 ) TEST_ERROR; @@ -3643,7 +3732,7 @@ main(void) fcpl_shmesg_all = H5Pcreate(H5P_FILE_CREATE); H5Pset_shared_mesg_nindexes(fcpl_shmesg_all, 1); H5Pset_shared_mesg_index(fcpl_shmesg_all, 0, H5O_SHMESG_ALL_FLAG, 20); - H5Pset_file_space(fcpl_shmesg_all, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); + H5Pset_file_space_strategy(fcpl_shmesg_all, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)0); /* ========= */ /* Run Tests */ diff --git a/test/dsets.c b/test/dsets.c index 39b7c22..c090174 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -633,7 +633,7 @@ error: *------------------------------------------------------------------------- */ static herr_t -test_userblock_offset(const char *env_h5_drvr, hid_t fapl) +test_userblock_offset(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { char filename[FILENAME_BUF_SIZE]; hid_t file = -1, fcpl = -1, dataset = -1, space = -1; @@ -651,6 +651,9 @@ test_userblock_offset(const char *env_h5_drvr, hid_t fapl) if((fcpl=H5Pcreate(H5P_FILE_CREATE)) < 0) goto error; if(H5Pset_userblock(fcpl, (hsize_t)USER_BLOCK) < 0) goto error; + if(new_format) + if(H5Pset_file_space_page_size(fcpl, (hsize_t)USER_BLOCK) < 0) + goto error; if((file=H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) goto error; @@ -12658,18 +12661,24 @@ main(void) { char filename[FILENAME_BUF_SIZE]; hid_t file, grp, fapl, fapl2; + hid_t fcpl = -1, fcpl2 = -1; unsigned new_format; + unsigned paged; int mdc_nelmts; size_t rdcc_nelmts; size_t rdcc_nbytes; double rdcc_w0; int nerrors = 0; const char *envval; + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ /* Don't run this test using certain file drivers */ envval = HDgetenv("HDF5_DRIVER"); if(envval == NULL) - envval = "sec2"; + envval = "nomatch"; + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") && HDstrcmp(envval, "multi")); /* Set the random # seed */ HDsrandom((unsigned)HDtime(NULL)); @@ -12691,101 +12700,136 @@ main(void) /* Set the "use the latest version of the format" bounds for creating objects in the file */ if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR + /* create a file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + if((fcpl2 = H5Pcopy(fcpl)) < 0) TEST_ERROR + + /* Set file space strategy to paged aggregation and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + TEST_ERROR + h5_fixname(FILENAME[0], fapl, filename, sizeof filename); - /* Test with old & new format groups */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { - hid_t my_fapl; + /* Test with paged aggregation enabled or not */ + for(paged = FALSE; paged <= TRUE; paged++) { - /* Set the FAPL for the type of format */ - if(new_format) { - puts("\nTesting with new file format:"); - my_fapl = fapl2; - } /* end if */ - else { - puts("Testing with old file format:"); - my_fapl = fapl; - } /* end else */ + /* Temporary: skip testing for multi/split drivers: + fail file create when persisting free-space or using paged aggregation strategy */ + if(!contig_addr_vfd && paged) + continue; - /* Create the file for this test */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) - goto error; + /* Test with old & new format groups */ + for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl, my_fcpl; + + /* Set the FAPL for the type of format */ + if(new_format) { + my_fapl = fapl2; + if(paged) { + my_fcpl = fcpl2; + puts("\nTesting with new file format and paged aggregation"); + } else { + my_fcpl = fcpl; + puts("\nTesting with new file format and non-paged aggregation"); + } + } /* end if */ + else { + my_fapl = fapl; + if(paged) { + my_fcpl = fcpl2; + puts("Testing with old file format and paged aggregation:"); + } else { + my_fcpl = fcpl; + puts("Testing with old file format and non-paged aggregation:"); + } + } /* end else */ - /* Cause the library to emit initial messages */ - if((grp = H5Gcreate2(file, "emit diagnostics", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; - if(H5Oset_comment(grp, "Causes diagnostic messages to be emitted") < 0) - goto error; - if(H5Gclose(grp) < 0) - goto error; + /* Create the file for this test */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, my_fcpl, my_fapl)) < 0) + goto error; + + /* Cause the library to emit initial messages */ + if((grp = H5Gcreate2(file, "emit diagnostics", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if(H5Oset_comment(grp, "Causes diagnostic messages to be emitted") < 0) + goto error; + if(H5Gclose(grp) < 0) + goto error; + + nerrors += (test_create(file) < 0 ? 1 : 0); + nerrors += (test_simple_io(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_compact_io(my_fapl) < 0 ? 1 : 0); + nerrors += (test_max_compact(my_fapl) < 0 ? 1 : 0); + nerrors += (test_conv_buffer(file) < 0 ? 1 : 0); + nerrors += (test_tconv(file) < 0 ? 1 : 0); + nerrors += (test_filters(file, my_fapl) < 0 ? 1 : 0); + nerrors += (test_onebyte_shuffle(file) < 0 ? 1 : 0); + nerrors += (test_nbit_int(file) < 0 ? 1 : 0); + nerrors += (test_nbit_float(file) < 0 ? 1 : 0); + nerrors += (test_nbit_double(file) < 0 ? 1 : 0); + nerrors += (test_nbit_array(file) < 0 ? 1 : 0); + nerrors += (test_nbit_compound(file) < 0 ? 1 : 0); + nerrors += (test_nbit_compound_2(file) < 0 ? 1 : 0); + nerrors += (test_nbit_compound_3(file) < 0 ? 1 : 0); + nerrors += (test_nbit_int_size(file) < 0 ? 1 : 0); + nerrors += (test_nbit_flt_size(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_int(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_int_2(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_float(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_float_2(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_double(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_double_2(file) < 0 ? 1 : 0); + nerrors += (test_multiopen (file) < 0 ? 1 : 0); + nerrors += (test_types(file) < 0 ? 1 : 0); + nerrors += (test_userblock_offset(envval, my_fapl, new_format) < 0 ? 1 : 0); + nerrors += (test_missing_filter(file) < 0 ? 1 : 0); + nerrors += (test_can_apply(file) < 0 ? 1 : 0); + nerrors += (test_can_apply2(file) < 0 ? 1 : 0); + nerrors += (test_set_local(my_fapl) < 0 ? 1 : 0); + nerrors += (test_can_apply_szip(file) < 0 ? 1 : 0); + nerrors += (test_compare_dcpl(file) < 0 ? 1 : 0); + nerrors += (test_copy_dcpl(file, my_fapl) < 0 ? 1 : 0); + nerrors += (test_filter_delete(file) < 0 ? 1 : 0); + nerrors += (test_filters_endianess() < 0 ? 1 : 0); + nerrors += (test_zero_dims(file) < 0 ? 1 : 0); + nerrors += (test_missing_chunk(file) < 0 ? 1 : 0); + nerrors += (test_random_chunks(my_fapl) < 0 ? 1 : 0); - nerrors += (test_create(file) < 0 ? 1 : 0); - nerrors += (test_simple_io(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_compact_io(my_fapl) < 0 ? 1 : 0); - nerrors += (test_max_compact(my_fapl) < 0 ? 1 : 0); - nerrors += (test_conv_buffer(file) < 0 ? 1 : 0); - nerrors += (test_tconv(file) < 0 ? 1 : 0); - nerrors += (test_filters(file, my_fapl) < 0 ? 1 : 0); - nerrors += (test_onebyte_shuffle(file) < 0 ? 1 : 0); - nerrors += (test_nbit_int(file) < 0 ? 1 : 0); - nerrors += (test_nbit_float(file) < 0 ? 1 : 0); - nerrors += (test_nbit_double(file) < 0 ? 1 : 0); - nerrors += (test_nbit_array(file) < 0 ? 1 : 0); - nerrors += (test_nbit_compound(file) < 0 ? 1 : 0); - nerrors += (test_nbit_compound_2(file) < 0 ? 1 : 0); - nerrors += (test_nbit_compound_3(file) < 0 ? 1 : 0); - nerrors += (test_nbit_int_size(file) < 0 ? 1 : 0); - nerrors += (test_nbit_flt_size(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_int(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_int_2(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_float(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_float_2(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_double(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_double_2(file) < 0 ? 1 : 0); - nerrors += (test_multiopen (file) < 0 ? 1 : 0); - nerrors += (test_types(file) < 0 ? 1 : 0); - nerrors += (test_userblock_offset(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_missing_filter(file) < 0 ? 1 : 0); - nerrors += (test_can_apply(file) < 0 ? 1 : 0); - nerrors += (test_can_apply2(file) < 0 ? 1 : 0); - nerrors += (test_set_local(my_fapl) < 0 ? 1 : 0); - nerrors += (test_can_apply_szip(file) < 0 ? 1 : 0); - nerrors += (test_compare_dcpl(file) < 0 ? 1 : 0); - nerrors += (test_copy_dcpl(file, my_fapl) < 0 ? 1 : 0); - nerrors += (test_filter_delete(file) < 0 ? 1 : 0); - nerrors += (test_filters_endianess() < 0 ? 1 : 0); - nerrors += (test_zero_dims(file) < 0 ? 1 : 0); - nerrors += (test_missing_chunk(file) < 0 ? 1 : 0); - nerrors += (test_random_chunks(my_fapl) < 0 ? 1 : 0); #ifndef H5_NO_DEPRECATED_SYMBOLS - nerrors += (test_deprec(file) < 0 ? 1 : 0); + nerrors += (test_deprec(file) < 0 ? 1 : 0); #endif /* H5_NO_DEPRECATED_SYMBOLS */ - nerrors += (test_huge_chunks(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_cache(my_fapl) < 0 ? 1 : 0); - nerrors += (test_big_chunks_bypass_cache(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_fast(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_reopen_chunk_fast(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_fast_bug1(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_expand(my_fapl) < 0 ? 1 : 0); - nerrors += (test_layout_extend(my_fapl) < 0 ? 1 : 0); - nerrors += (test_fixed_array(my_fapl) < 0 ? 1 : 0); - nerrors += (test_idx_compatible() < 0 ? 1 : 0); - nerrors += (test_unfiltered_edge_chunks(my_fapl) < 0 ? 1 : 0); - nerrors += (test_single_chunk(my_fapl) < 0 ? 1 : 0); - nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); - nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0); - nerrors += (test_swmr_non_latest(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_earray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_farray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_bt2_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_storage_size(my_fapl) < 0 ? 1 : 0); - - if(H5Fclose(file) < 0) - goto error; - } /* end for */ - /* Close 2nd FAPL */ + nerrors += (test_huge_chunks(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_cache(my_fapl) < 0 ? 1 : 0); + nerrors += (test_big_chunks_bypass_cache(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_fast(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_reopen_chunk_fast(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_fast_bug1(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_expand(my_fapl) < 0 ? 1 : 0); + nerrors += (test_layout_extend(my_fapl) < 0 ? 1 : 0); + nerrors += (test_fixed_array(my_fapl) < 0 ? 1 : 0); + nerrors += (test_idx_compatible() < 0 ? 1 : 0); + nerrors += (test_unfiltered_edge_chunks(my_fapl) < 0 ? 1 : 0); + nerrors += (test_single_chunk(my_fapl) < 0 ? 1 : 0); + nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); + nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0); + nerrors += (test_storage_size(my_fapl) < 0 ? 1 : 0); + + nerrors += (test_swmr_non_latest(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_earray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_farray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_bt2_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); + + if(H5Fclose(file) < 0) + goto error; + } /* end for new_format */ + } /* end for paged */ + + /* Close property lists */ if(H5Pclose(fapl2) < 0) TEST_ERROR + if(H5Pclose(fcpl) < 0) TEST_ERROR + if(H5Pclose(fcpl2) < 0) TEST_ERROR /* Tests that do not use files */ nerrors += (test_scatter() < 0 ? 1 : 0); diff --git a/test/evict_on_close.c b/test/evict_on_close.c index 3986d5a..b00c1e4 100644 --- a/test/evict_on_close.c +++ b/test/evict_on_close.c @@ -40,6 +40,7 @@ #include "H5Ipkg.h" /* Uncomment to manually inspect cache states */ +/* (Requires debug build of the library) */ /* #define EOC_MANUAL_INSPECTION */ const char *FILENAMES[] = { diff --git a/test/fheap.c b/test/fheap.c index 82859d2..f08c0f8 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -26,6 +26,10 @@ #define H5HF_TESTING #include "H5HFpkg.h" /* Fractal heaps */ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" + /* Other private headers that this test requires */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ @@ -126,6 +130,7 @@ typedef struct fheap_test_param_t { fheap_test_fill_t fill; /* How to "bulk" fill heap blocks */ size_t actual_id_len; /* The actual length of heap IDs for a test */ fheap_test_comp_t comp; /* Whether to compress the blocks or not */ + hid_t my_fcpl; /* File creation property list with file space strategy setting */ } fheap_test_param_t; /* Heap state information */ @@ -642,7 +647,7 @@ open_heap(char *filename, hid_t fapl, hid_t dxpl, const H5HF_create_t *cparam, h5_fixname(FILENAME[0], fapl, filename, (size_t)FHEAP_FILENAME_LEN); /* Create the file to work on */ - if((*file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((*file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Check for deleting the entire heap */ @@ -1827,7 +1832,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ char filename[FHEAP_FILENAME_LEN]; /* Filename to use */ @@ -1844,7 +1849,7 @@ test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Close file */ @@ -1944,7 +1949,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ char filename[FHEAP_FILENAME_LEN]; /* Filename to use */ @@ -1956,12 +1961,13 @@ test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED h5_stat_size_t file_size; /* File size, after deleting heap */ size_t id_len; /* Size of fractal heap IDs */ fheap_heap_state_t state; /* State of fractal heap */ + hbool_t page = FALSE; /* Paged aggregation strategy or not */ /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Close file */ @@ -1980,6 +1986,9 @@ test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED if(NULL == (f = (H5F_t *)H5I_object(file))) STACK_ERROR + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) + page = TRUE; + /* Ignore metadata tags in the file's cache */ if (H5AC_ignore_tags(f) < 0) FAIL_STACK_ERROR @@ -2058,8 +2067,9 @@ test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED TEST_ERROR /* Verify the file is correct size */ - if(file_size != empty_size) - TEST_ERROR + if(!page || (page && !tparam->reopen_heap)) + if(file_size != empty_size) + TEST_ERROR /* All tests passed */ PASSED() @@ -2090,7 +2100,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ hid_t file2 = -1; /* File ID */ @@ -2105,12 +2115,13 @@ test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UN h5_stat_size_t file_size; /* File size, after deleting heap */ size_t id_len; /* Size of fractal heap IDs */ fheap_heap_state_t state; /* State of fractal heap */ + hbool_t page = FALSE; /* Paged aggregation strategy or not */ /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Close file */ @@ -2129,6 +2140,9 @@ test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UN if(NULL == (f = (H5F_t *)H5I_object(file))) STACK_ERROR + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) + page = TRUE; + /* Ignore metadata tags in the file's cache */ if (H5AC_ignore_tags(f) < 0) FAIL_STACK_ERROR @@ -2226,8 +2240,9 @@ test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UN TEST_ERROR /* Verify the file is correct size */ - if(file_size != empty_size) - TEST_ERROR + if(!page || (page && !tparam->reopen_heap)) + if(file_size != empty_size) + TEST_ERROR /* All tests passed */ PASSED() @@ -2262,7 +2277,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_delete_open(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_delete_open(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ char filename[FHEAP_FILENAME_LEN]; /* Filename to use */ @@ -2280,7 +2295,7 @@ test_delete_open(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_U h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Close file */ @@ -2433,7 +2448,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_id_limits(hid_t fapl, H5HF_create_t *cparam) +test_id_limits(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file = -1; /* File ID */ hid_t dxpl = H5AC_ind_read_dxpl_id; /* DXPL to use */ @@ -2451,7 +2466,7 @@ test_id_limits(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -2778,7 +2793,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_filtered_create(hid_t fapl, H5HF_create_t *cparam) +test_filtered_create(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file = -1; /* File ID */ hid_t dxpl = H5AC_ind_read_dxpl_id; /* DXPL to use */ @@ -2794,7 +2809,7 @@ test_filtered_create(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -2901,7 +2916,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_size(hid_t fapl, H5HF_create_t *cparam) +test_size(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file = -1; /* File ID */ hid_t dxpl = H5AC_ind_read_dxpl_id; /* DXPL to use */ @@ -2917,7 +2932,7 @@ test_size(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3045,7 +3060,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_reopen_hdr(hid_t fapl, H5HF_create_t *cparam) +test_reopen_hdr(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file1 = -1; /* File ID */ hid_t file2 = -2; /* File ID */ @@ -3060,7 +3075,7 @@ test_reopen_hdr(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file1 = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3199,7 +3214,7 @@ test_man_insert_weird(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3309,7 +3324,7 @@ test_man_insert_first(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3410,7 +3425,7 @@ test_man_insert_second(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tp h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3507,7 +3522,7 @@ test_man_insert_root_mult(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3606,7 +3621,7 @@ test_man_insert_force_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test_par h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3712,7 +3727,7 @@ test_man_insert_fill_second(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3819,7 +3834,7 @@ test_man_insert_third_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_param h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3930,7 +3945,7 @@ test_man_fill_first_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4026,7 +4041,7 @@ test_man_start_second_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4129,7 +4144,7 @@ test_man_fill_second_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4230,7 +4245,7 @@ test_man_start_third_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4341,7 +4356,7 @@ test_man_fill_fourth_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4438,7 +4453,7 @@ test_man_fill_all_root_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_para h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4534,7 +4549,7 @@ test_man_first_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4636,7 +4651,7 @@ test_man_second_direct_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fhe h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4746,7 +4761,7 @@ test_man_fill_first_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4849,7 +4864,7 @@ test_man_second_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4960,7 +4975,7 @@ test_man_fill_second_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5068,7 +5083,7 @@ test_man_fill_recursive_indirect_row(hid_t fapl, H5HF_create_t *cparam, fheap_te h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5166,7 +5181,7 @@ test_man_start_2nd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5275,7 +5290,7 @@ test_man_recursive_indirect_two_deep(hid_t fapl, H5HF_create_t *cparam, fheap_te h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5378,7 +5393,7 @@ test_man_start_3rd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5488,7 +5503,7 @@ test_man_fill_first_3rd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fh h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5599,7 +5614,7 @@ test_man_fill_3rd_recursive_indirect_row(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5706,7 +5721,7 @@ test_man_fill_all_3rd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5814,7 +5829,7 @@ test_man_start_4th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5929,7 +5944,7 @@ test_man_fill_first_4th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fh h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6049,7 +6064,7 @@ test_man_fill_4th_recursive_indirect_row(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6161,7 +6176,7 @@ test_man_fill_all_4th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6276,7 +6291,7 @@ test_man_start_5th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6410,7 +6425,7 @@ test_man_remove_bogus(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6563,7 +6578,7 @@ test_man_remove_one(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6727,7 +6742,7 @@ test_man_remove_two(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6920,7 +6935,7 @@ test_man_remove_one_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -7089,7 +7104,7 @@ test_man_remove_two_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -7333,7 +7348,7 @@ test_man_remove_three_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -7632,7 +7647,7 @@ test_man_incr_insert_remove(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -16353,6 +16368,8 @@ main(void) H5HF_create_t small_cparam; /* Creation parameters for "small" heap */ H5HF_create_t large_cparam; /* Creation parameters for "large" heap */ hid_t fapl = -1; /* File access property list for data files */ + hid_t fcpl = -1; /* File creation property list for data files */ + hid_t fcpl2 = -1; /* File creation property list for data files */ fheap_test_type_t curr_test; /* Current test being worked on */ unsigned u; /* Local index variable */ unsigned nerrors = 0; /* Cumulative error count */ @@ -16375,6 +16392,16 @@ main(void) shared_wobj_g = (unsigned char *)H5MM_malloc(shared_obj_size_g); shared_robj_g = (unsigned char *)H5MM_malloc(shared_obj_size_g); + /* create a file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + if((fcpl2 = H5Pcopy(fcpl)) < 0) TEST_ERROR + + /* Set file space strategy and persisting free-space */ + /* This will be modified later on to run the test with different file space strategy setting */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_FSM_AGGR, FALSE, (hsize_t)1) < 0) + TEST_ERROR + /* Initialize the shared write buffer for objects */ for(u = 0; u < shared_obj_size_g; u++) shared_wobj_g[u] = (unsigned char)u; @@ -16385,6 +16412,9 @@ main(void) HDmemset(&tparam, 0, sizeof(fheap_test_param_t)); tparam.actual_id_len = HEAP_ID_LEN; + /* This will be modified later on to run the test with different file space strategy setting */ + tparam.my_fcpl = fcpl2; + /* Set appropriate testing parameters for each test */ switch(curr_test) { /* "Normal" testing parameters */ @@ -16409,10 +16439,11 @@ main(void) nerrors += test_reopen(fapl, &small_cparam, &tparam); nerrors += test_open_twice(fapl, &small_cparam, &tparam); nerrors += test_delete_open(fapl, &small_cparam, &tparam); - nerrors += test_id_limits(fapl, &small_cparam); - nerrors += test_filtered_create(fapl, &small_cparam); - nerrors += test_size(fapl, &small_cparam); - nerrors += test_reopen_hdr(fapl, &small_cparam); + + nerrors += test_id_limits(fapl, &small_cparam, tparam.my_fcpl); + nerrors += test_filtered_create(fapl, &small_cparam, tparam.my_fcpl); + nerrors += test_size(fapl, &small_cparam, tparam.my_fcpl); + nerrors += test_reopen_hdr(fapl, &small_cparam, tparam.my_fcpl); { fheap_test_fill_t fill; /* Size of objects to fill heap blocks with */ @@ -16732,6 +16763,9 @@ main(void) H5MM_xfree(shared_lens_g); H5MM_xfree(shared_offs_g); + if(H5Pclose(fcpl) < 0) TEST_ERROR + if(H5Pclose(fcpl2) < 0) TEST_ERROR + /* Clean up file used */ #ifndef QAK h5_cleanup(FILENAME, fapl); diff --git a/test/filespace_1_8.h5 b/test/filespace_1_8.h5 Binary files differindex 85138b0..3fa2822 100644 --- a/test/filespace_1_8.h5 +++ b/test/filespace_1_8.h5 diff --git a/test/freespace.c b/test/freespace.c index 181e6a1..2bf9111 100644 --- a/test/freespace.c +++ b/test/freespace.c @@ -86,7 +86,7 @@ typedef struct TEST_free_section_t { static herr_t TEST_sect_init_cls(H5FS_section_class_t *, void *); static herr_t TEST_sect_free(H5FS_section_info_t *_sect); static herr_t TEST_sect_can_merge(const H5FS_section_info_t *, const H5FS_section_info_t *, void H5_ATTR_UNUSED *); -static herr_t TEST_sect_merging(H5FS_section_info_t *, H5FS_section_info_t *, void H5_ATTR_UNUSED *); +static herr_t TEST_sect_merging(H5FS_section_info_t **, H5FS_section_info_t *, void H5_ATTR_UNUSED *); static herr_t TEST_sect_can_shrink(const H5FS_section_info_t *, void *); static herr_t TEST_sect_shrinking(H5FS_section_info_t **, void *); @@ -245,26 +245,26 @@ TEST_sect_can_merge(const H5FS_section_info_t *_sect1, * Merge the two sections (second section is merged into the first section) */ static herr_t -TEST_sect_merging(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +TEST_sect_merging(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) { - TEST_free_section_t *sect1 = (TEST_free_section_t *)_sect1; + TEST_free_section_t **sect1 = (TEST_free_section_t **)_sect1; TEST_free_section_t *sect2 = (TEST_free_section_t *)_sect2; herr_t ret_value = SUCCEED; /* Return value */ /* Check arguments. */ HDassert(sect1); - HDassert((sect1->sect_info.type == TEST_FSPACE_SECT_TYPE) || - (sect1->sect_info.type == TEST_FSPACE_SECT_TYPE_NEW) || - (sect1->sect_info.type == TEST_FSPACE_SECT_TYPE_NONE)); + HDassert(((*sect1)->sect_info.type == TEST_FSPACE_SECT_TYPE) || + ((*sect1)->sect_info.type == TEST_FSPACE_SECT_TYPE_NEW) || + ((*sect1)->sect_info.type == TEST_FSPACE_SECT_TYPE_NONE)); HDassert(sect2); HDassert((sect2->sect_info.type == TEST_FSPACE_SECT_TYPE) || (sect2->sect_info.type == TEST_FSPACE_SECT_TYPE_NEW) || (sect2->sect_info.type == TEST_FSPACE_SECT_TYPE_NONE)); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ if(TEST_sect_free((H5FS_section_info_t *)sect2) < 0) @@ -2540,7 +2540,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend a block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_SIZE80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE40)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_SIZE80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE40, 0, NULL)) < 0) FAIL_STACK_ERROR if(FALSE == status) TEST_ERROR @@ -2616,7 +2616,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend the block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE50)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE50, 0, NULL)) < 0) FAIL_STACK_ERROR if(TRUE == status) TEST_ERROR @@ -2689,7 +2689,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend the block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE30)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE30, 0, NULL)) < 0) TEST_ERROR if(FALSE == status) TEST_ERROR @@ -2763,7 +2763,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend the block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE15, (hsize_t)TEST_SECT_SIZE40)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE15, (hsize_t)TEST_SECT_SIZE40, 0, NULL)) < 0) TEST_ERROR if(TRUE == status) TEST_ERROR diff --git a/test/fsm_aggr_nopersist.h5 b/test/fsm_aggr_nopersist.h5 Binary files differnew file mode 100644 index 0000000..159e7f7 --- /dev/null +++ b/test/fsm_aggr_nopersist.h5 diff --git a/test/fsm_aggr_persist.h5 b/test/fsm_aggr_persist.h5 Binary files differnew file mode 100644 index 0000000..1a837dd --- /dev/null +++ b/test/fsm_aggr_persist.h5 diff --git a/test/gen_filespace.c b/test/gen_filespace.c index e0c42e8..52715ae 100644 --- a/test/gen_filespace.c +++ b/test/gen_filespace.c @@ -14,69 +14,105 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hdf5.h" +#include <assert.h> #define NELMTS(X) (sizeof(X)/sizeof(X[0])) /* # of elements */ -#define TEST_THRESHOLD2 2 /* Free space section threshold */ const char *FILENAMES[] = { - "filespace_persist.h5", /* H5F_FILE_SPACE_ALL_PERSIST */ - "filespace_default.h5", /* H5F_FILE_SPACE_ALL */ - "filespace_aggr_vfd.h5", /* H5F_FILE_SPACE_AGGR_VFD */ - "filespace_vfd.h5", /* H5F_FILE_SPACE_VFD */ - "filespace_threshold.h5" /* H5F_FILE_SPACE_ALL, non-default threshold */ + "fsm_aggr_nopersist.h5", /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ + "fsm_aggr_persist.h5", /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ + "paged_nopersist.h5", /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ + "paged_persist.h5", /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ + "aggr.h5", /* H5F_FSPACE_STRATEGY_AGGR */ + "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ }; #define DATASET "dset" #define NUM_ELMTS 100 +#define FALSE 0 +#define TRUE 1 +#define INC_ENUM(TYPE,VAR) (VAR)=((TYPE)((VAR)+1)) /* - * Compile and run this program in file-space branch to generate - * HDF5 files with different kinds of file space strategies - * Move the HDF5 files to the 1.6 and 1.8 branch for compatibility - * testing:test_filespace_compatible() will use the files + * Compile and run this program in the trunk to generate + * HDF5 files with combinations of 4 file space strategies + * and persist/not persist free-space. + * The library creates the file space info message with "mark if unknown" + * in these files. + * + * Move these files to 1.8 branch for compatibility testing: + * test_filespace_compatible() in test/tfile.c will use these files. + * + * Copy these files from the 1.8 branch back to the trunk for + * compatibility testing via test_filespace_round_compatible() in test/tfile.c. + * */ -static void gen_file(void) -{ - hid_t fid; - hid_t fcpl; - hid_t dataset, space; - hsize_t dim[1]; - int data[NUM_ELMTS]; - size_t j; /* Local index variable */ - int i; /* Local index variable */ - H5F_file_space_type_t fs_type; /* File space handling strategy */ - - for(j = 0, fs_type = H5F_FILE_SPACE_ALL_PERSIST; j < NELMTS(FILENAMES); j++, fs_type = (H5F_file_space_type_t)(fs_type + 1)) { - /* Get a copy of the default file creation property */ - fcpl = H5Pcreate(H5P_FILE_CREATE); - - if(fs_type == H5F_FILE_SPACE_NTYPES) /* last file */ - /* Set default strategy but non-default threshold */ - H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL, (hsize_t)TEST_THRESHOLD2); - else - /* Set specified file space strategy and free space section threshold */ - H5Pset_file_space(fcpl, fs_type, (hsize_t)0); - - /* Create the file with the file space info */ - fid = H5Fcreate(FILENAMES[j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT); - - dim[0] = NUM_ELMTS; - space = H5Screate_simple(1, dim, NULL); - dataset = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - for(i = 0; i < NUM_ELMTS; i++) - data[i] = i; - - H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - H5Dclose(dataset); - H5Sclose(space); - H5Fclose(fid); - } -} - int main(void) { - gen_file(); + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hsize_t dim[1]; /* Dimension sizes */ + int data[NUM_ELMTS]; /* Buffer for data */ + int i, j; /* Local index variables */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ + unsigned fs_persist; /* Persisting free-space or not */ + + j = 0; + for(fs_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_strategy < H5F_FSPACE_STRATEGY_NTYPES; INC_ENUM(H5F_fspace_strategy_t, fs_strategy)) { + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + + if(fs_persist && fs_strategy >= H5F_FSPACE_STRATEGY_AGGR) + continue; + + /* Get a copy of the default file creation property */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + + if(H5Pset_file_space_strategy(fcpl, fs_strategy, fs_persist, (hsize_t)1) < 0) + goto error; + + /* Create the file with the file space info */ + if((fid = H5Fcreate(FILENAMES[j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Create the dataset */ + dim[0] = NUM_ELMTS; + if((sid = H5Screate_simple(1, dim, NULL)) < 0) + goto error; + if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + for(i = 0; i < NUM_ELMTS; i++) + data[i] = i; + + /* Write the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + goto error; + + /* Closing */ + if(H5Dclose(did) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Fclose(fid) < 0) + goto error; + if(H5Pclose(fcpl) < 0) + goto error; + ++j; + } + } + assert(j == NELMTS(FILENAMES)); return 0; + +error: + H5E_BEGIN_TRY { + H5Sclose(sid); + H5Sclose(did); + H5Pclose(fcpl); + H5Fclose(fid); + } H5E_END_TRY; + return -1; } diff --git a/test/gen_idx.c b/test/gen_idx.c deleted file mode 100644 index 8c24198..0000000 --- a/test/gen_idx.c +++ /dev/null @@ -1,126 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the files COPYING and Copyright.html. COPYING can be found at the root * - * of the source code distribution tree; Copyright.html can be found at the * - * root level of an installed copy of the electronic HDF5 document set and * - * is linked from the top-level documents page. It can also be found at * - * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * - * access to either file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Purpose: This program is run to generate an HDF5 data file with datasets - * that use Fixed Array indexing method. - * - * To test compatibility, compile and run this program - * which will generate a file called "fixed_idx.h5". - * Move it to the test directory in the HDF5 v1.6/1.8 source tree. - * The test: test_idx_compatible() in dsets.c will read it. - */ -#include <assert.h> -#include "hdf5.h" - -const char *FILENAME[1] = { - "fixed_idx.h5" /* file with datasets that use Fixed Array indexing method */ -}; - -#define DSET "dset" -#define DSET_FILTER "dset_filter" - -/* - * Function: gen_idx_file - * - * Purpose: Create a file with datasets that use Fixed Array indexing: - * one dataset: fixed dimension, chunked layout, w/o filters - * one dataset: fixed dimension, chunked layout, w/ filters - * - */ -static void gen_idx_file(void) -{ - hid_t fapl; /* file access property id */ - hid_t fid; /* file id */ - hid_t sid; /* space id */ - hid_t dcpl; /* dataset creation property id */ - hid_t did, did2; /* dataset id */ - hsize_t dims[1] = {10}; /* dataset dimension */ - hsize_t c_dims[1] = {2}; /* chunk dimension */ - herr_t status; /* return status */ - int i; /* local index variable */ - int buf[10]; /* data buffer */ - - - /* Get a copy of the file aaccess property */ - fapl = H5Pcreate(H5P_FILE_ACCESS); - assert(fapl >= 0); - - /* Set the "use the latest format" bounds for creating objects in the file */ - status = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); - assert(status >= 0); - - /* Create dataset */ - fid = H5Fcreate(FILENAME[0], H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - assert(fid >= 0); - - /* Create data */ - for(i = 0; i < 10; i++) - buf[i] = i; - - /* Set chunk */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - assert(dcpl >= 0); - status = H5Pset_chunk(dcpl, 1, c_dims); - assert(status >= 0); - - sid = H5Screate_simple(1, dims, NULL); - assert(sid >= 0); - - /* Create a 1D dataset */ - did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - assert(did >= 0); - - /* Write to the dataset */ - status = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); - assert(status >= 0); - -#if defined (H5_HAVE_FILTER_DEFLATE) - /* set deflate data */ - status = H5Pset_deflate(dcpl, 9); - assert(status >= 0); - - /* Create and write the dataset */ - did2 = H5Dcreate2(fid, DSET_FILTER, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - assert(did2 >= 0); - - status = H5Dwrite(did2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); - assert(status >= 0); - - /* Close the dataset */ - status = H5Dclose(did2); - assert(status >= 0); -#endif - - /* closing */ - status = H5Dclose(did); - assert(status >= 0); - status = H5Sclose(sid); - assert(status >= 0); - status = H5Pclose(dcpl); - assert(status >= 0); - status = H5Pclose(fapl); - assert(status >= 0); - status = H5Fclose(fid); - assert(status >= 0); -} /* gen_idx_file() */ - -int main(void) -{ - gen_idx_file(); - - return 0; -} - diff --git a/test/gen_plist.c b/test/gen_plist.c index 8cb6c00..5c54ce8 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -403,6 +403,12 @@ main(void) if((ret = H5Pset_sizes(fcpl1, 8, 4) < 0)) assert(ret > 0); + if((ret = H5Pset_file_space_strategy(fcpl1, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1)) < 0) + assert(ret > 0); + + if((ret = H5Pset_file_space_page_size(fcpl1, (hsize_t)4096)) < 0) + assert(ret > 0); + if((ret = encode_plist(fcpl1, little_endian, word_length, "testfiles/plist_files/fcpl_")) < 0) assert(ret > 0); diff --git a/test/h5fc_ext1_f.h5 b/test/h5fc_ext1_f.h5 Binary files differnew file mode 100644 index 0000000..b5c5867 --- /dev/null +++ b/test/h5fc_ext1_f.h5 diff --git a/test/h5fc_ext1_i.h5 b/test/h5fc_ext1_i.h5 Binary files differnew file mode 100644 index 0000000..960a8d5 --- /dev/null +++ b/test/h5fc_ext1_i.h5 diff --git a/test/h5fc_ext2_if.h5 b/test/h5fc_ext2_if.h5 Binary files differnew file mode 100644 index 0000000..88e42e7 --- /dev/null +++ b/test/h5fc_ext2_if.h5 diff --git a/test/h5fc_ext2_sf.h5 b/test/h5fc_ext2_sf.h5 Binary files differnew file mode 100644 index 0000000..c59a3ca --- /dev/null +++ b/test/h5fc_ext2_sf.h5 diff --git a/test/h5fc_ext3_isf.h5 b/test/h5fc_ext3_isf.h5 Binary files differnew file mode 100644 index 0000000..d00fc55 --- /dev/null +++ b/test/h5fc_ext3_isf.h5 diff --git a/test/h5fc_ext_none.h5 b/test/h5fc_ext_none.h5 Binary files differnew file mode 100644 index 0000000..b1b1553 --- /dev/null +++ b/test/h5fc_ext_none.h5 diff --git a/test/links.c b/test/links.c index 3364c7e..e3bbdb2 100644 --- a/test/links.c +++ b/test/links.c @@ -11102,6 +11102,8 @@ corder_delete(hid_t fapl) for(reopen_file = FALSE; reopen_file <= TRUE; reopen_file++) { /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + + /* Creating file with latest format will enable paged aggregation with persistent fs */ if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR /* Close file */ @@ -20,6 +20,7 @@ * test_mf_aggr_*() tests for file memory that interact with the aggregators * test_mf_align_*() tests for file memory with alignment setting * test_filespace_*() tests for file space management + * test_page_*() tests for file space paging */ #include "h5test.h" @@ -38,41 +39,63 @@ #include "H5Iprivate.h" #include "H5VMprivate.h" -#define FILENAME_LEN 1024 - -#define TEST_BLOCK_SIZE1 1 -#define TEST_BLOCK_SIZE2 2 -#define TEST_BLOCK_SIZE3 3 -#define TEST_BLOCK_SIZE4 4 -#define TEST_BLOCK_SIZE5 5 -#define TEST_BLOCK_SIZE6 6 -#define TEST_BLOCK_SIZE7 7 -#define TEST_BLOCK_SIZE8 8 -#define TEST_BLOCK_SIZE20 20 -#define TEST_BLOCK_SIZE30 30 -#define TEST_BLOCK_SIZE40 40 -#define TEST_BLOCK_SIZE50 50 -#define TEST_BLOCK_SIZE80 80 -#define TEST_BLOCK_SIZE200 200 -#define TEST_BLOCK_SIZE600 600 -#define TEST_BLOCK_SIZE700 700 -#define TEST_BLOCK_SIZE1034 1034 -#define TEST_BLOCK_SIZE1970 1970 -#define TEST_BLOCK_SIZE2058 2058 -#define TEST_BLOCK_SIZE8000 8000 -#define TEST_BLOCK_SIZE2048 2048 - -#define TEST_BLOCK_ADDR70 70 -#define TEST_BLOCK_ADDR100 100 - -#define TEST_ALIGN1024 1024 -#define TEST_ALIGN4096 4096 - -#define TEST_THRESHOLD10 10 -#define TEST_THRESHOLD3 3 - -#define CORE_INCREMENT 1024 -#define FAMILY_SIZE 1024 +#define FILENAME_LEN 1024 + +#define TBLOCK_SIZE1 1 +#define TBLOCK_SIZE2 2 +#define TBLOCK_SIZE3 3 +#define TBLOCK_SIZE4 4 +#define TBLOCK_SIZE5 5 +#define TBLOCK_SIZE6 6 +#define TBLOCK_SIZE7 7 +#define TBLOCK_SIZE8 8 +#define TBLOCK_SIZE10 10 +#define TBLOCK_SIZE11 11 +#define TBLOCK_SIZE20 20 +#define TBLOCK_SIZE30 30 +#define TBLOCK_SIZE36 36 +#define TBLOCK_SIZE40 40 +#define TBLOCK_SIZE50 50 +#define TBLOCK_SIZE80 80 +#define TBLOCK_SIZE90 90 +#define TBLOCK_SIZE98 98 +#define TBLOCK_SIZE100 100 +#define TBLOCK_SIZE150 150 +#define TBLOCK_SIZE200 200 +#define TBLOCK_SIZE600 600 +#define TBLOCK_SIZE700 700 +#define TBLOCK_SIZE1034 1034 +#define TBLOCK_SIZE1970 1970 +#define TBLOCK_SIZE2048 2048 +#define TBLOCK_SIZE2058 2058 +#define TBLOCK_SIZE2192 2192 +#define TBLOCK_SIZE3080 3080 +#define TBLOCK_SIZE3088 3088 +#define TBLOCK_SIZE3198 3198 +#define TBLOCK_SIZE3286 3286 +#define TBLOCK_SIZE3248 3248 +#define TBLOCK_SIZE3900 3900 +#define TBLOCK_SIZE4020 4020 +#define TBLOCK_SIZE4086 4086 +#define TBLOCK_SIZE4096 4096 +#define TBLOCK_SIZE4106 4106 +#define TBLOCK_SIZE5000 5000 +#define TBLOCK_SIZE6000 6000 +#define TBLOCK_SIZE8000 8000 +#define TBLOCK_SIZE8100 8100 +#define TBLOCK_SIZE8192 8192 +#define TBLOCK_SIZE8190 8190 +#define TBLOCK_SIZE12000 12000 + +#define TBLOCK_ADDR70 70 +#define TBLOCK_ADDR100 100 + +#define TEST_ALIGN16 16 +#define TEST_ALIGN1024 1024 +#define TEST_ALIGN4096 4096 + +#define TEST_THRESHOLD10 10 +#define TEST_THRESHOLD3 3 const char *FILENAME[] = { "mf", @@ -80,24 +103,17 @@ const char *FILENAME[] = { }; typedef enum { - TEST_NORMAL, /* size of aggregator is >= alignment size */ - TEST_AGGR_SMALL, /* size of aggregator is smaller than alignment size */ - TEST_NTESTS /* The number of test types, must be last */ + TEST_NORMAL, /* size of aggregator is >= alignment size */ + TEST_AGGR_SMALL, /* size of aggregator is smaller than alignment size */ + TEST_NTESTS /* The number of test types, must be last */ } test_type_t; -typedef struct frspace_state_t { - hsize_t tot_space; /* Total amount of space tracked */ - hsize_t tot_sect_count; /* Total # of sections tracked */ - hsize_t serial_sect_count; /* # of serializable sections tracked */ - hsize_t ghost_sect_count; /* # of un-serializable sections tracked */ -} frspace_state_t; +static int check_stats(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *state); - -static int check_stats(const H5F_t *, const H5FS_t *, frspace_state_t *); static unsigned test_mf_eoa(const char *env_h5_drvr, hid_t fapl); static unsigned test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl); static unsigned test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl); -static unsigned test_mf_tmp(const char *env_h5_drvr, hid_t fapl); +static unsigned test_dichotomy(hid_t fapl); static unsigned test_mf_fs_start(hid_t fapl); static unsigned test_mf_fs_alloc_free(hid_t fapl); static unsigned test_mf_fs_extend(hid_t fapl); @@ -119,21 +135,27 @@ static unsigned test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t static unsigned test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl); static unsigned test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl); static unsigned test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl); -static unsigned test_mf_fs_persist(hid_t fapl_new, hid_t fcpl); -static unsigned test_mf_fs_gone(hid_t fapl_new, hid_t fcpl); -static unsigned test_mf_fs_split(hid_t fapl_new, hid_t fcpl); -static unsigned test_mf_fs_multi(hid_t fapl, hid_t fcpl); -static unsigned test_mf_fs_drivers(hid_t fapl); +static unsigned test_mf_tmp(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_strat_thres_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_fs_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_strat_thres_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_fs_persist_split(void); +static unsigned test_mf_fs_persist_multi(void); +static unsigned test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_small(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_large(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_large_try_extend(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_small_try_extend(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_try_shrink(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_alignment(const char *env_h5_drvr, hid_t fapl); /* * Verify statistics for the free-space manager * - * Modifications: - * Vailin Choi; July 2012 - * To ensure "f" and "frsp" are valid pointers */ static int -check_stats(const H5F_t *f, const H5FS_t *frsp, frspace_state_t *state) +check_stats(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *state) { H5FS_stat_t frspace_stats; /* Statistics about the heap */ @@ -146,22 +168,22 @@ check_stats(const H5F_t *f, const H5FS_t *frsp, frspace_state_t *state) if(frspace_stats.tot_space != state->tot_space) { HDfprintf(stdout, "frspace_stats.tot_space = %Hu, state->tot_space = %Zu\n", - frspace_stats.tot_space, state->tot_space); + frspace_stats.tot_space, state->tot_space); TEST_ERROR } /* end if */ if(frspace_stats.tot_sect_count != state->tot_sect_count) { HDfprintf(stdout, "frspace_stats.tot_sect_count = %Hu, state->tot_sect_count = %Hu\n", - frspace_stats.tot_sect_count, state->tot_sect_count); + frspace_stats.tot_sect_count, state->tot_sect_count); TEST_ERROR } /* end if */ if(frspace_stats.serial_sect_count != state->serial_sect_count) { HDfprintf(stdout, "frspace_stats.serial_sect_count = %Hu, state->serial_sect_count = %Hu\n", - frspace_stats.serial_sect_count, state->serial_sect_count); + frspace_stats.serial_sect_count, state->serial_sect_count); TEST_ERROR } /* end if */ if(frspace_stats.ghost_sect_count != state->ghost_sect_count) { HDfprintf(stdout, "frspace_stats.ghost_sect_count = %Hu, state->ghost_sect_count = %Hu\n", - frspace_stats.ghost_sect_count, state->ghost_sect_count); + frspace_stats.ghost_sect_count, state->ghost_sect_count); TEST_ERROR } /* end if */ @@ -173,13 +195,15 @@ error: } /* check_stats() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from file allocation * * Set up: - * Turn off using meta/small data aggregator - * There is nothing in free-space manager + * Turn off using meta/small data aggregator + * There is nothing in free-space manager * * Allocate two blocks which should be from file allocation + *------------------------------------------------------------------------- */ static unsigned test_mf_eoa(const char *env_h5_drvr, hid_t fapl) @@ -234,7 +258,7 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* nothing should be changed in meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &ma_size); @@ -244,7 +268,7 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) if (addr1 < (haddr_t)file_size) TEST_ERROR - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* nothing should be changed in meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &ma_size); @@ -262,7 +286,7 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)) + if (new_file_size != (file_size+TBLOCK_SIZE30+TBLOCK_SIZE50)) TEST_ERROR /* Re-open the file */ @@ -273,8 +297,8 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -308,6 +332,7 @@ error: } /* test_mf_eoa() */ /* + *------------------------------------------------------------------------- * To verify that an allocated block from file allocation is shrunk. * * Set up: @@ -323,6 +348,7 @@ error: * Test 4: Allocate a block of 30 from file allocation * H5MF_try_shrink() the block by 20 from the end: succeed * + *------------------------------------------------------------------------- */ static unsigned test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) @@ -377,7 +403,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if (addr < (haddr_t)file_size) TEST_ERROR @@ -395,7 +421,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30)) + if (new_file_size != (file_size+TBLOCK_SIZE30)) TEST_ERROR /* Re-open the file */ @@ -409,7 +435,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* should succeed */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30) <= 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -453,13 +479,13 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if (addr < (haddr_t)file_size) TEST_ERROR /* should not succeed in shrinking */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30 - 10) > 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30 - 10) > 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -475,7 +501,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30)) + if(new_file_size != (file_size + TBLOCK_SIZE30)) TEST_ERROR PASSED() @@ -503,7 +529,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* should not succeed in shrinking */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30 + 10) > 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30 + 10) > 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -519,7 +545,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30)) + if(new_file_size != (file_size + TBLOCK_SIZE30)) TEST_ERROR PASSED() @@ -546,7 +572,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* should succeed in shrinking */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr+10, (hsize_t)(TEST_BLOCK_SIZE30 - 10)) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr+10, (hsize_t)(TBLOCK_SIZE30 - 10)) <= 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -588,6 +614,7 @@ error: } /* test_mf_eoa_shrink() */ /* + *------------------------------------------------------------------------- * To verify that an allocated block from file allocation is extended. * * Set up: @@ -599,6 +626,7 @@ error: * * Test 2: Allocate a block of 30 * H5MF_try_extend() the block of size 20 by 50: fail + *------------------------------------------------------------------------- */ static unsigned test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) @@ -656,7 +684,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if (addr < (haddr_t)file_size) TEST_ERROR @@ -673,7 +701,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30)) + if(new_file_size != (file_size + TBLOCK_SIZE30)) TEST_ERROR /* Re-open the file */ @@ -685,7 +713,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) FAIL_STACK_ERROR /* should succeed */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)TEST_BLOCK_SIZE50); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)TBLOCK_SIZE30, (hsize_t)TBLOCK_SIZE50); if(was_extended <= 0) TEST_ERROR @@ -703,7 +731,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50)) + if(new_file_size != (file_size + TBLOCK_SIZE30 + TBLOCK_SIZE50)) TEST_ERROR PASSED() @@ -734,7 +762,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if(addr < (haddr_t)file_size) TEST_ERROR @@ -744,7 +772,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) if(new_ma_addr != ma_addr) TEST_ERROR - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)(TEST_BLOCK_SIZE30-10), (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)(TBLOCK_SIZE30-10), (hsize_t)(TBLOCK_SIZE50)); /* should not succeed */ if(was_extended > 0) @@ -763,7 +791,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != file_size + TEST_BLOCK_SIZE30) + if(new_file_size != file_size + TBLOCK_SIZE30) TEST_ERROR if(H5Pclose(fapl_new) < 0) @@ -787,6 +815,7 @@ error: } /* test_mf_eoa_extend() */ /* + *------------------------------------------------------------------------- * To verify that temporary blocks are allocated correctly * * Set up: @@ -806,13 +835,19 @@ error: * space fails * - Check that allocating another 1/2 of the file as normal address * space fails + *------------------------------------------------------------------------- */ static unsigned -test_mf_tmp(const char *env_h5_drvr, hid_t fapl) +test_mf_tmp(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t file = -1; /* File ID */ + hid_t file = -1; /* File ID */ + hid_t fapl2 = -1; /* File access property list */ + hid_t fcpl = -1; /* File creation property list */ - TESTING("'temporary' file space allocation"); + if(new_format) + TESTING("'temporary' file space allocation with new library format") + else + TESTING("'temporary' file space allocation with old library format") /* Can't run this test with multi-file VFDs */ if(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi") && HDstrcmp(env_h5_drvr, "family")) { @@ -829,8 +864,22 @@ test_mf_tmp(const char *env_h5_drvr, hid_t fapl) /* Set the filename to use for this test */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + if(new_format) { + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR + + /* Set the "use the latest version of the format" bounds for creating objects in the file */ + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + } /* end if */ + /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, new_format?fapl2:fapl)) < 0) FAIL_STACK_ERROR /* Close file */ @@ -855,13 +904,13 @@ test_mf_tmp(const char *env_h5_drvr, hid_t fapl) FAIL_STACK_ERROR /* Allocate some temporary address space */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc_tmp(f, (hsize_t)TEST_BLOCK_SIZE30))) + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc_tmp(f, (hsize_t)TBLOCK_SIZE30))) FAIL_STACK_ERROR /* Check if temporary file address is valid */ if(!H5F_IS_TMP_ADDR(f, tmp_addr)) TEST_ERROR - if(tmp_addr < (haddr_t)(maxaddr - TEST_BLOCK_SIZE30)) + if(tmp_addr < (haddr_t)(maxaddr - TBLOCK_SIZE30)) TEST_ERROR /* Reading & writing with a temporary address value should fail */ @@ -878,7 +927,7 @@ test_mf_tmp(const char *env_h5_drvr, hid_t fapl) /* Freeing a temporary address value should fail */ H5E_BEGIN_TRY { - status = H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, tmp_addr, (hsize_t)TEST_BLOCK_SIZE30); + status = H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, tmp_addr, (hsize_t)TBLOCK_SIZE30); } H5E_END_TRY; if(status >= 0) TEST_ERROR @@ -963,10 +1012,12 @@ error: } /* test_mf_tmp() */ /* + *------------------------------------------------------------------------- * To verify that the free-space manager is created or opened * * Set up: * Turn off using meta/small data aggregator + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_start(hid_t fapl) @@ -976,11 +1027,10 @@ test_mf_fs_start(hid_t fapl) char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ h5_stat_size_t file_size, new_file_size; /* file size */ - H5FD_mem_t type; - frspace_state_t state; + H5FS_stat_t state; - TESTING("H5MF_alloc_create()/H5MF__alloc_open() of free-space manager"); + TESTING("H5MF_create_fstype()/H5MF_open_fstype() of free-space manager"); /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -1011,20 +1061,18 @@ test_mf_fs_start(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - /* Start up free-space manager */ - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR - - HDmemset(&state, 0, sizeof(frspace_state_t)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -1055,6 +1103,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that a block is allocated/freed from/to the free-space manager * * Set up: @@ -1078,10 +1127,7 @@ error: * The block is allocated from file allocation * Deallocate the block which will be returned to free-space manager * (the space is shrunk and freed since it is at end of file) - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_alloc_free(hid_t fapl) @@ -1091,12 +1137,10 @@ test_mf_fs_alloc_free(hid_t fapl) char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ h5_stat_size_t file_size, new_file_size; /* file size */ - H5FD_mem_t type; H5MF_free_section_t *sect_node = NULL; - haddr_t addr; - frspace_state_t state; - H5MF_sect_ud_t udata; - H5FS_section_info_t *node; + haddr_t addr; + haddr_t tmp; + H5FS_stat_t state; TESTING("H5MF_alloc()/H5MF_xfree() of free-space manager:test 1"); @@ -1129,69 +1173,59 @@ test_mf_fs_alloc_free(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space */ - if (addr != TEST_BLOCK_ADDR70) + if(addr != TBLOCK_ADDR70) TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); - state.tot_space += TEST_BLOCK_SIZE30; + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove section A from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE30, (H5FS_section_info_t **)&node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != TBLOCK_ADDR70) + TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1216,67 +1250,57 @@ test_mf_fs_alloc_free(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); - /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add section A to free-space manager */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 20 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE20)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE20)); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) + if(addr != TBLOCK_ADDR70) TEST_ERROR /* should still have 1 section of size 10 left in free-space manager */ - state.tot_space -= (TEST_BLOCK_SIZE20); + state.tot_space -= (TBLOCK_SIZE20); - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space manager */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE20)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE20)); /* Still 1 section in free-space because of merging */ - state.tot_space += TEST_BLOCK_SIZE20; - if(check_stats(f, f->shared->fs_man[type], &state)) + state.tot_space += TBLOCK_SIZE20; + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR - /* Remove section A from free-space manager */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE30, (H5FS_section_info_t **)&node) < 0) - FAIL_STACK_ERROR + /* Remove section A from free-space */ + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != TBLOCK_ADDR70) + TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1301,36 +1325,28 @@ test_mf_fs_alloc_free(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* @@ -1338,38 +1354,37 @@ test_mf_fs_alloc_free(hid_t fapl) * Since free-space manager cannot fulfull the request, * the block is obtained from file allocation */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE40)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE40)); /* Verify that the allocated block is not section A in free-space */ - if (addr == TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr == TBLOCK_ADDR70) + TEST_ERROR /* free-space info should be the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove section A from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE30, (H5FS_section_info_t **)&node) < 0) - FAIL_STACK_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + /* Verify that the block is section A in free-space */ + if(tmp != TBLOCK_ADDR70) + TEST_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block of size 40 to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE40)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE40)); /* * Free-space info is the same. * The block is returned to free-space. * It is shrunk and freed because it is at end of file. */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -1400,6 +1415,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that a block allocated from the free-space manager can be extended * * Set up: @@ -1435,10 +1451,7 @@ error: * Try to extend the allocated block by 50 from the free-space_manager: * Fail: section A does not adjoin section B (70+20 != address of section B) even though * the requested-size (50) equal to size of section B (50) - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_extend(hid_t fapl) @@ -1448,13 +1461,11 @@ test_mf_fs_extend(hid_t fapl) char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ h5_stat_size_t file_size, new_file_size; /* file size */ - H5FD_mem_t type; H5MF_free_section_t *sect_node1 = NULL, *sect_node2=NULL; - haddr_t addr; - frspace_state_t state; /* State of free space*/ - H5MF_sect_ud_t udata; - H5FS_section_info_t *node; - htri_t was_extended; + haddr_t addr; + haddr_t tmp; + H5FS_stat_t state; /* State of free space*/ + htri_t was_extended; TESTING("H5MF_try_extend() of free-space manager:test 1"); @@ -1487,101 +1498,91 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)TEST_BLOCK_SIZE50); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30, (hsize_t)TBLOCK_SIZE50); /* should succeed */ if(was_extended <= 0) TEST_ERROR /* Section B is removed from free-space manager */ - state.tot_space -= TEST_BLOCK_SIZE50; + state.tot_space -= TBLOCK_SIZE50; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the extended block to free-space manager */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50)); /* Verify that the extended block is back into free-space */ - state.tot_space += TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50; + state.tot_space += (TBLOCK_SIZE30+TBLOCK_SIZE50); state.tot_sect_count = 1; state.serial_sect_count = 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove the extended block */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR + if(tmp != TBLOCK_ADDR70) + TEST_ERROR + if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1605,95 +1606,84 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); - /* Add section A to free-space */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add section A to free-space manager */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)(TEST_BLOCK_SIZE50+10)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30, (hsize_t)(TBLOCK_SIZE50+10)); /* Should not be able to extend the allocated block */ if(was_extended) TEST_ERROR /* free-space info should remain the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the allocated block A to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); /* the returned section A is merged with section B in free-space */ /* rest of the info remains the same */ - state.tot_space += TEST_BLOCK_SIZE30; + state.tot_space += TBLOCK_SIZE30; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove the merged sections A & B from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != addr) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1718,95 +1708,84 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; - - /* Add section A to free-space */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add section A to free-space manager */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)(TEST_BLOCK_SIZE40)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30, (hsize_t)(TBLOCK_SIZE40)); /* Should succeed in extending the allocated block */ if(was_extended <=0) TEST_ERROR /* Should have 1 section of size=10 left in free-space manager */ - state.tot_space -= (TEST_BLOCK_SIZE40); - if(check_stats(f, f->shared->fs_man[type], &state)) + state.tot_space -= (TBLOCK_SIZE40); + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the extended block */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE40)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE40)); /* rest info is same, the extended section returned is merged with the section in free-space */ - state.tot_space += (TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE40); + state.tot_space += (TBLOCK_SIZE30+TBLOCK_SIZE40); - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove the merged sections A & B from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR + + if(tmp != addr) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1831,104 +1810,88 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)(TEST_BLOCK_SIZE30-10)); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)(TBLOCK_SIZE30-10)); /* Add section A of size=20 to free-space */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += (TEST_BLOCK_SIZE30-10); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += (TBLOCK_SIZE30-10); state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of size=20 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE30-10)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30-10)); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= (TEST_BLOCK_SIZE30-10); + state.tot_space -= (TBLOCK_SIZE30-10); state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)(TEST_BLOCK_SIZE30-10), (hsize_t)TEST_BLOCK_SIZE50); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)(TBLOCK_SIZE30-10), (hsize_t)TBLOCK_SIZE50); /* Should not succeed in extending the allocated block */ if(was_extended) TEST_ERROR /* Free-space info should be the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the allocated block */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE30-10)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE30-10)); - state.tot_space += (TEST_BLOCK_SIZE30-10); + state.tot_space += (TBLOCK_SIZE30-10); state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove section A from free-space manger */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30-10), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30-10), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != addr) TEST_ERROR /* Remove section B from free-space manager */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE50, (H5FS_section_info_t **)&node) < 0) - TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1957,6 +1920,7 @@ error: } /* test_mf_fs_extend() */ /* + *------------------------------------------------------------------------- * To verify that an aggregator is absorbed into a section. * * Test 1: To aborb the aggregator onto the beginning of the section @@ -1979,25 +1943,20 @@ error: * which will absorb meta_aggr to the section: * section size + remaining size of aggregator is > aggr->alloc_size, * section is allowed to absorb an aggregator (allow_sect_absorb is true) - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) { - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type, stype; - haddr_t addr, saddr; - haddr_t ma_addr=HADDR_UNDEF; - hsize_t ma_size=0; + hid_t file = -1; /* File ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr, saddr; + haddr_t tmp; + haddr_t ma_addr=HADDR_UNDEF; + hsize_t ma_size=0; H5MF_free_section_t *sect_node=NULL; - H5MF_sect_ud_t udata; - H5FS_section_info_t *node; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("A free-space section absorbs an aggregator: test 1"); @@ -2023,53 +1982,39 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR /* Allocate a section from meta_aggr */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - /* Add a section to free-space that adjoins end of the aggregator */ - sect_node = H5MF_sect_simple_new((haddr_t)(ma_addr+ma_size), (hsize_t)TEST_BLOCK_SIZE2048); + /* Create a section */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)(ma_addr+ma_size), (hsize_t)TBLOCK_SIZE2048); - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; - - /* When adding, meta_aggr is absorbed onto the beginning of the section */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add a section to free-space that adjoins end of the aggregator */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR /* Verify that the section did absorb the aggregator */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE2048, (H5FS_section_info_t **)&node) < 0) + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(ma_addr+ma_size), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) TEST_ERROR - if (node->addr != ma_addr) TEST_ERROR - if (node->size != (ma_size + TEST_BLOCK_SIZE2048)) TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != ma_addr) TEST_ERROR /* Restore info for aggregator */ f->shared->meta_aggr.addr = ma_addr; f->shared->meta_aggr.size = ma_size; /* Remove section from meta_aggr */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2093,48 +2038,34 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR /* Allocate a section from meta_aggr */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* Allocate a section from sdata_aggr */ - stype = H5FD_MEM_DRAW; - saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Add a section to free-space that adjoins the beginning of meta_aggr */ - sect_node = H5MF_sect_simple_new((haddr_t)addr, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)addr, (hsize_t)TBLOCK_SIZE30); /* When adding, meta_aggr is absorbed onto the end of the section */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR /* Verify that the section did absorb the aggregator */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(ma_size+TEST_BLOCK_SIZE30), (H5FS_section_info_t **)&node) < 0) + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(ma_size+TBLOCK_SIZE30), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) TEST_ERROR - if ((node->addr + TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR - if (node->size != (ma_size + TEST_BLOCK_SIZE30)) TEST_ERROR - - /* free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) + if((tmp + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* restore info to meta_aggr */ @@ -2142,9 +2073,9 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) f->shared->meta_aggr.size = ma_size; /* Remove section from meta_aggr */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); /* Remove section from sdata_aggr */ - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2166,6 +2097,7 @@ error: } /* test_mf_fs_absorb() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2180,6 +2112,7 @@ error: * Result: * The second block of 50 is allocated from meta_aggr * There is space of 1968 left in meta_aggr + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) @@ -2187,6 +2120,7 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) hid_t file = -1; /* File ID */ char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ + hid_t fcpl; /* File creation property list */ h5_stat_size_t file_size, new_file_size; /* file size */ H5FD_mem_t type; haddr_t addr1, addr2; @@ -2202,13 +2136,21 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set to H5F_FSPACE_STRATEGY_AGGR strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_AGGR, FALSE, (hsize_t)1) < 0) + TEST_ERROR + /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Close file */ if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get the size of a file */ if((file_size = h5_get_file_size(filename, fapl)) < 0) @@ -2216,54 +2158,58 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) /* Re-open the file */ if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get a pointer to the internal file object */ if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR + TEST_ERROR /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + if((addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30)) == HADDR_UNDEF) + TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + if((addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50)) == HADDR_UNDEF) + TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get the size of the file */ if((new_file_size = h5_get_file_size(filename, fapl)) < 0) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)) + if(new_file_size != (file_size+TBLOCK_SIZE30+TBLOCK_SIZE50)) TEST_ERROR /* Re-open the file */ if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get a pointer to the internal file object */ if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR + TEST_ERROR - /* Free the two blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); + /* Free the two blocks: order matters because of H5F_FSPACE_STRATEGY_AGGR strategy */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50) < 0) + TEST_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30) < 0) + TEST_ERROR if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get the size of the file */ if((new_file_size = h5_get_file_size(filename, fapl)) < 0) @@ -2273,6 +2219,9 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) if (new_file_size != file_size) TEST_ERROR + if(H5Pclose(fcpl) < 0) + TEST_ERROR + PASSED() } /* end if */ else { @@ -2285,11 +2234,13 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) error: H5E_BEGIN_TRY { H5Fclose(file); + H5Pclose(fcpl); } H5E_END_TRY; return(1); } /* test_mf_aggr_alloc1() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2311,6 +2262,7 @@ error: * A block of request-size is extended via file allocation and is merged with meta_aggr * The block of 2058 is allocated out of meta_aggr * There is space of 1968 left in meta_aggr + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) @@ -2354,29 +2306,29 @@ test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) FAIL_STACK_ERROR type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if (ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3+TEST_BLOCK_SIZE2058) != ma_addr) + if((addr3+TBLOCK_SIZE2058) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR if(H5Fclose(file) < 0) @@ -2388,7 +2340,7 @@ test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) /* Verify the file is the correct size */ /* Unused space is freed from the end of the file */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50+TEST_BLOCK_SIZE2058)) + if(new_file_size != (file_size+TBLOCK_SIZE30+TBLOCK_SIZE50+TBLOCK_SIZE2058)) TEST_ERROR /* Re-open the file */ @@ -2399,7 +2351,7 @@ test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50+TEST_BLOCK_SIZE2058); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30+TBLOCK_SIZE50+TBLOCK_SIZE2058); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2429,6 +2381,7 @@ error: } /* test_mf_aggr_alloc2() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr : (nothing in the aggregator) @@ -2463,6 +2416,7 @@ error: * Result: * The fourth block of 50 is allocated from what is left in meta_aggr * There is space of 1968 left in meta_aggr + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc3(const char *env_h5_drvr, hid_t fapl) @@ -2509,55 +2463,55 @@ test_mf_aggr_alloc3(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR - if (sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) TEST_ERROR + if(sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate third block, which is from file allocation not from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE2058)); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE2058)); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); /* info for meta_aggr shouldn't be changed */ - if (addr3 != (sdata_addr+sdata_size)) TEST_ERROR - if ((addr3+TEST_BLOCK_SIZE2058) == new_ma_addr) TEST_ERROR - if ((new_ma_addr != ma_addr) || (new_ma_size != ma_size)) TEST_ERROR + if(addr3 != (sdata_addr+sdata_size)) TEST_ERROR + if((addr3+TBLOCK_SIZE2058) == new_ma_addr) TEST_ERROR + if((new_ma_addr != ma_addr) || (new_ma_size != ma_size)) TEST_ERROR /* Allocate fourth block, which should be from meta_aggr */ - addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr4+TEST_BLOCK_SIZE50) != ma_addr) + if((addr4+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50 + TBLOCK_SIZE50))) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE2058); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE2058); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2567,7 +2521,7 @@ test_mf_aggr_alloc3(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -2588,6 +2542,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2626,6 +2581,7 @@ error: * The new block's address is returned * The block does not adjoin meta_aggr * meta_aggr's info is unchanged + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc4(const char *env_h5_drvr, hid_t fapl) @@ -2670,59 +2626,59 @@ test_mf_aggr_alloc4(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) + if(ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* Allocate second block from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE2048 - TBLOCK_SIZE30)); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (saddr2+(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30) != sdata_addr) + if(saddr2+(TBLOCK_SIZE2048 - TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* Allocate third block from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr3+TEST_BLOCK_SIZE50) != sdata_addr) + if((saddr3+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR - if(sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE50)) + if(sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE50)) TEST_ERROR /* Allocate second block of 2058, which is from file allocation, not from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); - if (addr2 != sdata_addr) + if(addr2 != sdata_addr) TEST_ERROR /* sdata_aggr is reset 0 */ H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((sdata_addr != 0) && (sdata_size != 0)) + if((sdata_addr != 0) && (sdata_size != 0)) TEST_ERROR /* info is unchanged in meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if ((new_ma_addr != ma_addr) && (new_ma_size != ma_size)) + if((new_ma_addr != ma_addr) && (new_ma_size != ma_size)) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE2058); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE2058); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)TBLOCK_SIZE2048 - TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2732,7 +2688,7 @@ test_mf_aggr_alloc4(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -2752,6 +2708,7 @@ error: } /* test_mf_aggr_alloc4() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2772,6 +2729,7 @@ error: * The block of 1970 is allocated from there * There is space of 2046 left in meta_aggr * + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc5(const char *env_h5_drvr, hid_t fapl) @@ -2816,34 +2774,34 @@ test_mf_aggr_alloc5(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (addr2+TEST_BLOCK_SIZE50 != ma_addr) + if(addr2+TBLOCK_SIZE50 != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30+TBLOCK_SIZE50))) TEST_ERROR /* Allocate third block from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (addr3 != ma_addr) TEST_ERROR - if ((addr3+TEST_BLOCK_SIZE1970) != new_ma_addr) TEST_ERROR - if (new_ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE1970 - ma_size))) + if(addr3 != ma_addr) TEST_ERROR + if((addr3+TBLOCK_SIZE1970) != new_ma_addr) TEST_ERROR + if(new_ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE1970 - ma_size))) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2853,7 +2811,7 @@ test_mf_aggr_alloc5(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -2873,6 +2831,7 @@ error: } /* test_mf_aggr_alloc5() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2903,6 +2862,7 @@ error: * sdata_aggr is untouched * meta_aggr's unused space of [880, 1968] is freed to free-space * meta_aggr is updated to point to the new block + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) @@ -2915,7 +2875,7 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2, addr3, saddr1; haddr_t ma_addr=HADDR_UNDEF, new_ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, new_ma_size=0, sdata_size=0; - frspace_state_t state; + H5FS_stat_t state; hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 6"); @@ -2949,44 +2909,44 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) type = H5FD_MEM_SUPER; /* Allocate first block from meta_aggr */ - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (addr2+TEST_BLOCK_SIZE50 != ma_addr) + if(addr2+TBLOCK_SIZE50 != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) TEST_ERROR - if (sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) TEST_ERROR + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR + if(sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate third block from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if ((addr3+TEST_BLOCK_SIZE1970) != new_ma_addr) TEST_ERROR - if (addr3 != (sdata_addr+sdata_size)) TEST_ERROR + if((addr3+TBLOCK_SIZE1970) != new_ma_addr) TEST_ERROR + if(addr3 != (sdata_addr+sdata_size)) TEST_ERROR - if ((ma_addr+TEST_BLOCK_SIZE1970) == new_ma_addr) TEST_ERROR - if (new_ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE1970)) + if((ma_addr+TBLOCK_SIZE1970) == new_ma_addr) TEST_ERROR + if(new_ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE1970)) TEST_ERROR /* Verify that meta_aggr's unused space of 1968 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += (TBLOCK_SIZE2048 - (TBLOCK_SIZE30+TBLOCK_SIZE50)); state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -2994,10 +2954,10 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3007,7 +2967,7 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -3027,13 +2987,14 @@ error: } /* test_mf_aggr_alloc6() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * - * Allocate first block (30) from meta_aggr: (nothing in meta_aggr) - * request-size is > what is left in meta_aggr and < meta_aggr->alloc_size - * Result: - * A block of meta_aggr->alloc_size is obtained via file allocation - * The first block of 30 is allocated from there + * Allocate first block (30) from meta_aggr: (nothing in meta_aggr) + * request-size is > what is left in meta_aggr and < meta_aggr->alloc_size + * Result: + * A block of meta_aggr->alloc_size is obtained via file allocation + * The first block of 30 is allocated from there * * Allocate second block (50) from meta_aggr: * request-size is <= what is left in meta_aggr @@ -3068,6 +3029,7 @@ error: * The block does not adjoin meta_aggr * meta_aggr's unused space of [880, 1968] is freed to free-space * meta_aggr is updated to point to the new block + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) @@ -3080,7 +3042,7 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2, addr3, saddr1, saddr2, saddr3; haddr_t ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, sdata_size=0; - frspace_state_t state; + H5FS_stat_t state; hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 7"); @@ -3113,51 +3075,51 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) /* Allocate the first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate the second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (addr2+TEST_BLOCK_SIZE50 != ma_addr) + if (addr2+TBLOCK_SIZE50 != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if (ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR /* Allocate the first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) + if ((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* Allocate the second block from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2048 - TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr2+(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) != sdata_addr) + if ((saddr2+(TBLOCK_SIZE2048 - TBLOCK_SIZE30)) != sdata_addr) TEST_ERROR if (sdata_size != 0) TEST_ERROR /* Allocate the third block from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr3+TEST_BLOCK_SIZE50) != sdata_addr) + if ((saddr3+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR - if (sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE50)) + if (sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE50)) TEST_ERROR /* Allocate the third block from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); if (addr3 != sdata_addr) TEST_ERROR - if ((addr3 + TEST_BLOCK_SIZE1970) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE1970)) TEST_ERROR + if ((addr3 + TBLOCK_SIZE1970) != ma_addr) TEST_ERROR + if (ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE1970)) TEST_ERROR /* sdata_aggr info is reset to 0 */ H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); @@ -3165,8 +3127,8 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) if (sdata_size != 0) TEST_ERROR /* Verify that meta_aggr's unused space of 1968 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50)); state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -3174,12 +3136,12 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)(TBLOCK_SIZE2048 - TBLOCK_SIZE30)); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3209,6 +3171,7 @@ error: } /* test_mf_aggr_alloc7() */ /* + *------------------------------------------------------------------------- * To verify that a block can be extended from the aggregator * * Test 1: Allocate block A from meta_aggr which is at end of file @@ -3229,6 +3192,7 @@ error: * Allocate block B from sdata_aggr so that meta_aggr is not at end of file * Try to extend a block which adjoins meta_aggr but meta_aggr cannot fulfill the extended-request * H5MF_try_extend() fails + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) @@ -3274,9 +3238,9 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) /* Allocate the first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr+TEST_BLOCK_SIZE30) != ma_addr) + if((addr+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Adjust meta_aggr's info info for testing */ @@ -3286,7 +3250,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) new_addr = addr - 10; /* Try to extend the block by an amount < (% * aggr->alloc_size) */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE50)); /* should succeed */ if(!was_extended) @@ -3294,15 +3258,15 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr+TEST_BLOCK_SIZE50)) + if(new_ma_addr != (addr+TBLOCK_SIZE50)) TEST_ERROR - if (new_ma_size != (f->shared->meta_aggr.alloc_size - TEST_BLOCK_SIZE50)) TEST_ERROR + if(new_ma_size != (f->shared->meta_aggr.alloc_size - TBLOCK_SIZE50)) TEST_ERROR /* Free the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE50); /* Try to extend the block by an amount > (% * aggr->alloc_size) but amount < aggr->alloc_size */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE700)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE700)); /* should succeed */ if(!was_extended) @@ -3310,15 +3274,15 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr + TEST_BLOCK_SIZE700)) + if(new_ma_addr != (addr + TBLOCK_SIZE700)) TEST_ERROR - if (new_ma_size != (f->shared->meta_aggr.alloc_size * 2 - TEST_BLOCK_SIZE700)) TEST_ERROR + if(new_ma_size != (f->shared->meta_aggr.alloc_size * 2 - TBLOCK_SIZE700)) TEST_ERROR /* Free the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE700); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE700); /* Try to extend the block by an amount > (% * aggr->alloc_size) but amount > aggr->alloc_size */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE2058)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE2058)); /* should succeed */ if(!was_extended) @@ -3326,12 +3290,12 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr + TEST_BLOCK_SIZE2058)) + if (new_ma_addr != (addr + TBLOCK_SIZE2058)) TEST_ERROR if (new_ma_size != f->shared->meta_aggr.size) TEST_ERROR /* Free the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE2058); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE2058); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3365,16 +3329,16 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) /* Allocate the first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr+TEST_BLOCK_SIZE30) != ma_addr) + if((addr+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate the first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr+TEST_BLOCK_SIZE50) != sdata_addr) + if((saddr+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR /* Adjust meta_aggr's info info for testing */ @@ -3384,24 +3348,24 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) new_addr = addr - 10; /* should be able to fulfill request from the aggreqator itself */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE50)); if(!was_extended) TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr+TEST_BLOCK_SIZE50)) + if(new_ma_addr != (addr+TBLOCK_SIZE50)) TEST_ERROR - if (new_ma_size != (f->shared->meta_aggr.alloc_size-TEST_BLOCK_SIZE50)) + if(new_ma_size != (f->shared->meta_aggr.alloc_size-TBLOCK_SIZE50)) TEST_ERROR /* Restore info for meta_aggr */ f->shared->meta_aggr.addr = ma_addr; f->shared->meta_aggr.size = ma_size; - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3411,7 +3375,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3435,16 +3399,16 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr+TEST_BLOCK_SIZE50) != sdata_addr) + if((saddr+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR /* Adjust meta_aggr's info info for testing */ @@ -3454,7 +3418,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) new_addr = addr - 10; /* unable to fulfill request from the aggreqator itself */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE50)); if(was_extended) TEST_ERROR @@ -3468,8 +3432,8 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) f->shared->meta_aggr.addr = ma_addr; f->shared->meta_aggr.size = ma_size; - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3479,7 +3443,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3499,6 +3463,7 @@ error: } /* test_mf_aggr_extend() */ /* + *------------------------------------------------------------------------- * To verify that a block is absorbed into an aggregator * * MF_try_shrink() only allows blocks to be absorbed into an aggregator @@ -3517,10 +3482,7 @@ error: * H5MF_alloc() block C from meta_aggr * H5MF_try_shrink() block B should fail since it does not adjoin the * beginning nor the end of meta_aggr - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) @@ -3567,20 +3529,20 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) /* Allocate block A from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - ma_addr = new_ma_addr - TEST_BLOCK_SIZE30; + ma_addr = new_ma_addr - TBLOCK_SIZE30; - if((addr1 + TEST_BLOCK_SIZE30) != new_ma_addr) + if((addr1 + TBLOCK_SIZE30) != new_ma_addr) TEST_ERROR /* should succeed */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30) <= 0) TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != ma_addr) TEST_ERROR + if(new_ma_addr != ma_addr) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3590,7 +3552,7 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3614,32 +3576,32 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) /* Allocate block A from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) TEST_ERROR + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR + if(ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate block B from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), NULL, &sdata_size); /* should succeed */ - if(H5MF_try_shrink(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE50) <= 0) + if(H5MF_try_shrink(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE50) <= 0) TEST_ERROR H5MF_aggr_query(f, &(f->shared->sdata_aggr), &new_sdata_addr, &new_sdata_size); - if (new_sdata_addr != saddr1) TEST_ERROR - if (new_sdata_size != sdata_size + TEST_BLOCK_SIZE50) TEST_ERROR + if(new_sdata_addr != saddr1) TEST_ERROR + if(new_sdata_size != sdata_size + TBLOCK_SIZE50) TEST_ERROR /* meta_aggr info should be updated because the block is absorbed into the meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); if (new_ma_addr != ma_addr) TEST_ERROR if (new_ma_size != (ma_size)) TEST_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3649,7 +3611,7 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3673,35 +3635,35 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) /* Allocate block A from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate block B from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* Allocate block C from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50)); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50) != ma_addr) + if((addr3+TBLOCK_SIZE30+TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* should not succeed */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50) > 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50) > 0) TEST_ERROR /* aggregator info should be the same as before */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != ma_addr) TEST_ERROR + if(new_ma_addr != ma_addr) TEST_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50)); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3711,7 +3673,7 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3731,6 +3693,7 @@ error: } /* test_mf_aggr_absorb() */ /* + *------------------------------------------------------------------------- * To verify that a block allocated from file allocation is aligned, can be shrunk and extended * * Alignment = 1024 or 4096 @@ -3759,6 +3722,7 @@ error: * Allocate a block which should be from file allocation * The return address should be aligned * H5MF_try_extend() the block with aligned address should succeed + *------------------------------------------------------------------------- */ static unsigned test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -3773,7 +3737,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t ma_addr=HADDR_UNDEF; hsize_t ma_size=0; htri_t was_extended; - frspace_state_t state; + H5FS_stat_t state; hsize_t alignment=0, mis_align=0, tmp=0, accum=0; hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ @@ -3819,23 +3783,23 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; - accum = mis_align + TEST_BLOCK_SIZE30; + accum = mis_align + TBLOCK_SIZE30; /* Allocate a block of 30 from file allocation */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* there should be nothing in the aggregator */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (ma_addr || ma_size) TEST_ERROR + if(ma_addr || ma_size) TEST_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); if (mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; @@ -3848,18 +3812,18 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = 0; if ((tmp = ((hsize_t)file_size + accum) % alignment)) mis_align = alignment - tmp; - accum += (mis_align + TEST_BLOCK_SIZE50); + accum += (mis_align + TBLOCK_SIZE50); - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ if (addr2 % alignment) TEST_ERROR /* there should be nothing in the aggregator */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (ma_addr || ma_size) TEST_ERROR + if(ma_addr || ma_size) TEST_ERROR - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -3867,8 +3831,8 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR } - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3902,7 +3866,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* allocate a block of 50 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* address should be aligned */ if (addr1 % alignment) TEST_ERROR @@ -3923,7 +3887,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* shrink the block */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE50) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE50) <= 0) TEST_ERROR if(H5Fclose(file) < 0) @@ -3932,7 +3896,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if((new_file_size = h5_get_file_size(filename, fapl1)) < 0) TEST_ERROR - if (new_file_size != (file_size-TEST_BLOCK_SIZE50)) TEST_ERROR + if (new_file_size != (file_size-TBLOCK_SIZE50)) TEST_ERROR PASSED() } /* end if */ @@ -3957,10 +3921,10 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* allocate a block of 50 */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* address should be aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* Close file */ if(H5Fclose(file) < 0) @@ -3978,9 +3942,9 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* try to extend the block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr1, (hsize_t)TEST_BLOCK_SIZE50, (hsize_t)TEST_BLOCK_SIZE30); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE50, (hsize_t)TBLOCK_SIZE30); - if (was_extended <=0) TEST_ERROR + if(was_extended <=0) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3988,7 +3952,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if((new_file_size = h5_get_file_size(filename, fapl1)) < 0) TEST_ERROR - if (new_file_size != (file_size+TEST_BLOCK_SIZE30)) TEST_ERROR + if (new_file_size != (file_size+TBLOCK_SIZE30)) TEST_ERROR PASSED() } /* end if */ @@ -4007,6 +3971,7 @@ error: } /* test_mf_align_eoa() */ /* + *------------------------------------------------------------------------- * To verify that a block allocated from the free-space manager is aligned * * Alignment = 1024 or 4096 @@ -4035,10 +4000,7 @@ error: * Allocate a block of size=40 * The free-space manager is unable to fulfill the request * The block is allocated from file allocation and should be aligned - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -4047,12 +4009,10 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) char filename[FILENAME_LEN]; /* Filename to use */ h5_stat_size_t file_size; H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; H5MF_free_section_t *sect_node = NULL; haddr_t addr; - frspace_state_t state; - H5MF_sect_ud_t udata; - htri_t was_extended; + H5FS_stat_t state; + htri_t was_extended; hsize_t alignment=0, tmp=0, mis_align=0; hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ @@ -4081,58 +4041,51 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - sect_node = H5MF_sect_simple_new((haddr_t)alignment, (hsize_t)TEST_BLOCK_SIZE50); + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) + TEST_ERROR - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + /* Create section A */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)alignment, (hsize_t)TBLOCK_SIZE50); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE50; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 50 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is section A in free-space */ - if (addr != (haddr_t)alignment) TEST_ERROR - if (addr % alignment) TEST_ERROR + if(addr != (haddr_t)alignment) TEST_ERROR + if(addr % alignment) TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE50; + state.tot_space -= TBLOCK_SIZE50; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE50); - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -4151,70 +4104,63 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR - - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE8000); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + /* Create section A */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE8000); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE8000; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE8000; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 600 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE600); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE600); /* Verify that the allocated block is aligned */ if (addr % alignment) TEST_ERROR /* should have 1 more section in free-space */ - state.tot_space -= TEST_BLOCK_SIZE600; + state.tot_space -= TBLOCK_SIZE600; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* try to extend the block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)TEST_BLOCK_SIZE600, (hsize_t)TEST_BLOCK_SIZE200); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)addr, (hsize_t)TBLOCK_SIZE600, (hsize_t)TBLOCK_SIZE200); - if (was_extended <=0) TEST_ERROR + if(was_extended <=0) TEST_ERROR /* space should be decreased by 200, # of sections remain the same */ - state.tot_space -= TEST_BLOCK_SIZE200; + state.tot_space -= TBLOCK_SIZE200; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space manager */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE600+TEST_BLOCK_SIZE200)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE600+TBLOCK_SIZE200)); /* only 1 section in free-space because of merging */ - state.tot_space += (TEST_BLOCK_SIZE600+TEST_BLOCK_SIZE200); + state.tot_space += (TBLOCK_SIZE600+TBLOCK_SIZE200); state.tot_sect_count = 1; state.serial_sect_count = 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -4241,45 +4187,38 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE700); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + /* Create section A */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE700); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE700; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE700; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* * Allocate a block of 40 * Since free-space manager cannot fulfull the request because of alignment, * the block is obtained from file allocation */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE40)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE40)); /* Verify that the allocated block is aligned */ - if (addr % alignment) + if(addr % alignment) TEST_ERROR /* verify that the allocated block is from file allocation, not section A in free-space */ @@ -4296,7 +4235,7 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } /* free-space info should be the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -4319,6 +4258,7 @@ error: } /* test_mf_align_fs() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -4413,6 +4353,7 @@ error: * Fragment from alignment of aggregator allocation is freed to free-space:[12368, 4016] * There is space of 2018 left in meta_aggr * EOA is at 20372 + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -4424,7 +4365,7 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5FD_mem_t type; haddr_t addr1, addr2, addr3, addr4; - frspace_state_t state; + H5FS_stat_t state; haddr_t ma_addr=HADDR_UNDEF; hsize_t ma_size=0, mis_align=0; hsize_t alignment=0, tmp=0; @@ -4467,19 +4408,19 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4487,22 +4428,22 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 50 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 50 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4510,7 +4451,7 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE50) != ma_addr) + if((addr2 + TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 80 */ @@ -4518,13 +4459,13 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if ((tmp = ma_addr % alignment)) mis_align = alignment - tmp; - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ - if (addr3 % alignment) TEST_ERROR + if(addr3 % alignment) TEST_ERROR /* fragment for alignment of block 80 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4532,22 +4473,22 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3 + TEST_BLOCK_SIZE80) != ma_addr) + if((addr3 + TBLOCK_SIZE80) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 1970 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 1970 from meta_aggr */ - addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); /* Verify that the allocated block is aligned */ - if (addr4 % alignment) TEST_ERROR + if(addr4 % alignment) TEST_ERROR /* fragment for alignment of block 1970 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4555,17 +4496,17 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr4 + TEST_BLOCK_SIZE1970) != ma_addr) + if((addr4 + TBLOCK_SIZE1970) != ma_addr) TEST_ERROR /* Verify total size of free space after all the allocations */ if(check_stats(f, f->shared->fs_man[type], &state)) TEST_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE80); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE80); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -4587,6 +4528,7 @@ error: } /* test_mf_align_alloc1() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -4669,10 +4611,7 @@ error: * The third block of 80 is allocated from the aggregator and should be aligned * There is space of 1968 left in meta_aggr * EOA is at 18432 - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -4683,7 +4622,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) h5_stat_size_t file_size; /* File size */ H5FD_mem_t type, stype; haddr_t addr1, addr2, addr3, saddr1; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, sdata_size=0, mis_align=0; hsize_t alignment=0, tmp=0; @@ -4725,19 +4664,19 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -4745,21 +4684,21 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) + if((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -4767,7 +4706,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE50) != ma_addr) + if((addr2 + TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* @@ -4782,17 +4721,17 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) * block 30 is allocated from ma_addr */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) mis_align = alignment - tmp; else if ((alignment == TEST_ALIGN4096) && (tmp = (ma_addr % alignment))) mis_align = alignment - tmp; /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* fragment for alignment of block 30 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; @@ -4804,7 +4743,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (sdata_addr != (saddr1 + TEST_BLOCK_SIZE30)) TEST_ERROR + if(sdata_addr != (saddr1 + TBLOCK_SIZE30)) TEST_ERROR /* * Calculate fragment for the allocation of block 80 from meta_aggr: @@ -4816,19 +4755,19 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) * block 30 is allocated from ma_addr */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) mis_align = alignment - tmp; else if ((alignment == TEST_ALIGN4096) && (tmp = ((sdata_addr + sdata_size) % alignment))) mis_align = alignment - tmp; /* Allocate a block of 80 from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ if (addr3 % alignment) TEST_ERROR /* fragment for alignment of block 80 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -4836,7 +4775,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3 + TEST_BLOCK_SIZE80) != ma_addr) + if((addr3 + TBLOCK_SIZE80) != ma_addr) TEST_ERROR /* Verify total size of free space after all the allocations */ @@ -4850,10 +4789,10 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR } - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE80); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE80); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -4875,6 +4814,7 @@ error: } /* test_mf_align_alloc2() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -5008,11 +4948,7 @@ error: * The meta_aggr is updated to point to the new space * The block of 1034 is allocated from the new block and should be aligned * There is space of 1014 left in meta_aggr - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY - * + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5024,7 +4960,7 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5FD_mem_t type, stype; haddr_t addr1, addr2, addr3; haddr_t saddr1, saddr2, saddr3; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, sdata_size=0, mis_align=0; hsize_t alignment=0, tmp=0; @@ -5067,48 +5003,48 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 50 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 50 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE50) != ma_addr) + if((addr2 + TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* @@ -5123,84 +5059,84 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) * block 30 is allocated from ma_addr */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) mis_align = alignment - tmp; else if ((alignment == TEST_ALIGN4096) && (tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (saddr1 % alignment) TEST_ERROR + if(saddr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr1+TEST_BLOCK_SIZE30)) TEST_ERROR + if(sdata_addr != (saddr1+TBLOCK_SIZE30)) TEST_ERROR /* calculate fragment for alignment of block 50 in sdata_aggr */ mis_align = 0; - if ((tmp = sdata_addr % alignment)) + if((tmp = sdata_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 50 from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (saddr2 % alignment) TEST_ERROR + if(saddr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr2 + TEST_BLOCK_SIZE50)) TEST_ERROR + if(sdata_addr != (saddr2 + TBLOCK_SIZE50)) TEST_ERROR /* calculate fragment for alignment of block 80 in sdata_aggr */ mis_align = 0; - if ((tmp = sdata_addr % alignment)) + if((tmp = sdata_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 80 from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ - if (saddr3 % alignment) TEST_ERROR + if(saddr3 % alignment) TEST_ERROR /* fragment for alignment of block 80 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr3 + TEST_BLOCK_SIZE80) != sdata_addr) + if ((saddr3 + TBLOCK_SIZE80) != sdata_addr) TEST_ERROR /* calculate fragment for alignment of block 1034 */ mis_align = 0; - if ((tmp = sdata_addr % alignment)) + if((tmp = sdata_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 1034 for meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1034); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034); /* Verify that the allocated block is aligned */ - if (addr3 % alignment) TEST_ERROR + if(addr3 % alignment) TEST_ERROR /* fragment for alignment of block 1034 for meta_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5208,11 +5144,11 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* calculate unused space in meta_aggr that is freed to free-space after block 1034 */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) mis_align = alignment - tmp; /* fragment for unused space in meta_aggr after block 1034 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5220,7 +5156,7 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3 + TEST_BLOCK_SIZE1034) != ma_addr) + if((addr3 + TBLOCK_SIZE1034) != ma_addr) TEST_ERROR /* Verify total size of free space after all allocations */ @@ -5255,6 +5191,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 4096 aggr->alloc_size = 2048 @@ -5317,6 +5254,7 @@ error: * Fragment from alignment of aggregator allocation is freed to free-space:[10250, 2038] * There is space of 2023 left in meta_aggr * + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5327,7 +5265,7 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) h5_stat_size_t file_size; H5FD_mem_t type; haddr_t addr1, addr2, addr3; - frspace_state_t state; + H5FS_stat_t state; haddr_t ma_addr=HADDR_UNDEF; hsize_t ma_size=0, saved_ma_size=0; hsize_t alignment=0, mis_align=0, tmp=0; @@ -5370,19 +5308,19 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -5390,21 +5328,21 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); saved_ma_size = ma_size; - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 2058 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 2058 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 2058 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -5412,28 +5350,28 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE2058) != ma_addr) TEST_ERROR + if((addr2 + TBLOCK_SIZE2058) != ma_addr) TEST_ERROR /* meta_aggr->size remains the same */ - if (ma_size != saved_ma_size) TEST_ERROR + if(ma_size != saved_ma_size) TEST_ERROR /* calculate fragment for alignment of block 5 from meta_aggr */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 5 from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5); /* fragment for alignment of block 5 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; } /* Verify that the allocated block is aligned */ - if (addr3 % alignment) TEST_ERROR + if(addr3 % alignment) TEST_ERROR /* Verify total size of free space after all allocations */ if(f->shared->fs_man[type]) { @@ -5461,6 +5399,7 @@ error: } /* test_mf_align_alloc4() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -5525,10 +5464,7 @@ error: * sdata_aggr is reset to 0 * EOA is 14346 * meta_aggr and sdata_aggr are all 0 - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5539,7 +5475,7 @@ test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) h5_stat_size_t file_size; H5FD_mem_t type, stype; haddr_t addr1, addr2, saddr1; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, new_ma_addr=HADDR_UNDEF; haddr_t sdata_addr=HADDR_UNDEF, new_sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, new_ma_size=0, sdata_size=0, new_sdata_size=0; @@ -5583,23 +5519,23 @@ test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR + if((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5607,39 +5543,39 @@ test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* calculate fragment for alignment of block 30 from sdata_aggr */ mis_align = 0; - if ((tmp = (ma_addr + ma_size) % alignment)) + if((tmp = (ma_addr + ma_size) % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (saddr1 % alignment) TEST_ERROR + if(saddr1 % alignment) TEST_ERROR /* fragment of alignment for block 30 in sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) TEST_ERROR + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* calculate fragment for alignment of block 2058 from meta_aggr */ mis_align = 0; - if ((tmp = (sdata_addr + sdata_size) % alignment)) + if((tmp = (sdata_addr + sdata_size) % alignment)) mis_align = alignment - tmp; /* Allocate a block of 2058 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); /* Verify that the allocated block is aligned */ if (addr2 % alignment) TEST_ERROR /* fragment for alignment of block 2058 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5687,6 +5623,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -5792,10 +5729,7 @@ error: * Fragment from alignment of file allocation is freed to free-space:[16464, 4016] * EOA is at 22538 * meta_aggr is unchanged - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5807,7 +5741,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5FD_mem_t type, stype; haddr_t addr1, addr2; haddr_t saddr1, saddr2, saddr3; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, new_ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, new_ma_size=0, sdata_size=0; hsize_t alignment=0, mis_align=0, tmp=0; @@ -5849,26 +5783,26 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ if (addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 in meta_aggr is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 30 in sdata_aggr */ @@ -5878,7 +5812,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ if (saddr1 % alignment) TEST_ERROR @@ -5891,7 +5825,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr1+TEST_BLOCK_SIZE30)) TEST_ERROR + if (sdata_addr != (saddr1+TBLOCK_SIZE30)) TEST_ERROR /* calculate fragment for alignment of block 50 in sdata_aggr */ mis_align = 0; @@ -5899,7 +5833,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = alignment - tmp; /* Allocate a block of 50 from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ if (saddr2 % alignment) TEST_ERROR @@ -5912,7 +5846,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr2+TEST_BLOCK_SIZE50)) TEST_ERROR + if (sdata_addr != (saddr2+TBLOCK_SIZE50)) TEST_ERROR /* calculate fragment for alignment of block 80 in sdata_aggr */ mis_align = 0; @@ -5920,7 +5854,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = alignment - tmp; /* Allocate a block of 80 from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ if (saddr3 % alignment) TEST_ERROR @@ -5933,7 +5867,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr3+TEST_BLOCK_SIZE80)) TEST_ERROR + if (sdata_addr != (saddr3+TBLOCK_SIZE80)) TEST_ERROR /* calculate fragment for alignment of block 2058 */ /* remaining space in sdata_aggr is freed and shrunk */ @@ -5942,7 +5876,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = alignment - tmp; /* Allocate a block of 2058 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); /* Verify that the allocated block is aligned */ if (addr2 % alignment) TEST_ERROR @@ -5993,7 +5927,6 @@ error: return(1); } /* test_mf_align_alloc6() */ - /* * Test a bug that occurs when an allocator with zero size left and an unaligned * endpoint is extended to allocate an aligned object @@ -6156,334 +6089,111 @@ error: return(1); } /* test_mf_bug1() */ - /* - * Verify that the file's free-space manager persists where there are free sections in the manager + * Verify that the file's free-space manager(s) are persistent for a split-file + *------------------------------------------------------------------------- */ static unsigned -test_mf_fs_persist(hid_t fapl_new, hid_t fcpl) +test_mf_fs_persist_split(void) { - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ - haddr_t tmp_addr; /* Temporary variable for address */ - - TESTING("file's free-space manager is persistent"); - - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Allocate 6 blocks */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - - /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Verify that H5FD_MEM_SUPER free-space manager is there */ - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) - FAIL_STACK_ERROR - - /* Get info for free-space manager */ - if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0) - FAIL_STACK_ERROR - - /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3+TEST_BLOCK_SIZE5)) - TEST_ERROR - - if(fs_stat.serial_sect_count < 3) - TEST_ERROR - - /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(tmp_addr != addr3) - TEST_ERROR - - /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(tmp_addr != addr1) - TEST_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl = -1; /* File access property list ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type, stype, btype; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ + haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_DRAW */ + haddr_t baddr5, baddr6, baddr7, baddr8; /* File address for H5FD_MEM_BTREE */ + haddr_t tmp_addr; /* temporary variable for address */ + + TESTING("File's free-space managers are persistent for split-file"); + + /* for now, we don't support persistant free space managers + * with the split file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ - /* Verify that H5FD_MEM_SUPER free-space manager is there */ - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - /* Retrieve block #5 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(tmp_addr != addr5) - TEST_ERROR + /* for now, we don't support persistant free space managers + * with the split file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - PASSED() + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR - return(0); + /* Set up split driver */ + if(H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT)<0) + FAIL_STACK_ERROR -error: - H5E_BEGIN_TRY { - H5Fclose(file); - } H5E_END_TRY; - return(1); -} /* test_mf_fs_persist() */ - -/* - * Verify that the free-space manager goes away - */ -static unsigned -test_mf_fs_gone(hid_t fapl_new, hid_t fcpl) -{ - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR - TESTING("file's free-space manager is going away"); + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - /* Allocate 4 blocks */ + /* Allocate 4 blocks of type H5FD_MEM_SUPER */ type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - - /* Put block #1, #3 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - - /* Retrieve block #1, #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Put block #3 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) FAIL_STACK_ERROR - - /* Verify that H5FD_MEM_SUPER free-space manager is there */ - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) FAIL_STACK_ERROR - /* Get info for H5FD_MEM_SUPER free-space manager */ - if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0) + /* Put block #1, #3 into H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) FAIL_STACK_ERROR - - /* Verify free-space info */ - if(!H5F_addr_defined(fs_stat.addr) || !H5F_addr_defined(fs_stat.sect_addr)) - TEST_ERROR - if(fs_stat.tot_space < TEST_BLOCK_SIZE3) - TEST_ERROR - - /* Put block #4 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TEST_BLOCK_SIZE4) < 0) + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) FAIL_STACK_ERROR - /* The H5FD_MEM_SUPER free-space manager will go away at H5MF_close() */ - if(H5Fclose(file) < 0) + /* Allocate 4 blocks of type H5FD_MEM_DRAW */ + stype = H5FD_MEM_DRAW; + if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) FAIL_STACK_ERROR - - /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - if(H5Fclose(file) < 0) + if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) FAIL_STACK_ERROR - PASSED() - - return(0); - -error: - H5E_BEGIN_TRY { - H5Fclose(file); - } H5E_END_TRY; - return(1); -} /* test_mf_fs_gone() */ - - -/* - * Verify that the file's free-space manager(s) are persistent for a split-file - */ -static unsigned -test_mf_fs_split(hid_t fapl_new, hid_t fcpl) -{ - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type, stype, btype; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ - haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_DRAW */ - haddr_t baddr5, baddr6, baddr7, baddr8; /* File address for H5FD_MEM_BTREE */ - haddr_t tmp_addr; /* temporary variable for address */ - - TESTING("file's free-space managers are persistent for split-file"); - - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + /* Put block #1, #3 into H5FD_MEM_DRAW free-space manager */ + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE1) < 0) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE3) < 0) FAIL_STACK_ERROR - /* Allocate 4 blocks of type H5FD_MEM_SUPER */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - - /* Put block #1, #3 into H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - - /* Allocate 4 blocks of type H5FD_MEM_DRAW */ - stype = H5FD_MEM_DRAW; - if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - - /* Put block #1, #3 into H5FD_MEM_DRAW free-space manager */ - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6495,7 +6205,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6503,23 +6213,24 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR - /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #2 still in free-space */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #3 still in free-space */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != addr1) - TEST_ERROR + TEST_ERROR /* Verify that the free-space manager for H5FD_MEM_DRAW is there */ if(!H5F_addr_defined(f->shared->fs_addr[stype])) TEST_ERROR + /* Start up H5FD_MEM_DRAW free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)stype) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6527,47 +6238,47 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #1 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != saddr1) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #3 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != saddr3) - TEST_ERROR + TEST_ERROR /* H5FD_MEM_DRAW free-space manager is going away at closing */ /* works for this one because the freeing of sect_addr is to H5FD_MEM_SUPER fs, not against itself */ /* Allocate 4 blocks of type H5FD_MEM_BTREE */ btype = H5FD_MEM_BTREE; - if(HADDR_UNDEF == (baddr5 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr6 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr7 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE7))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr8 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE8))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr5 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr6 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr7 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE7))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr8 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8))) + FAIL_STACK_ERROR /* Put block #5 & #7 into H5FD_MEM_BTREE free-space manager */ - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr7, (hsize_t)TEST_BLOCK_SIZE7) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr7, (hsize_t)TBLOCK_SIZE7) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6583,7 +6294,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6591,27 +6302,27 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE3+TEST_BLOCK_SIZE5+TEST_BLOCK_SIZE7)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE3+TBLOCK_SIZE5+TBLOCK_SIZE7)) + TEST_ERROR /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != addr3) - TEST_ERROR + TEST_ERROR /* Retrieve block #7 from H5FD_MEM_BTREE free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE7))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE7))) + FAIL_STACK_ERROR if(tmp_addr != baddr7) - TEST_ERROR + TEST_ERROR /* There should still be block #5 of H5FD_MEM_BTREE in H5FD_MEM_BTREE free-space manager */ if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6620,10 +6331,10 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) /* Verify that the H5FD_MEM_SUPER free-space manager is there */ if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR + TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6631,11 +6342,16 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < TEST_BLOCK_SIZE5) - TEST_ERROR + if(fs_stat.tot_space < TBLOCK_SIZE5) + TEST_ERROR + /* Closing */ if(H5Fclose(file) < 0) FAIL_STACK_ERROR + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR PASSED() @@ -6643,37 +6359,113 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(file); + H5Pclose(fcpl); + H5Pclose(fapl); } H5E_END_TRY; return(1); -} /* test_mf_fs_split() */ +} /* test_mf_fs_persist_split() */ + +#define MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) { \ + H5FD_mem_t mt; \ + HDmemset(memb_map, 0, sizeof memb_map); \ + HDmemset(memb_fapl, 0, sizeof memb_fapl); \ + HDmemset(memb_name, 0, sizeof memb_name); \ + HDmemset(memb_addr, 0, sizeof memb_addr); \ + HDmemset(sv, 0, sizeof sv); \ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { \ + memb_map[mt] = H5FD_MEM_SUPER; \ + memb_fapl[mt] = H5P_DEFAULT; \ + } \ + memb_map[H5FD_MEM_BTREE] = H5FD_MEM_BTREE; \ + memb_map[H5FD_MEM_DRAW] = H5FD_MEM_DRAW; \ + memb_map[H5FD_MEM_GHEAP] = H5FD_MEM_GHEAP; \ + memb_map[H5FD_MEM_LHEAP] = H5FD_MEM_LHEAP; \ + sprintf(sv[H5FD_MEM_SUPER], "%%s-%c.h5", 's'); \ + memb_name[H5FD_MEM_SUPER] = sv[H5FD_MEM_SUPER]; \ + memb_addr[H5FD_MEM_SUPER] = 0; \ + sprintf(sv[H5FD_MEM_BTREE], "%%s-%c.h5", 'b'); \ + memb_name[H5FD_MEM_BTREE] = sv[H5FD_MEM_BTREE]; \ + memb_addr[H5FD_MEM_BTREE] = HADDR_MAX/6; \ + sprintf(sv[H5FD_MEM_DRAW], "%%s-%c.h5", 'r'); \ + memb_name[H5FD_MEM_DRAW] = sv[H5FD_MEM_DRAW]; \ + memb_addr[H5FD_MEM_DRAW] = HADDR_MAX/3; \ + sprintf(sv[H5FD_MEM_GHEAP], "%%s-%c.h5", 'g'); \ + memb_name[H5FD_MEM_GHEAP] = sv[H5FD_MEM_GHEAP]; \ + memb_addr[H5FD_MEM_GHEAP] = HADDR_MAX/2; \ + sprintf(sv[H5FD_MEM_LHEAP], "%%s-%c.h5", 'l'); \ + memb_name[H5FD_MEM_LHEAP] = sv[H5FD_MEM_LHEAP]; \ + memb_addr[H5FD_MEM_LHEAP] = HADDR_MAX*2/3; \ + sprintf(sv[H5FD_MEM_OHDR], "%%s-%c.h5", 'o'); \ + memb_name[H5FD_MEM_OHDR] = sv[H5FD_MEM_OHDR]; \ + memb_addr[H5FD_MEM_OHDR] = HADDR_MAX*5/6; \ +} /* + *------------------------------------------------------------------------- * Verify that the file's free-space manager(s) are persistent for a multi-file + *------------------------------------------------------------------------- */ static unsigned -test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) +test_mf_fs_persist_multi(void) { - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type, stype, btype, gtype; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4; /* File allocation type */ - haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_SUPER */ - haddr_t baddr1, baddr2, baddr3, baddr4; /* File address for H5FD_MEM_DRAW */ - haddr_t gaddr1, gaddr2; /* File address for H5FD_MEM_GHEAP */ - haddr_t tmp_addr; /* Temporary variable for address */ - H5FS_section_info_t *node; /* Free space section node */ - htri_t node_found = FALSE; /* Indicate section is in free-space */ - - TESTING("file's free-space managers are persistent for multi-file"); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl = -1; /* File access property list ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type, stype, btype, gtype; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4; /* File allocation type */ + haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_SUPER */ + haddr_t baddr1, baddr2, baddr3, baddr4; /* File address for H5FD_MEM_DRAW */ + haddr_t gaddr1, gaddr2; /* File address for H5FD_MEM_GHEAP */ + haddr_t tmp_addr; /* Temporary variable for address */ + H5FS_section_info_t *node; /* Free space section node */ + htri_t node_found = FALSE; /* Indicate section is in free-space */ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; /* Memory usage map */ + hid_t memb_fapl[H5FD_MEM_NTYPES]; /* Member access properties */ + char sv[H5FD_MEM_NTYPES][64]; /* Name generators */ + const char *memb_name[H5FD_MEM_NTYPES]; /* Name generators */ + haddr_t memb_addr[H5FD_MEM_NTYPES]; /* Member starting address */ + + + TESTING("File's free-space managers are persistent for multi-file"); + + /* for now, we don't support persistant free space managers + * with the multi file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ + + /* for now, we don't support persistant free space managers + * with the multi file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ + + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR + + MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) + + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, TRUE) < 0) + TEST_ERROR; /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6682,43 +6474,43 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) /* Allocate 4 blocks of type H5FD_MEM_SUPER */ type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR /* Put block #1, #3 into H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR /* Allocate 4 blocks of type H5FD_MEM_DRAW */ stype = H5FD_MEM_DRAW; - if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR /* Put block #1, #3 into H5FD_MEM_DRAW free-space manager */ - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6730,7 +6522,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6738,23 +6530,23 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR - /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #2 still in free-space */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #3 still in free-space */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != addr1) - TEST_ERROR + TEST_ERROR /* Verify that the free-space manager for H5FD_MEM_DRAW is there */ if(!H5F_addr_defined(f->shared->fs_addr[stype])) TEST_ERROR /* Start up H5FD_MEM_DRAW free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)stype) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6762,45 +6554,45 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #1 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != saddr1) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #3 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != saddr3) - TEST_ERROR + TEST_ERROR /* Allocate 4 blocks of type H5FD_MEM_BTREE */ btype = H5FD_MEM_BTREE; - if(HADDR_UNDEF == (baddr1 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr2 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr3 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr4 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr1 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr2 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr3 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr4 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR /* Put block #1 & #3 into H5FD_MEM_BTREE free-space manager */ - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6812,7 +6604,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6820,14 +6612,14 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < TEST_BLOCK_SIZE3) - TEST_ERROR + if(fs_stat.tot_space < TBLOCK_SIZE3) + TEST_ERROR /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != addr3) - TEST_ERROR + TEST_ERROR /* Verify that the free-space manager for H5FD_MEM_DRAW is not there */ if(H5F_addr_defined(f->shared->fs_addr[stype])) @@ -6838,7 +6630,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_BTREE free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, btype) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)btype) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6846,27 +6638,27 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR /* Allocate 2 blocks of type H5FD_MEM_GHEAP */ gtype = H5FD_MEM_GHEAP; - if(HADDR_UNDEF == (gaddr2 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (gaddr1 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (gaddr2 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (gaddr1 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR /* Put block #2 into H5FD_MEM_GHEAP free-space manager */ - if(H5MF_xfree(f, gtype, H5AC_ind_read_dxpl_id, gaddr2, (hsize_t)TEST_BLOCK_SIZE2) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, gtype, H5AC_ind_read_dxpl_id, gaddr2, (hsize_t)TBLOCK_SIZE2) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6875,39 +6667,44 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) /* If H5FD_MEM_SUPER is there, should not find block #1 & #3 */ if(H5F_addr_defined(f->shared->fs_addr[type])) { - /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) - FAIL_STACK_ERROR - - if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE1, (H5FS_section_info_t **)&node)) < 0) - FAIL_STACK_ERROR - if(node_found) TEST_ERROR - - if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE3, (H5FS_section_info_t **)&node)) < 0) - FAIL_STACK_ERROR - if(node_found) TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) + FAIL_STACK_ERROR + + if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], + (hsize_t)TBLOCK_SIZE1, (H5FS_section_info_t **)&node)) < 0) + FAIL_STACK_ERROR + if(node_found) TEST_ERROR + + if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], + (hsize_t)TBLOCK_SIZE3, (H5FS_section_info_t **)&node)) < 0) + FAIL_STACK_ERROR + if(node_found) TEST_ERROR } /* Verify that the H5FD_MEM_GHEAP free-space manager is there */ if(!H5F_addr_defined(f->shared->fs_addr[gtype])) - TEST_ERROR + TEST_ERROR /* Start up H5FD_MEM_GHEAP free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, gtype) < 0) - FAIL_STACK_ERROR + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)gtype) < 0) + FAIL_STACK_ERROR /* Get free-space info */ if(H5FS_stat_info(f, f->shared->fs_man[gtype], &fs_stat) < 0) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < TEST_BLOCK_SIZE2) - TEST_ERROR + if(fs_stat.tot_space < TBLOCK_SIZE2) + TEST_ERROR + /* Closing */ if(H5Fclose(file) < 0) FAIL_STACK_ERROR + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR PASSED() @@ -6915,460 +6712,540 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(file); + H5Pclose(fcpl); + H5Pclose(fapl); } H5E_END_TRY; return(1); -} /* test_mf_fs_multi() */ - -#define MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) { \ - H5FD_mem_t mt; \ - HDmemset(memb_map, 0, sizeof memb_map); \ - HDmemset(memb_fapl, 0, sizeof memb_fapl); \ - HDmemset(memb_name, 0, sizeof memb_name); \ - HDmemset(memb_addr, 0, sizeof memb_addr); \ - HDmemset(sv, 0, sizeof sv); \ - for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { \ - memb_map[mt] = H5FD_MEM_SUPER; \ - memb_fapl[mt] = H5P_DEFAULT; \ - } \ - memb_map[H5FD_MEM_BTREE] = H5FD_MEM_BTREE; \ - memb_map[H5FD_MEM_DRAW] = H5FD_MEM_DRAW; \ - memb_map[H5FD_MEM_GHEAP] = H5FD_MEM_GHEAP; \ - memb_map[H5FD_MEM_LHEAP] = H5FD_MEM_LHEAP; \ - sprintf(sv[H5FD_MEM_SUPER], "%%s-%c.h5", 's'); \ - memb_name[H5FD_MEM_SUPER] = sv[H5FD_MEM_SUPER]; \ - memb_addr[H5FD_MEM_SUPER] = 0; \ - sprintf(sv[H5FD_MEM_BTREE], "%%s-%c.h5", 'b'); \ - memb_name[H5FD_MEM_BTREE] = sv[H5FD_MEM_BTREE]; \ - memb_addr[H5FD_MEM_BTREE] = HADDR_MAX/6; \ - sprintf(sv[H5FD_MEM_DRAW], "%%s-%c.h5", 'r'); \ - memb_name[H5FD_MEM_DRAW] = sv[H5FD_MEM_DRAW]; \ - memb_addr[H5FD_MEM_DRAW] = HADDR_MAX/3; \ - sprintf(sv[H5FD_MEM_GHEAP], "%%s-%c.h5", 'g'); \ - memb_name[H5FD_MEM_GHEAP] = sv[H5FD_MEM_GHEAP]; \ - memb_addr[H5FD_MEM_GHEAP] = HADDR_MAX/2; \ - sprintf(sv[H5FD_MEM_LHEAP], "%%s-%c.h5", 'l'); \ - memb_name[H5FD_MEM_LHEAP] = sv[H5FD_MEM_LHEAP]; \ - memb_addr[H5FD_MEM_LHEAP] = HADDR_MAX*2/3; \ - sprintf(sv[H5FD_MEM_OHDR], "%%s-%c.h5", 'o'); \ - memb_name[H5FD_MEM_OHDR] = sv[H5FD_MEM_OHDR]; \ - memb_addr[H5FD_MEM_OHDR] = HADDR_MAX*5/6; \ -} +} /* test_mf_fs_persist_multi() */ /* - * Tests to verify that file's free-space managers are persistent or going away - * for different drivers. + *------------------------------------------------------------------------- + * Verify that the file's free-space persists where there are free sections in the manager + *------------------------------------------------------------------------- */ static unsigned -test_mf_fs_drivers(hid_t fapl) +test_mf_fs_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t fcpl = -1; /* file creation property list */ - hid_t fapl_new = -1; /* copy of file access property list */ - hid_t fapl2 = -1; /* copy of file access property list */ - unsigned new_format; /* To use new library format or not */ - unsigned ret = 0; /* return value */ - - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; /* Memory usage map */ - hid_t memb_fapl[H5FD_MEM_NTYPES]; /* Member access properties */ - char sv[H5FD_MEM_NTYPES][64]; /* Name generators */ - const char *memb_name[H5FD_MEM_NTYPES]; /* Name generators */ - haddr_t memb_addr[H5FD_MEM_NTYPES]; /* Member starting address */ - - /* Create a non-standard file-creation template */ - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - if(H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0) < 0) - TEST_ERROR - - /* Copy the file access property list */ - if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR - - /* Set the "use the latest version of the format" bounds for creating objects in the file */ - if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) - TEST_ERROR - - /* Test with old and new format */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { - - if(new_format) - HDputs("Testing the following tests for free-space managers with new library format..."); - else - HDputs("Testing the following tests for free-space managers with old library format..."); - - /* SEC2 */ - HDputs("Testing free-space manager(s) with sec2 driver"); - - if((fapl_new = H5Pcopy(new_format ? fapl2 : fapl)) < 0) TEST_ERROR - - if(H5Pset_fapl_sec2(fapl_new) < 0) - FAIL_STACK_ERROR - - ret += test_mf_fs_gone(fapl_new, fcpl); - ret += test_mf_fs_persist(fapl_new, fcpl); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl2 = -1; /* File access property list ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FD_mem_t tt; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ + haddr_t tmp_addr; /* Temporary variable for address */ + + if(new_format) + TESTING("File's free-space is persistent with new library format") + else + TESTING("File's free-space is persistent with old library format") + + if(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")) { + + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR + + if(new_format) { + /* Latest format */ + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + /* Set to paged aggregation and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + TEST_ERROR + } else { + /* Setting: aggregation with persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + TEST_ERROR + } - h5_clean_files(FILENAME, fapl_new); + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR - /* STDIO */ - HDputs("Testing free-space managers with stdio driver"); + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_stdio(fapl_new) < 0) - FAIL_STACK_ERROR + /* Allocate 6 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR - ret += test_mf_fs_gone(fapl_new, fcpl); - ret += test_mf_fs_persist(fapl_new, fcpl); + /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - /* CORE */ - HDputs("Testing free-space managers with core driver"); + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR - /* create fapl to be a "core" file */ - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_core(fapl_new, (size_t)CORE_INCREMENT, TRUE) < 0) - FAIL_STACK_ERROR + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - ret += test_mf_fs_gone(fapl_new, fcpl); - ret += test_mf_fs_persist(fapl_new, fcpl); + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE6, (H5F_mem_page_t *)&tt); - h5_clean_files(FILENAME, fapl_new); + /* Verify that H5FD_MEM_SUPER free-space manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR - /* FAMILY */ - HDputs("Testing free-space managers with family driver"); + /* Since we are about to open a self referential free space + * manager prior to the first file space allocation / deallocation + * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid + * assertion failures on the first file space alloc / dealloc. + */ + if((f->shared->first_alloc_dealloc) && + (SUCCEED != + H5MF_tidy_self_referential_fsm_hack(f, H5AC_ind_read_dxpl_id))) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_family(fapl_new, (hsize_t)FAMILY_SIZE, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(!(f->shared->fs_man[tt])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)tt) < 0) + FAIL_STACK_ERROR - ret += test_mf_fs_persist(fapl_new, fcpl); + /* Get info for free-space manager */ + if(H5FS_stat_info(f, f->shared->fs_man[tt], &fs_stat) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + /* Verify free-space info */ + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3+TBLOCK_SIZE5)) + TEST_ERROR - /* SPLIT */ - HDputs("Testing free-space managers with split driver"); + if(fs_stat.serial_sect_count < 3) + TEST_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_split(fapl_new, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT)<0) - FAIL_STACK_ERROR + /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(tmp_addr != addr3) + TEST_ERROR - ret += test_mf_fs_persist(fapl_new, fcpl); - ret += test_mf_fs_split(fapl_new, fcpl); + /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(tmp_addr != addr1) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - /* MULTI */ - HDputs("Testing free-space managers with multi driver"); + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + FAIL_STACK_ERROR - MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl, memb_name, memb_addr, TRUE) < 0) - TEST_ERROR; + /* Verify that H5FD_MEM_SUPER free-space manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR - ret += test_mf_fs_multi(fapl_new, fcpl); + /* Retrieve block #5 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(tmp_addr != addr5) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR - } /* end for new_format */ + PASSED() - if(H5Pclose(fcpl) < 0) - FAIL_STACK_ERROR - if(H5Pclose(fapl2) < 0) - FAIL_STACK_ERROR + } else { + SKIPPED(); + puts(" Current VFD doesn't support persisting free-space or paged aggregation strategy"); + } - return(ret); + return(0); error: H5E_BEGIN_TRY { + H5Fclose(file); H5Pclose(fcpl); H5Pclose(fapl2); - H5Pclose(fapl_new); } H5E_END_TRY; return(1); -} /* test_mf_fs_drivers() */ - +} /* test_mf_fs_persist() */ /* - * Verify that file space management performs according to the - * file space strategy and free space threshold as specified. + *------------------------------------------------------------------------- + * Verify free-space are merged/shrunk away + *------------------------------------------------------------------------- */ static unsigned -test_filespace_strategy_threshold(hid_t fapl_new) +test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t file = -1; /* File ID */ - hid_t fcpl = -1; /* File creation property list template */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ - haddr_t tmp_addr; /* Temporary variable for address */ - H5F_file_space_type_t fs_type; /* File space handling strategy */ - hsize_t fs_threshold; /* Free space section threshold */ - hsize_t tot_space, saved_tot_space; /* Total amount of free space */ - hsize_t tot_sect_count, saved_tot_sect_count; /* # of free-space sections */ - - TESTING("file space strategy and threshold"); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t fapl2 = -1; /* File access property list */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ + H5FD_mem_t fs_type; + hbool_t contig_addr_vfd; + hbool_t ran_H5MF_tidy_self_referential_fsm_hack = FALSE; + + if(new_format) + TESTING("File's free-space is going away with new library format") + else + TESTING("File's free-space is going away with old library format") + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - for(fs_threshold = 0; fs_threshold <= TEST_THRESHOLD10; fs_threshold++) { - - for(fs_type = H5F_FILE_SPACE_ALL_PERSIST; fs_type < H5F_FILE_SPACE_NTYPES; H5_INC_ENUM(H5F_file_space_type_t, fs_type)) { - - /* Create file-creation template */ - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set default file space information */ - if(H5Pset_file_space(fcpl, fs_type, fs_threshold) < 0) - FAIL_STACK_ERROR - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Allocate 6 blocks */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - - /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - - /* Retrieve the total amount of free space and # of free-space sections */ - if(f->shared->fs_man[type] && - H5FS_sect_stats(f->shared->fs_man[type], &saved_tot_space, &saved_tot_sect_count) < 0) - FAIL_STACK_ERROR - - /* H5F_FILE_SPACE_AGGR_VFD and H5F_FILE_SPACE_VFD: should not have free-space manager */ - if(fs_type > H5F_FILE_SPACE_ALL && f->shared->fs_man[type]) - TEST_ERROR + if(contig_addr_vfd) { - /* Close the file */ - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - switch(fs_type) { - case H5F_FILE_SPACE_ALL_PERSIST: - if(fs_threshold <= TEST_BLOCK_SIZE5) { - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Open the free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) - FAIL_STACK_ERROR - - /* Retrieve the total amount of free space and # of free-space sections */ - if(H5FS_sect_stats(f->shared->fs_man[type], &tot_space, &tot_sect_count) < 0) - FAIL_STACK_ERROR - - /* Verify that tot_space should be >= saved_tot_space */ - /* Verify that tot_sect_count should be >= saved_tot_sect_count */ - if(tot_space < saved_tot_space || tot_sect_count < saved_tot_sect_count) - TEST_ERROR - - /* Retrieve block #5 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == - (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - - /* Should be the same as before */ - if(tmp_addr != addr5) - TEST_ERROR - } else if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - break; - - case H5F_FILE_SPACE_ALL: - case H5F_FILE_SPACE_AGGR_VFD: - case H5F_FILE_SPACE_VFD: - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - break; - - case H5F_FILE_SPACE_DEFAULT: - case H5F_FILE_SPACE_NTYPES: - default: - TEST_ERROR - break; - } /* end switch */ + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR - /* Closing */ - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - if(H5Pclose(fcpl) < 0) - FAIL_STACK_ERROR + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR - } /* end for fs_type */ - } /* end for fs_threshold */ + if(new_format) + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR - PASSED() + /* Set to aggregation and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); + + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Allocate 4 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + + /* Put block #1, #3 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + + /* Retrieve block #1, #3 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE4, (H5F_mem_page_t *)&fs_type); + + /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ + if(H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Put block #3 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Verify that H5FD_MEM_SUPER free-space manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Since we are about to open a self referential free space + * manager prior to the first file space allocation / deallocation + * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid + * assertion failures on the first file space alloc / dealloc. + */ + if(f->shared->first_alloc_dealloc){ + if(SUCCEED!=H5MF_tidy_self_referential_fsm_hack(f,H5AC_ind_read_dxpl_id)) + FAIL_STACK_ERROR + ran_H5MF_tidy_self_referential_fsm_hack = TRUE; + } + + /* Start up H5FD_MEM_SUPER free-space manager */ + if(!(f->shared->fs_man[fs_type])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)fs_type) < 0) + FAIL_STACK_ERROR + + /* Get info for H5FD_MEM_SUPER free-space manager */ + if(H5FS_stat_info(f, f->shared->fs_man[fs_type], &fs_stat) < 0) + FAIL_STACK_ERROR + + /* if we ran H5MF_tidy_self_referential_fsm_hack(), the + * H5FD_MEM_SUPER free space manager must be floating. + * Thus fs_stat.addr must be undefined. + */ + if((!ran_H5MF_tidy_self_referential_fsm_hack) && + (!H5F_addr_defined(fs_stat.addr))) + TEST_ERROR + + if(fs_stat.tot_space < TBLOCK_SIZE3) + TEST_ERROR + + /* Put block #4 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TBLOCK_SIZE4) < 0) + FAIL_STACK_ERROR + + /* The H5FD_MEM_SUPER free-space manager will go away at H5MF_close() */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ + if(H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Closing */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support persistent free-space manager"); + } return(0); error: H5E_BEGIN_TRY { - H5Pclose(fcpl); H5Fclose(file); + H5Pclose(fcpl); + H5Pclose(fapl2); } H5E_END_TRY; return(1); -} /* test_filespace_strategy_threshold() */ +} /* test_mf_fs_gone() */ /* - * Verify section is merged/shrunk away for - * H5F_FILE_SPACE_ALL_PERSIST and H5F_FILE_SPACE_ALL strategy. + *------------------------------------------------------------------------- + * Verify that free-space persist with combinations of + * file space strategy and free space threshold as specified. + *------------------------------------------------------------------------- */ static unsigned -test_filespace_gone(hid_t fapl_new) +test_mf_strat_thres_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t file = -1; /* File ID */ - hid_t fcpl = -1; /* File creation propertly list template */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ - H5F_file_space_type_t fs_type; /* File space handling strategy */ - hsize_t fs_threshold; /* Free space section threshold */ - frspace_state_t state; /* State of free space manager */ - - TESTING("file space merge/shrink for section size < threshold"); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list template */ + hid_t fapl2 = -1; /* File access property list template */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FD_mem_t tt; /* File allocation type */ + haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ + H5F_fspace_strategy_t fs_type; /* File space handling strategy */ + hsize_t fs_threshold; /* Free-space section threshold */ + unsigned fs_persist; /* To persist free-space or not */ + hbool_t contig_addr_vfd; + + if(new_format) + TESTING("File space strategy/persisting/threshold with new library format") + else + TESTING("File space strategy/persisting/threshold with old library format") + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); + /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - /* Set free-space threshold */ - fs_threshold = TEST_THRESHOLD3; - - for(fs_type = H5F_FILE_SPACE_ALL_PERSIST; fs_type <= H5F_FILE_SPACE_ALL; H5_INC_ENUM(H5F_file_space_type_t, fs_type)) { - /* Create file-creation template */ - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set default file space information */ - if(H5Pset_file_space(fcpl, fs_type, fs_threshold) < 0) - FAIL_STACK_ERROR - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Allocate 6 blocks */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - - /* Put block #3, #5 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE3 + TEST_BLOCK_SIZE5; - state.tot_sect_count += 2; - state.serial_sect_count += 2; - - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - /* section #2 is less than threshold but is merged into section #3 */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE2) < 0) - FAIL_STACK_ERROR - - state.tot_space += TEST_BLOCK_SIZE2; - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TEST_BLOCK_SIZE4) < 0) - FAIL_STACK_ERROR - - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr6, (hsize_t)TEST_BLOCK_SIZE6) < 0) - FAIL_STACK_ERROR - - /* all sections should be shrunk away except section #1 */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - /* section #1 is less than threshold but is shrunk away */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - - /* free-space manager should be empty */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* free-space manager should be empty */ - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - if(H5Pclose(fcpl) < 0) - FAIL_STACK_ERROR - - } /* end for fs_type */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) + FAIL_STACK_ERROR + + if(new_format) + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + + for(fs_threshold = 0; fs_threshold <= TEST_THRESHOLD10; fs_threshold++) { + + /* Testing for H5F_FSPACE_STRATEGY_FSM_AGGR and H5F_FSPACE_STRATEGY_PAGE strategies only */ + for(fs_type = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_type < H5F_FSPACE_STRATEGY_AGGR; H5_INC_ENUM(H5F_fspace_strategy_t, fs_type)) { + + if(!contig_addr_vfd && (fs_persist || fs_type == H5F_FSPACE_STRATEGY_PAGE)) + continue; + + /* Create file-creation template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set default file space information */ + if(H5Pset_file_space_strategy(fcpl, fs_type, (hbool_t)fs_persist, fs_threshold) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Allocate 6 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR + + /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR + + + /* Close the file */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE6, (H5F_mem_page_t *)&tt); + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + if(f->shared->fs_persist) { + hssize_t nsects; /* # of free-space sections */ + int i; /* local index variable */ + H5F_sect_info_t *sect_info; /* array to hold the free-space information */ + + /* Get the # of free-space sections in the file */ + if((nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, NULL)) < 0) + FAIL_STACK_ERROR + + /* Verify no free-space sections */ + /* paged aggregation has 1 section for last_small */ + if(fs_threshold > TBLOCK_SIZE5 && nsects && fs_type != H5F_FSPACE_STRATEGY_PAGE) + TEST_ERROR + + if(nsects) { + /* Allocate storage for the free space section information */ + sect_info = (H5F_sect_info_t *)HDcalloc((size_t)nsects, sizeof(H5F_sect_info_t)); + + H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)nsects, sect_info); + + /* Verify the size of free-space sections */ + for(i = 0; i < nsects; i++) + if(sect_info[i].size < fs_threshold) + TEST_ERROR + if(sect_info) + HDfree(sect_info); + } + } else { + if(H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR + } + + /* Closing */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + } /* end for fs_type */ + } /* end for fs_threshold */ + } /* end for fs_persist */ + + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR PASSED() @@ -7377,141 +7254,200 @@ test_filespace_gone(hid_t fapl_new) error: H5E_BEGIN_TRY { H5Pclose(fcpl); + H5Pclose(fapl2); H5Fclose(file); } H5E_END_TRY; return(1); -} /* test_filespace_gone() */ +} /* test_mf_strat_thres_persist() */ /* - * Tests to verify file space management for different drivers. + *------------------------------------------------------------------------- + * Verify free-space are merged/shrunk away with file space settings: + * --strategy, persist/not persist file space + *------------------------------------------------------------------------- */ static unsigned -test_filespace_drivers(hid_t fapl) +test_mf_strat_thres_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t fapl_new = -1; /* copy of file access property list */ - hid_t fapl2 = -1; /* copy of file access property list */ - unsigned new_format; /* Using library new format or not */ - unsigned ret = 0; /* return value */ + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list template */ + hid_t fapl2 = -1; /* File access property list template */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FD_mem_t tt; /* File allocation type */ + haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ + H5F_fspace_strategy_t fs_type; /* File space handling strategy */ + unsigned fs_persist; /* To persist free-space or not */ + H5FS_stat_t fs_state; /* Information for free-space manager */ + H5FS_stat_t fs_state_zero; /* Information for free-space manager */ + hbool_t contig_addr_vfd; + + if(new_format) + TESTING("File space merge/shrink for section size < threshold with new library format") + else + TESTING("File space merge/shrink for section size < threshold with old library format") + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; /* Memory usage map */ - hid_t memb_fapl[H5FD_MEM_NTYPES]; /* Member access properties */ - char sv[H5FD_MEM_NTYPES][64]; /* Name generators */ - const char *memb_name[H5FD_MEM_NTYPES]; /* Name generators */ - haddr_t memb_addr[H5FD_MEM_NTYPES]; /* Member starting address */ - /* Copy the file access property list */ - if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - /* Set the "use the latest version of the format" bounds for creating objects in the file */ - if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) - TEST_ERROR + HDmemset(&fs_state_zero, 0, sizeof(H5FS_stat_t)); - /* Test with old and new format */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) + FAIL_STACK_ERROR - if(new_format) - HDputs("Testing the following tests for file space management with new library format..."); - else - HDputs("Testing the following tests for file space management with old library format..."); + if(new_format) + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR - /* SEC2 */ - HDputs("Testing file space management with sec2 driver"); + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + /* Testing for H5F_FSPACE_STRATEGY_FSM_AGGR and H5F_FSPACE_STRATEGY_PAGE strategies only */ + for(fs_type = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_type < H5F_FSPACE_STRATEGY_AGGR; H5_INC_ENUM(H5F_fspace_strategy_t, fs_type)) { - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_sec2(fapl_new) < 0) - FAIL_STACK_ERROR + /* Skip for multi/split driver: persisting free-space or paged aggregation strategy */ + if(!contig_addr_vfd && (fs_persist || fs_type == H5F_FSPACE_STRATEGY_PAGE)) + continue; - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Clear out free-space statistics */ + HDmemset(&fs_state, 0, sizeof(H5FS_stat_t)); - h5_clean_files(FILENAME, fapl_new); + /* Create file-creation template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR - /* STDIO */ - HDputs("Testing file space management with stdio driver"); + /* Set default file space information */ + if(H5Pset_file_space_strategy(fcpl, fs_type, fs_persist, (hsize_t)TEST_THRESHOLD3) < 0) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_stdio(fapl_new) < 0) - FAIL_STACK_ERROR + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + /* Allocate 6 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR - /* CORE */ - HDputs("Testing file space management with core driver"); + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE6, (H5F_mem_page_t *)&tt); - /* create fapl to be a "core" file */ - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_core(fapl_new, (size_t)CORE_INCREMENT, TRUE) < 0) - FAIL_STACK_ERROR + /* For paged aggregation, the section in the page at EOF for small meta fs is not shrunk away */ + if(fs_type == H5F_FSPACE_STRATEGY_PAGE) { + if(H5FS_stat_info(f, f->shared->fs_man[tt], &fs_state) < 0) + FAIL_STACK_ERROR + } - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Put block #3, #5 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + fs_state.tot_space += TBLOCK_SIZE3 + TBLOCK_SIZE5; + fs_state.tot_sect_count += 2; + fs_state.serial_sect_count += 2; - /* FAMILY */ - HDputs("Testing file space managers with family driver"); + if(check_stats(f, f->shared->fs_man[tt], &fs_state)) + TEST_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_family(fapl_new, (hsize_t)FAMILY_SIZE, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR + /* section #2 is less than threshold but is merged into section #3 */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE2) < 0) + FAIL_STACK_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + fs_state.tot_space += TBLOCK_SIZE2; + if(check_stats(f, f->shared->fs_man[tt], &fs_state)) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TBLOCK_SIZE4) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr6, (hsize_t)TBLOCK_SIZE6) < 0) + FAIL_STACK_ERROR - /* SPLIT */ - HDputs("Testing file space managers with split driver"); + /* For paged aggregation, the sections in the page at EOF for small meta fs are merged but are not shrunk away */ + if(fs_type == H5F_FSPACE_STRATEGY_PAGE) { + fs_state.tot_sect_count = fs_state.serial_sect_count = 1; + fs_state.tot_space += (TBLOCK_SIZE4 + TBLOCK_SIZE6); + } - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_split(fapl_new, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT)<0) - FAIL_STACK_ERROR + /* For old format: the sections at EOF are shrunk away */ + if(check_stats(f, f->shared->fs_man[tt], (fs_type == H5F_FSPACE_STRATEGY_PAGE) ? &fs_state:&fs_state_zero)) + TEST_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* section #1 is less than threshold but is shrunk away */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + /* For paged aggregation, the section in the page at EOF for small meta fs is not shrunk away */ + if(fs_type == H5F_FSPACE_STRATEGY_PAGE) + fs_state.tot_space += TBLOCK_SIZE1; - /* MULTI */ - HDputs("Testing file space managers with multi driver"); + /* For old format: the sections at EOF are shrunk away */ + if(check_stats(f, f->shared->fs_man[tt], (fs_type == H5F_FSPACE_STRATEGY_PAGE) ? &fs_state : &fs_state_zero)) + TEST_ERROR - MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - TEST_ERROR - if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl, memb_name, memb_addr, TRUE) < 0) - TEST_ERROR; + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Free-space manager should be empty */ + if(!(fs_type == H5F_FSPACE_STRATEGY_PAGE && fs_persist)) + if(H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + /* Closing */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - } /* end for new_format */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + } /* end for fs_type */ + } /* end for fs_persist */ - if (H5Pclose(fapl2) < 0) + if(H5Pclose(fapl2) < 0) FAIL_STACK_ERROR - return(ret); + PASSED() + + return(0); error: H5E_BEGIN_TRY { - H5Pclose(fapl_new); + H5Pclose(fcpl); H5Pclose(fapl2); + H5Fclose(file); } H5E_END_TRY; return(1); -} /* test_filespace_drivers() */ +} /* test_mf_strat_thres_gone() */ /* + *------------------------------------------------------------------------- * To verify that file space is allocated from the corresponding free-space manager * because H5FD_FLMAP_DICHOTOMY is used as the default free-list mapping. * @@ -7530,80 +7466,1388 @@ error: * * (9) Allocate the third block (size 30) of type H5FD_MEM_SUPER * (10) Verify that this third block is not freed block from (8) + *------------------------------------------------------------------------- */ static unsigned -test_dichotomy(const char *env_h5_drvr, hid_t fapl) +test_dichotomy(hid_t fapl) { hid_t file = -1; /* File ID */ char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ H5FD_mem_t type, stype; haddr_t addr1, addr3, saddr1, saddr2; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("Allocation from raw or metadata free-space manager"); - /* Skip test when using VFDs that don't use the metadata aggregator */ + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Allocate the first block of type H5FD_MEM_SUPER */ + type = H5FD_MEM_SUPER; + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Allocate the second block of type H5FD_MEM_SUPER */ + H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Allocate the first block of type H5FD_MEM_DRAW */ + stype = H5FD_MEM_DRAW; + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Free the first block of type H5FD_MEM_SUPER */ + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + + /* Allocate the second block of type H5FD_MEM_DRAW */ + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Verify that saddr1 is not addr1 */ + if(saddr2 == addr1) TEST_ERROR + + /* Free the first block of type H5FD_MEM_DRAW */ + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + + /* Allocate the third block of type H5FD_MEM_SUPER */ + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Verify that addr3 is not saddr1 */ + if(addr3 == saddr1) TEST_ERROR + + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + PASSED() + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(file); + } H5E_END_TRY; + return(1); +} /* test_dichotomy() */ + +/* + *------------------------------------------------------------------------- + * set_multi_split(): + * Internal routine to set up page-aligned address space for multi/split driver + * when testing paged aggregation. + *------------------------------------------------------------------------- + */ +static int +set_multi_split(hid_t fapl, hsize_t pagesize, hbool_t multi, hbool_t split) +{ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + HDassert(multi || split); + + HDmemset(memb_name, 0, sizeof memb_name); + + /* Get current split settings */ + if(H5Pget_fapl_multi(fapl, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + if(split) { + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + pagesize - 1) / pagesize) * pagesize; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + pagesize - 1) / pagesize) * pagesize; + } else { + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + memb_addr[mt] = ((memb_addr[mt] + pagesize - 1) / pagesize) * pagesize; + } /* end else */ + + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); + + return 0; + +error: + return(-1); +} /* set_multi_split() */ + +/*------------------------------------------------------------------------- + * Function: test_page_alloc_xfree + * + * Purpose: To verify allocations and de-allocations for large/small + * sections are done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t fapl_new = -1; /* File access property list ID */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr2, addr3; /* Addresses for small meta data blocks */ + haddr_t saddr1; /* Addresses for small raw data blocks */ + haddr_t gaddr1; /* Addresses for large data blocks */ + hbool_t split = FALSE, multi = FALSE; + char filename[FILENAME_LEN]; /* Filename to use */ + haddr_t found_addr; /* Address of the found section */ + unsigned fs_persist; /* To persist free-space or not */ + + TESTING("Paged aggregation for file space: H5MF_alloc/H5MF_xfree"); + + /* Check for split or multi driver */ + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + + if(!multi && !split) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + if((fapl_new = H5Pcopy(fapl)) < 0) TEST_ERROR + + if(multi || split) + if(set_multi_split(fapl_new, 4096, multi, split) < 0) + TEST_ERROR; + + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + H5F_mem_page_t fs_type; + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, fs_persist, (hsize_t)1) < 0) + TEST_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 3 small meta data blocks: addr1, addr2, addr3 */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + addr2 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034); + addr3 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Free the block with addr2 */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE1034); + + if(!fs_persist) { + + H5MF_alloc_to_fs_type(f, H5FD_MEM_OHDR, TBLOCK_SIZE1034, (H5F_mem_page_t *)&fs_type); + + /* Verify that the freed block with addr2 is found from the small meta data manager */ + if(H5MF_find_sect(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034, f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + if(found_addr != addr2) + TEST_ERROR + } /* end if */ + + /* Allocate 2 small raw data blocks: saddr1, saddr2 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034); + + /* Free the block with saddr1 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + + if(!fs_persist) { + /* Verify that the freed block with saddr1 is found from the small raw data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5F_MEM_PAGE_DRAW], &found_addr) < 0) + TEST_ERROR + if(found_addr != saddr1) + TEST_ERROR + } /* end if */ + + /* Allocate 2 large data blocks: gaddr1, gaddr2 */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + + /* Free the block with gaddr1 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE5000); + + if(!fs_persist) { + + H5MF_alloc_to_fs_type(f, H5FD_MEM_DRAW, TBLOCK_SIZE5000, (H5F_mem_page_t *)&fs_type); + + /* Verify that the freed block with gaddr1 is found from the large data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8192, f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + if(found_addr != gaddr1) + TEST_ERROR + } /* end if */ + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + TEST_ERROR + + if(fs_persist) { + /* Re-open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Verify that the large generic manager is there */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_DRAW, TBLOCK_SIZE5000, (H5F_mem_page_t *)&fs_type); + if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Verify that the small meta data manager is there */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_OHDR, f->shared->fs_page_size - 1, (H5F_mem_page_t *)&fs_type); + if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Since we are about to open a self referential free space + * manager prior to the first file space allocation / deallocation + * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid + * assertion failures on the first file space alloc / dealloc. + */ + if(f->shared->first_alloc_dealloc){ + if(SUCCEED!=H5MF_tidy_self_referential_fsm_hack(f,H5AC_ind_read_dxpl_id)) + FAIL_STACK_ERROR + } + + /* Set up to use the small meta data manager */ + if(!(f->shared->fs_man[fs_type])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, fs_type) < 0) + TEST_ERROR + + /* Verify that the freed block with addr2 is found from the small meta data manager */ + if(H5MF_find_sect(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)(f->shared->fs_page_size-(addr3+TBLOCK_SIZE50)), f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + + if(found_addr != (addr3+TBLOCK_SIZE50)) + TEST_ERROR + + /* Verify that the small raw data manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[H5F_MEM_PAGE_DRAW])) + TEST_ERROR + + /* Set up to use the small raw data manager */ + if(!(f->shared->fs_man[H5F_MEM_PAGE_DRAW])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, H5F_MEM_PAGE_DRAW) < 0) + TEST_ERROR + + /* Verify that the freed block with saddr1 is found from the small raw data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5F_MEM_PAGE_DRAW], &found_addr) < 0) + TEST_ERROR + if(found_addr != saddr1) + TEST_ERROR + + H5MF_alloc_to_fs_type(f, H5FD_MEM_DRAW, TBLOCK_SIZE5000, (H5F_mem_page_t *)&fs_type); + + if(!(f->shared->fs_man[fs_type])) + /* Set up to use the large data manager */ + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, fs_type) < 0) + TEST_ERROR + + /* Verify that the freed block with gaddr1 is found from the large data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8192, f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + if(found_addr != gaddr1) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + } /* end if fs_persist */ + } /* end for */ + + if(H5Pclose(fapl_new) < 0) + TEST_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support persisting free-space or paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + H5Pclose(fapl_new); + } H5E_END_TRY; + return(1); + +} /* test_page_alloc_xfree() */ + +/*------------------------------------------------------------------------- + * Function: test_page_try_shrink + * + * Purpose: To verify that shrinking via H5MF_try_shrink() work properly + * when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_try_shrink(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr1; /* Address for small meta data block */ + haddr_t saddr1; /* Address for small raw data block */ + haddr_t gaddr1; /* Address for large data block */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + htri_t status; /* status from shrinking */ + h5_stat_size_t file_size; /* File size */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: H5MF_try_shrink()"); + + /* Current VFD that does not support continuous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi") ); + + if(contig_addr_vfd) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate a small meta data block with addr1 */ + addr1 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Try to shrink the block with addr1 */ + if((status = H5MF_try_shrink(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE50)) < 0) + FAIL_STACK_ERROR + + /* Couldn't shrink due to the section (remaining space in the page) is in the small meta data free-space manager */ + if(status == TRUE) TEST_ERROR + + /* Allocate a small raw data block with saddr1 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Try to shrink the block with saddr1 */ + if((status = H5MF_try_shrink(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE50)) < 0) + FAIL_STACK_ERROR + + /* Couldn't shrink due to the section (remaining space in the page) is in the small raw data free-space manager */ + if(status == TRUE) TEST_ERROR + + /* Allocate a large data block with gaddr1 */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + + /* Try to shrink the block with gaddr1 */ + if((status = H5MF_try_shrink(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE5000)) < 0) + FAIL_STACK_ERROR + + /* Couldn't shrink due to the section (remaining space in the page) is in the large-sized free-space manager */ + if(status == TRUE) TEST_ERROR + + /* Free the block with saddr1--merge to become 1 page, then return to the large manager */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE50); + + /* Merge all 3 sections and shrunk */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE5000); + + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Get the size of the file */ + if((file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + + /* Should be on page boundary */ + if(file_size % TBLOCK_SIZE4096) + TEST_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + } H5E_END_TRY; + return(1); + +} /* test_page_try_shrink() */ + +/*------------------------------------------------------------------------- + * Function: test_page_small_try_extend + * + * Purpose: To verify that extending a small block via H5MF_try_extend() works + * properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_small_try_extend(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr1, addr2, addr3; /* Addresses for small meta data blocks */ + haddr_t saddr1; /* Address for small raw data block */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + htri_t was_extended; /* Whether the block can be extended or not */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: H5MF_try_extend() a small block"); + + /* Current VFD that does not support continuous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi") && HDstrcmp(env_h5_drvr, "family")); + + if(contig_addr_vfd) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate a small meta data block with addr1 */ + addr1 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE98); + + /* Try extending the block with addr1 at EOF not crossing page boundary */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE98, (hsize_t)3100); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Allocate 2 small meta data blocks with addr2 and addr3--will be on another meta data page */ + addr2 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE100); + addr3 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE150); + + /* The block with addr2 should be page aligned */ + /* The block with addr3 resides right next to the block with addr2 */ + if(addr2 % TBLOCK_SIZE4096) + TEST_ERROR + if(addr3 != (addr2 + TBLOCK_SIZE100)) + TEST_ERROR + + /* Free the block with addr2 */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE100); + + /* Try extending the block with addr1 that will cross to the next page where the freed block with addr2 resides */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3198, (hsize_t)TBLOCK_SIZE100); + /* Shouldn't succeed--should not cross page boundary */ + if(was_extended == TRUE) TEST_ERROR + + /* Try extending the block with addr1 into the free-space section that is big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3198, (hsize_t)TBLOCK_SIZE50); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Free the block with addr1 */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE3248); + + /* Allocate a new meta data block with addr1 */ + /* There is a page end threshold of size H5F_FILE_SPACE_PGEND_META_THRES at the end of the block */ + /* The block is right next to the threshold */ + addr1 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3286); + + /* Try extending the block into the threshold with size > H5F_FILE_SPACE_PGEND_META_THRES */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3286, (hsize_t)TBLOCK_SIZE11); + /* Shouldn't succeed */ + if(was_extended == TRUE) TEST_ERROR + + /* Try extending the block into the threshold with size < H5F_FILE_SPACE_PGEND_META_THRES */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3286, (hsize_t)TBLOCK_SIZE2); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Free the block with addr3--will merge with the remaining sections to become a page and then free the page */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE150); + + /* Allocate a small raw data block with saddr1 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4086); + + /* Try extending the block crossing the page boundary */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4086, (hsize_t)TBLOCK_SIZE11); + /* Shouldn't succeed */ + if(was_extended == TRUE) TEST_ERROR + + /* Try extending the block not crossing page boundary */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4086, (hsize_t)TBLOCK_SIZE10); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* The extended block is now "large" in size */ + /* Try extending the block */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4096, (hsize_t)TBLOCK_SIZE10); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Try extending the large-sized block */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4106, (hsize_t)TBLOCK_SIZE5000); + /* Should not succeed because the mis-aligned fragment in the page is in the large-sized free-space manager */ + if(was_extended == TRUE) TEST_ERROR + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + } H5E_END_TRY; + return(1); + +} /* test_page_small_try_extend() */ + +/*------------------------------------------------------------------------- + * Function: test_page_large_try_extend + * + * Purpose: To verify that extending a large block via H5MF_try_extend() + * is done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_large_try_extend(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t gaddr1, gaddr2, gaddr3, gaddr4; /* Addresses for large data blocks */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + htri_t was_extended; /* Whether the block can be extended or not */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: H5MF_try_extend() a large block"); + + /* Current VFD that does not support continuous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); + if(contig_addr_vfd) { + /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(NULL == (f = (H5F_t *)H5I_object(fid))) FAIL_STACK_ERROR - /* Allocate the first block of type H5FD_MEM_SUPER */ - type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Allocate a large data block with gaddr1 */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)6000); - /* Allocate the second block of type H5FD_MEM_SUPER */ - H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + /* Should be page aligned */ + if(gaddr1 % TBLOCK_SIZE4096) + TEST_ERROR - /* Allocate the first block of type H5FD_MEM_DRAW */ - stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Extending the block with gaddr1 at EOF to become 2 pages */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr1, (hsize_t)TBLOCK_SIZE6000, (hsize_t)TBLOCK_SIZE2192); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Allocate a large data block with gaddr2 */ + gaddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + /* Should be page aligned */ + if(gaddr2 % TBLOCK_SIZE4096) + TEST_ERROR - /* Free the first block of type H5FD_MEM_SUPER */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); + /* Try extending the block with gaddr1 */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr1, (hsize_t)TBLOCK_SIZE8192, (hsize_t)TBLOCK_SIZE50); + /* Should not succeed */ + if(was_extended == TRUE) TEST_ERROR - /* Allocate the second block of type H5FD_MEM_DRAW */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Allocate a large data block with gaddr3 */ + gaddr3 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + /* Should be page aligned */ + if(gaddr3 % TBLOCK_SIZE4096) + TEST_ERROR - /* Verify that saddr1 is not addr1 */ - if(saddr2 == addr1) TEST_ERROR + /* Try extending the block with gaddr2--there is a free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8000, (hsize_t)TBLOCK_SIZE100); + /* Should succeed */ + if(was_extended == FALSE) TEST_ERROR - /* Free the first block of type H5FD_MEM_DRAW */ - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + /* Try extending the block with gaddr2--there is no free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8100, (hsize_t)TBLOCK_SIZE100); + /* Should not succeed */ + if(was_extended == TRUE) TEST_ERROR - /* Allocate the third block of type H5FD_MEM_SUPER */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Try extending the block with gaddr2--there is a free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8100, (hsize_t)TBLOCK_SIZE90); + /* Should succeed */ + if(was_extended == FALSE) TEST_ERROR - /* Verify that addr3 is not saddr1 */ - if(addr3 == saddr1) TEST_ERROR + /* Try extending the block with gaddr2 */ + /* There is no free-space section big enough to fulfill the request (request is < H5F_FILE_SPACE_PGEND_META_THRES) */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8190, (hsize_t)TBLOCK_SIZE5); + /* Should not succeed */ + if(was_extended == TRUE) TEST_ERROR - if(H5Fclose(file) < 0) + /* Allocate a large data block with gaddr4 */ + gaddr4 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + /* Should be page aligned */ + if(gaddr4 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Free the block with gaddr3--will merge with remaining free space to become 2 pages + section (size 2) in previous page */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr3, (hsize_t)TBLOCK_SIZE8000); + + /* Try extending the block with gaddr2 crossing page boundary--there is free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8190, (hsize_t)TBLOCK_SIZE5); + /* Should succeed */ + if(was_extended == FALSE) TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) FAIL_STACK_ERROR PASSED() - } /* end if */ - else { - SKIPPED(); - puts(" Current VFD doesn't support metadata aggregator"); - } /* end else */ + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation strategy"); + } return(0); error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(fid); + H5Pclose(fcpl); } H5E_END_TRY; return(1); -} /* test_dichotomy() */ + +} /* test_page_large_try_extend() */ + +/*------------------------------------------------------------------------- + * Function: test_page_large + * + * Purpose: To verify that allocations and de-allocations for large data + * are done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_large(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t gaddr1, gaddr2, gaddr3, gaddr4; /* Addresses for blocks */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + h5_stat_size_t file_size; /* File size */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: large allocations and de-allocations"); + + /* Current VFD that does not support continuous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); + + if(contig_addr_vfd) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate a large data block with gaddr1 */ + /* 1 page + 1904 bytes; 2192 bytes in free-space manager */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6000); + + /* Should be page aligned */ + if(gaddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a large data block with gaddr2--should be on another page */ + /* Allocate 1 page + 3904 bytes; 192 bytes in free-space manager */ + gaddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + /* Should be page aligned */ + if(gaddr2 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a large data block with gaddr3--should be on another page */ + /* Allocate 2 pages + 3808 bytes; 288 bytes in free-space manager */ + gaddr3 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE12000); + if(!H5F_addr_defined(gaddr3)) + TEST_ERROR + + /* Free the block with gaddr2 */ + /* Merged sections: 2192 + 8000 + 192 = 10384 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr2, (hsize_t)TBLOCK_SIZE8000); + + /* Get free-space info */ + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + + /* Verify that the manager contains 2 free-space sections: 10384 and 288 */ + if(fs_stat.tot_sect_count != 2) + TEST_ERROR + if(fs_stat.tot_space != (10384+288)) + TEST_ERROR + + /* Allocate a large data block with gaddr4--there is a free-space section able to fulfill the request */ + /* Free-space sections: 2192 + 3192 + 288 = 5672 bytes */ + gaddr4 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + + /* Should be page aligned */ + if(gaddr4 % TBLOCK_SIZE4096) + TEST_ERROR + if(gaddr4 != gaddr2) + TEST_ERROR + + /* Get free-space info */ + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + /* Verify that that there are 3 free-space sections */ + if(fs_stat.tot_sect_count != 3) + TEST_ERROR + if(fs_stat.tot_space != (2192+3192+288)) + TEST_ERROR + + /* Free the two blocks with gaddr1 and gaddr4 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE6000); + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr4, (hsize_t)TBLOCK_SIZE5000); + + /* Get free-space info */ + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + /* Verify that that there are 2 free-space sections: 16384 (4 pages) + 288 */ + if(fs_stat.tot_sect_count != 2) + TEST_ERROR + if(fs_stat.tot_space != (16384+288)) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + /* Get the size of the file */ + if((file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + + /* Verify that file size end on a page boundary */ + if(file_size % TBLOCK_SIZE4096) + TEST_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + } H5E_END_TRY; + return(1); + +} /* test_page_large() */ + +/*------------------------------------------------------------------------- + * Function: test_page_small + * + * Purpose: To verify allocations and de-allocations for small meta/raw data + * are done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_small(const char *env_h5_drvr, hid_t fapl) +{ + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr2, addr3, addr4, addr5; /* Addresses for blocks */ + haddr_t addr9, addr10, addr11; /* Address for small meta data blocks */ + haddr_t saddr1, saddr2; /* Addresses for small raw data blocks */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + char filename[FILENAME_LEN]; /* Filename to use */ + hbool_t multi= FALSE, split = FALSE, family = FALSE; + + TESTING("Paged aggregation for file space: small allocations and de-allocations"); + + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + else if(!HDstrcmp(env_h5_drvr, "family")) + family = TRUE; + + if(!multi && !split) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate 2 small meta data blocks: addr1, addr2 */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + addr2 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Allocate a small raw data block with saddr1 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + /* Should be on the second page and page aligned */ + if(saddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a small raw data block with saddr2 */ + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + /* Should not be page aligned */ + if(!(saddr2 % TBLOCK_SIZE4096)) + TEST_ERROR + /* Should be next to the block with saddr1 */ + if(saddr2 != (saddr1 + TBLOCK_SIZE30)) + TEST_ERROR + + /* Allocate a small meta data block with addr3--there is no free-space section big enough to fulfill the request */ + addr3 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4020); + /* Should be on the third page and page aligned */ + if(addr3 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a small meta data block with addr4--there is a free-space section big enough to fulfill the request */ + addr4 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + /* Should not be page aligned */ + if(!(addr4 % TBLOCK_SIZE4096)) + TEST_ERROR + /* Should be next to the block with addr2 */ + if(addr4 != (addr2 + TBLOCK_SIZE50)) + TEST_ERROR + + /* Allocate a small meta data block with addr5--there is a free-space section big enough to fulfill the request */ + addr5 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE40); + /* Should not be page aligned */ + if(!(addr5 % TBLOCK_SIZE4096)) + TEST_ERROR + + /* Should be next to the block with addr3 */ + if(addr5 != (addr3 + TBLOCK_SIZE4020)) + TEST_ERROR + + /* Allocate a small meta data block with addr6--taking up the remaining space in the first page */ + if(family) + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3080); + else + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3088); + + /* Allocate a small meta data block with addr7--taking up the remaining space in the third page */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE36); + + /* Allocate 2 small meta data blocks: addr8, addr9--there is no free-space to fulfill the request */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + addr9 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + + /* Free the block with saddr1 and saddr2--merge with remaining section to become a page which will be returned to the large manager */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)TBLOCK_SIZE50); + + /* Verify that the large manager does contain a section with file space page size (default is 4096) */ + if(!f->shared->fs_man[H5F_MEM_PAGE_GENERIC]) + TEST_ERROR + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + if(fs_stat.tot_space != TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a small meta data block with addr10--there is a free-space section big enough to fulfill the request */ + addr10 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3900); + /* The block should be next to the block with addr9 */ + if(addr10 != (addr9 + TBLOCK_SIZE80)) + TEST_ERROR + + /* Allocate a small meta data block with addr11 */ + /* The current free-space section is unable to fulfill the request; obtain a page from the large manager */ + addr11 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + /* The address of the block should be the same the freed block with saddr1 */ + if(addr11 != saddr1) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + } H5E_END_TRY; + return(1); + +} /* test_page_small() */ + +/*------------------------------------------------------------------------- + * Function: test_page_alignment + * + * Purpose: To verify the proper alignment is used when H5Pset_alignment() + * is set and paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_alignment(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fcpl2 = -1; /* File creation property list ID */ + hid_t fapl_new = -1; /* File access property list ID */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr1, addr2; /* Addresses for small meta data blocks */ + haddr_t saddr1, saddr2; /* Addresses for small raw data blocks */ + haddr_t gaddr1, gaddr2; /* Addresses for blocks */ + char filename[FILENAME_LEN]; /* Filename to use */ + hbool_t split = FALSE, multi = FALSE; + + TESTING("Paged aggregation and H5Pset_alignment: verify proper alignment is used"); + + /* Check for split or multi driver */ + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + + if(!multi && !split) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* + * Case 1: Verify that the alignment in use is the default file space + * page size when paged aggregation is enabled. + */ + + if((fapl_new = H5Pcopy(fapl)) < 0) TEST_ERROR + + /* The alignment to use will be the library's default file space page size */ + if(H5Pset_libver_bounds(fapl_new, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + /* Set alignment value to 16 */ + if(H5Pset_alignment(fapl_new, (hsize_t)0, (hsize_t)TEST_ALIGN16) < 0) + TEST_ERROR + + if(split || multi) { + hid_t memb_fapl; + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + /* Create fapl */ + if((memb_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR + + /* Set alignment */ + if(H5Pset_alignment(memb_fapl, 0, (hsize_t)TEST_ALIGN16) < 0) + TEST_ERROR + + HDmemset(memb_name, 0, sizeof memb_name); + + if(split) { + /* Set split driver with new FAPLs */ + if(H5Pset_fapl_split(fapl_new, "-m.h5", memb_fapl, "-r.h5", memb_fapl) < 0) + TEST_ERROR + + /* Get current multi settings */ + if(H5Pget_fapl_multi(fapl_new, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + TBLOCK_SIZE4096 - 1) / TBLOCK_SIZE4096) * TBLOCK_SIZE4096; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + TBLOCK_SIZE4096 - 1) / TBLOCK_SIZE4096) * TBLOCK_SIZE4096; + + /* Set split driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + } else { + /* Get current multi settings */ + if(H5Pget_fapl_multi(fapl_new, memb_map, NULL, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + /* Populate memb_fapl_arr */ + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { + memb_fapl_arr[mt] = memb_fapl; + memb_addr[mt] = ((memb_addr[mt] + TBLOCK_SIZE4096 - 1) / TBLOCK_SIZE4096) * TBLOCK_SIZE4096; + } + + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + } /* end else */ + + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); + + /* Close memb_fapl */ + if(H5Pclose(memb_fapl) < 0) + TEST_ERROR + } /* end if */ + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set the strategy to paged aggregation and persisting free space */ + /* The alignment to use will be the library's default file space page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + TEST_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 2 small raw data blocks */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Should be on the second page and page aligned on 4096 (default file space page size) */ + if(saddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Should be next to the block with saddr1 */ + if(saddr2 != (saddr1 + TBLOCK_SIZE30)) + TEST_ERROR + + /* Allocate 2 large raw data blocks */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + gaddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + + /* Should be on the 3rd page and page aligned */ + if(gaddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Should be on the 4th page and page aligned */ + if(gaddr2 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Close the file creation property list */ + if(H5Pclose(fcpl) < 0) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + + /* + * Case 2: Verify that the alignment in use is the alignment set + * via H5Pset_alignment when paged aggregation not enabled. + */ + /* fapl_new has latest format and H5Pset_alignment set */ + /* Disable small data block mechanism */ + if(H5Pset_small_data_block_size(fapl_new, (hsize_t)0) < 0) + TEST_ERROR + /* Disable meta data block mechanism */ + if(H5Pset_meta_block_size(fapl_new, (hsize_t)0) < 0) + TEST_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 2 small meta data blocks */ + addr1 = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + addr2 = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Should be aligned on 16 */ + if(addr1 % TEST_ALIGN16 || addr2 % TEST_ALIGN16) + TEST_ERROR + + /* addr2 should be right next to the block with addr1 */ + if((addr1 + TBLOCK_SIZE30) % TEST_ALIGN16) + if(addr2 != (((addr1 + TBLOCK_SIZE30) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* Allocate 2 small raw data blocks */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE100); + + /* Should be aligned on 16 */ + if(saddr1 % TEST_ALIGN16 || saddr2 % TEST_ALIGN16) + TEST_ERROR + + if(!multi && !split) { + /* saddr1 should be right next to the block with addr2 */ + if((addr2 + TBLOCK_SIZE50) % TEST_ALIGN16) + if(saddr1 != (((addr2 + TBLOCK_SIZE50) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + } + + /* saddr2 should be right next to the block with saddr1 */ + if((saddr1 + TBLOCK_SIZE80) % TEST_ALIGN16) + if(saddr2 != (((saddr1 + TBLOCK_SIZE80) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + + /* + * Case 3: Verify that the alignment in use is the alignment set + * via H5Pset_alignment when paged aggregation not enabled. + */ + /* File creation property list */ + if((fcpl2 = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set file space page size */ + if(H5Pset_file_space_page_size(fcpl2, (hsize_t)TBLOCK_SIZE8192) < 0) + TEST_ERROR + + /* Set strategy to H5F_FSPACE_STRATEGY_AGGR but meta/raw data block is 0 as set in fapl_new */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_AGGR, FALSE, (hsize_t)1) < 0) + TEST_ERROR + + /* fapl_new has latest format, H5Pset_alignment set, and disable meta/raw block */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl2, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 2 small raw data blocks */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Should be aligned on 16 */ + if(saddr1 % TEST_ALIGN16) + TEST_ERROR + if(saddr2 % TEST_ALIGN16) + TEST_ERROR + + /* saddr2 should be right next to the block with saddr1 */ + if((saddr1 + TBLOCK_SIZE30) % TEST_ALIGN16) + if(saddr2 != (((saddr1 + TBLOCK_SIZE30) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* Allocate a large raw data block */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + + /* Should be aligned on 16 */ + if(gaddr1 % TEST_ALIGN16) + TEST_ERROR + + /* gaddr1 is right next to the block with saddr2 */ + if((saddr2 + TBLOCK_SIZE50) % TEST_ALIGN16) + if(gaddr1 != (((saddr2 + TBLOCK_SIZE50) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* There is no free-space manager involved for H5F_FSPACE_STRATEGY_AGGR strategy */ + if(f->shared->fs_man[H5FD_MEM_DRAW] || f->shared->fs_man[H5FD_MEM_SUPER]) + TEST_ERROR + + /* Closing */ + if(H5Fclose(fid) < 0) + TEST_ERROR + if(H5Pclose(fcpl2) < 0) + TEST_ERROR + if(H5Pclose(fapl_new) < 0) + TEST_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support persisting free-space or paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + H5Pclose(fapl_new); + } H5E_END_TRY; + return(1); + +} /* test_page_alignment() */ int main(void) @@ -7625,33 +8869,21 @@ main(void) /* Make a copy of the FAPL before adjusting the alignment */ if((new_fapl = H5Pcopy(fapl)) < 0) TEST_ERROR - - /* alignment is not set for the following tests */ - if(H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1) < 0) - TEST_ERROR - - /* meta/small data is set to 2048 for the following tests */ - if(H5Pset_meta_block_size(fapl, (hsize_t)TEST_BLOCK_SIZE2048) < 0) - TEST_ERROR - if(H5Pset_small_data_block_size(fapl, (hsize_t)TEST_BLOCK_SIZE2048) < 0) - TEST_ERROR - - /* interaction with file allocation */ + /* For old library format--interaction with file allocation */ nerrors += test_mf_eoa(env_h5_drvr, fapl); nerrors += test_mf_eoa_shrink(env_h5_drvr, fapl); nerrors += test_mf_eoa_extend(env_h5_drvr, fapl); - /* interaction with temporary file space allocation */ - nerrors += test_mf_tmp(env_h5_drvr, fapl); + /* For old library format */ + nerrors += test_dichotomy(new_fapl); - /* interaction with free-space manager */ + /* For old library format--interaction with free-space manager */ nerrors += test_mf_fs_start(fapl); nerrors += test_mf_fs_alloc_free(fapl); nerrors += test_mf_fs_extend(fapl); nerrors += test_mf_fs_absorb(env_h5_drvr, fapl); - nerrors += test_dichotomy(env_h5_drvr, new_fapl); - /* interaction with meta/sdata aggregator */ + /* For old library format--interaction with meta/sdata aggregator */ nerrors += test_mf_aggr_alloc1(env_h5_drvr, fapl); nerrors += test_mf_aggr_alloc2(env_h5_drvr, fapl); nerrors += test_mf_aggr_alloc3(env_h5_drvr, fapl); @@ -7662,7 +8894,7 @@ main(void) nerrors += test_mf_aggr_extend(env_h5_drvr, fapl); nerrors += test_mf_aggr_absorb(env_h5_drvr, fapl); - /* Tests for alignment */ + /* For old library format--tests for alignment */ for(curr_test = TEST_NORMAL; curr_test < TEST_NTESTS; H5_INC_ENUM(test_type_t, curr_test)) { switch(curr_test) { @@ -7692,11 +8924,54 @@ main(void) nerrors += test_mf_align_alloc6(env_h5_drvr, fapl, new_fapl); } /* end for */ - /* tests to verify that file's free-space managers are persistent */ - nerrors += test_mf_fs_drivers(fapl); + /* For old and new format--interaction with temporary file space allocation */ + nerrors += test_mf_tmp(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_tmp(env_h5_drvr, fapl, TRUE); + + /* For old and new format--free-space merge/shrunk away */ + + /* Temporary: modify to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_fs_gone(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_fs_gone(env_h5_drvr, fapl, TRUE); + + /* Temporary: modify to skip testing multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_strat_thres_gone(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_strat_thres_gone(env_h5_drvr, fapl, TRUE); - /* tests for file space management */ - nerrors += test_filespace_drivers(fapl); + /* For old and new format--persisting free-space */ + + /* Temporary: Modify to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_fs_persist(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_fs_persist(env_h5_drvr, fapl, TRUE); + + /* Temporary: modify to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_strat_thres_persist(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_strat_thres_persist(env_h5_drvr, fapl, TRUE); + + /* Temporary skipped for multi/split drivers: + fail file create when persisting free-space or using paged aggregation strategy */ +#ifdef PB_OUT + /* Tests specific for multi and split files--persisting free-space */ + nerrors += test_mf_fs_persist_split(); + nerrors += test_mf_fs_persist_multi(); +#endif + + /* + * Tests specific for file space paging + */ + /* Temporary: The following 7 tests are modified to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_page_small(env_h5_drvr, fapl); + nerrors += test_page_large(env_h5_drvr, fapl); + nerrors += test_page_large_try_extend(env_h5_drvr, fapl); + nerrors += test_page_small_try_extend(env_h5_drvr, fapl); + nerrors += test_page_try_shrink(env_h5_drvr, fapl); + nerrors += test_page_alloc_xfree(env_h5_drvr, fapl); /* can handle multi/split */ + nerrors += test_page_alignment(env_h5_drvr, fapl); /* can handle multi/split */ /* tests for specific bugs */ nerrors += test_mf_bug1(env_h5_drvr, fapl); diff --git a/test/none.h5 b/test/none.h5 Binary files differnew file mode 100644 index 0000000..bb6ff56 --- /dev/null +++ b/test/none.h5 diff --git a/test/page_buffer.c b/test/page_buffer.c new file mode 100644 index 0000000..b94ad03 --- /dev/null +++ b/test/page_buffer.c @@ -0,0 +1,2087 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*********************************************************** +* +* Test program: cache_page_buffer +* +* Tests the Page Buffer Feature. +* +*************************************************************/ + +#include "h5test.h" + +#include "H5PBprivate.h" +#include "H5Iprivate.h" + +/* + * This file needs to access private information from the H5F package. + */ +#define H5MF_FRIEND /*suppress error about including H5MFpkg */ +#include "H5MFpkg.h" + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" + + +#define FILENAME_LEN 1024 +#define NUM_DSETS 5 +#define NX 100 +#define NY 50 + +/* helper routines */ +static unsigned create_file(char *filename, hid_t fcpl, hid_t fapl); +static unsigned open_file(char *filename, hid_t fapl, hsize_t page_size, size_t page_buffer_size); + +/* test routines */ +static unsigned test_args(hid_t fapl, const char *env_h5_drvr); +static unsigned test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr); +static unsigned test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr); +static unsigned test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr); +static unsigned test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr); + +const char *FILENAME[] = { + "filepaged", + NULL +}; + + +/*------------------------------------------------------------------------- + * Function: create_file() + * + * Purpose: The purpose of this function appears to be a smoke check + * intended to exercise the page buffer. + * + * Specifically, the function creates a file, and then goes + * through a loop in which it creates four data sets, write + * data to one of them, verifies the data written, and then + * deletes the three that it didn't write to. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +static unsigned +create_file(char *filename, hid_t fcpl, hid_t fapl) +{ + hid_t file_id = -1; + hid_t dset_id = -1; + hid_t grp_id = -1; + hid_t filespace = -1; + hsize_t dimsf[2] = {NX, NY}; /* dataset dimensions */ + int *data = NULL; /* pointer to data buffer to write */ + hid_t dcpl = -1; + int i; + int num_elements; + int j; + char dset_name[10]; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + if((grp_id = H5Gcreate2(file_id, "GROUP", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + num_elements = NX * NY; + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR + for (i=0; i < (int)num_elements; i++) + data[i] = i; + + if((filespace = H5Screate_simple(2, dimsf, NULL)) < 0) + FAIL_STACK_ERROR; + + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + FAIL_STACK_ERROR; + + for(i=0 ; i<NUM_DSETS; i++) { + + HDsprintf(dset_name, "D1dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDsprintf(dset_name, "D2dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDsprintf(dset_name, "D3dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDsprintf(dset_name, "dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if(H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDmemset(data, 0, (size_t)num_elements * sizeof(int)); + if((dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + for (j=0; j < num_elements; j++) { + if(data[j] != j) { + HDfprintf(stderr, "Read different values than written\n"); + FAIL_STACK_ERROR; + } + } + + HDsprintf(dset_name, "D1dset%d", i); + if(H5Ldelete(grp_id, dset_name, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + HDsprintf(dset_name, "D2dset%d", i); + if(H5Ldelete(grp_id, dset_name, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + HDsprintf(dset_name, "D3dset%d", i); + if(H5Ldelete(grp_id, dset_name, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + } + + if(H5Gclose(grp_id) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(filespace) < 0) + FAIL_STACK_ERROR; + + HDfree(data); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(filespace); + H5Gclose(grp_id); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return(1); +} /* create_file */ + + +/*------------------------------------------------------------------------- + * Function: open_file() + * + * Purpose: The purpose of this function appears to be a smoke check + * intended to exercise the page buffer. + * + * Specifically, the function opens a file (created by + * create_file()?), and verify the contents of its datasets. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +static unsigned +open_file(char *filename, hid_t fapl, hsize_t page_size, + size_t page_buffer_size) +{ + hid_t file_id = -1; + hid_t dset_id = -1; + hid_t grp_id = -1; + int *data = NULL; /* pointer to data buffer to write */ + int i; + int j; + int num_elements; + char dset_name[10]; + H5F_t *f = NULL; + + if((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + if(f->shared->page_buf == NULL) + FAIL_STACK_ERROR; + if(f->shared->page_buf->page_size != page_size) + FAIL_STACK_ERROR; + if(f->shared->page_buf->max_size != page_buffer_size) + FAIL_STACK_ERROR; + + if((grp_id = H5Gopen2(file_id, "GROUP", H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + num_elements = NX * NY; + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR + + for(i=0 ; i<NUM_DSETS; i++) { + + HDsprintf(dset_name, "dset%d", i); + if((dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if(H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + FAIL_STACK_ERROR; + + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + for (j=0; j < num_elements; j++) { + if(data[j] != j) { + HDfprintf(stderr, "Read different values than written\n"); + FAIL_STACK_ERROR; + } + } + } + + if(H5Gclose(grp_id) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Gclose(grp_id); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return 1; +} + +/* + * + * set_multi_split(): + * Internal routine to set up page-aligned address space for multi/split driver + * when testing paged aggregation. + * + */ +static unsigned +set_multi_split(const char *env_h5_drvr, hid_t fapl, hsize_t pagesize) +{ + hbool_t split = FALSE; + hbool_t multi = FALSE; + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + /* Check for split or multi driver */ + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + + if(split || multi) { + + HDmemset(memb_name, 0, sizeof memb_name); + + /* Get current split settings */ + if(H5Pget_fapl_multi(fapl, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + if(split) { + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + pagesize - 1) / pagesize) * pagesize; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + pagesize - 1) / pagesize) * pagesize; + } else { + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + memb_addr[mt] = ((memb_addr[mt] + pagesize - 1) / pagesize) * pagesize; + } /* end else */ + + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); + + } /* end if */ + + return 0; + +error: + return 1; + +} /* set_multi_split() */ + + +/*------------------------------------------------------------------------- + * Function: test_args() + * + * Purpose: This test appears to be a quick smoke check directed at: + * + * 1) verifying that API errors are caught. + * + * 2) verifying that the page buffer behaves more or less + * as advertized. + * + * Any data mis-matches or unexpected failures or successes + * reported by the HDF5 library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +static unsigned +test_args(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + herr_t ret; + + TESTING("Settings for Page Buffering"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) TEST_ERROR + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + + + /* Test setting a page buffer without Paged Aggregation enabled - + * should fail + */ + if(H5Pset_page_buffer_size(fapl, 512, 0, 0) < 0) + TEST_ERROR; + + H5E_BEGIN_TRY { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + + if(file_id >= 0) + TEST_ERROR; + + + /* Test setting a page buffer with a size smaller than a single + * page size - should fail + */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 511, 0, 0) < 0) + TEST_ERROR; + + H5E_BEGIN_TRY { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + + if(file_id >= 0) + TEST_ERROR; + + + /* Test setting a page buffer with sum of min meta and raw + * data percentage > 100 - should fail + */ + H5E_BEGIN_TRY { + ret = H5Pset_page_buffer_size(fapl, 512, 50, 51); + } H5E_END_TRY; + + if(ret >= 0) + TEST_ERROR; + + if(set_multi_split(env_h5_drvr, fapl, 512) != 0) + TEST_ERROR; + + /* Test setting a page buffer with a size equal to a single page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 512, 0, 0) < 0) + TEST_ERROR; + + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + + if(open_file(filename, fapl, 512, 512) != 0) + TEST_ERROR; + + + /* Test setting a page buffer with a size slightly larger than a + * single page size + */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 513, 0, 0) < 0) + TEST_ERROR; + + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + + if(open_file(filename, fapl, 512, 512) != 0) + TEST_ERROR; + + if(set_multi_split(env_h5_drvr, fapl, 4194304) != 0) + TEST_ERROR; + + + /* Test setting a large page buffer size and page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 4194304) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 16777216, 0, 0) < 0) + TEST_ERROR; + + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + + if(open_file(filename, fapl, 4194304, 16777216) != 0) + TEST_ERROR; + + if(set_multi_split(env_h5_drvr, fapl, 1) != 0) + TEST_ERROR; + + + /* Test setting a 512 byte page buffer size and page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + if(H5Pset_page_buffer_size(fapl, 512, 0, 0) < 0) + TEST_ERROR; + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + if(open_file(filename, fapl, 512, 512) != 0) + TEST_ERROR; + + + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + } H5E_END_TRY; + return 1; +} /* test_args */ + + +/*------------------------------------------------------------------------- + * Function: test_raw_data_handling() + * + * Purpose: The purpose of this function appears to be a smoke check + * of raw data reads and writes via the page buffer. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added base_page_cnt field as supporting code. This allows + * the test to adjust to the number of page buffer pages + * accessed during file open / create. + * + * The test for the value of base_page_cnt just after file + * open exists detect changes in library behavior. Assuming + * any such change is not indicative of other issues, these + * tests can be modified to reflect the change. + * + * JRM -- 2/23/17 + * + *------------------------------------------------------------------------- + */ + +/* Changes due to file space page size has a minimum size of 512 */ +static unsigned +test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + H5FD_io_info_t fdio_info; + H5P_genplist_t *meta_plist = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id); + H5P_genplist_t *raw_plist = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id); + size_t base_page_cnt; + size_t page_count = 0; + int i, num_elements = 2000; + haddr_t addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("Raw Data Handling"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + TEST_ERROR; + if(H5Pset_page_buffer_size(fapl, sizeof(int)*2000, 0, 0) < 0) + TEST_ERROR; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the number of pages inserted, and verify that it is the + * the expected value. + */ + base_page_cnt = H5SL_count(f->shared->page_buf->slist_ptr); + if(base_page_cnt != 1) + TEST_ERROR; + + /* allocate space for a 2000 elements */ + if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* update the first 50 elements to have values 0-49 - this will be + a page buffer update with 1 page resulting in the page + buffer. */ + /* Changes: 100 */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count ++; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* update elements 300 - 450, with values 300 - - this will + bring two more pages into the page buffer. */ + for(i=0 ; i<150 ; i++) + data[i] = i+300; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*150, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + page_count += 2; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* update elements 100 - 300, this will go to disk but also update + existing pages in the page buffer. */ + for(i=0 ; i<200 ; i++) + data[i] = i+100; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*100), sizeof(int)*200, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* Update elements 225-300 - this will update an existing page in the PB */ + /* Changes: 450 - 600; 150 */ + for(i=0 ; i<150 ; i++) + data[i] = i+450; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*450), sizeof(int)*150, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* Do a full page write to block 600-800 - should bypass the PB */ + for(i=0 ; i<200 ; i++) + data[i] = i+600; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*600), sizeof(int)*200, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* read elements 800 - 1200, this should not affect the PB, and should read -1s */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*800), sizeof(int)*400, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 400; i++) { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + FAIL_STACK_ERROR; + } + } + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* read elements 1200 - 1201, this should read -1 and bring in an + * entire page of addr 1200 + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*1200), sizeof(int)*1, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 1; i++) { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + page_count ++; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* read elements 175 - 225, this should use the PB existing pages */ + /* Changes: 350 - 450 */ + /* read elements 175 - 225, this should use the PB existing pages */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 100; i++) { + if(data[i] != i+350) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* read elements 0 - 800 using the VFD.. this should result in -1s + except for the writes that went through the PB (100-300 & 600-800) */ + fdio_info.file = f->shared->lf; + fdio_info.meta_dxpl = meta_plist; + fdio_info.raw_dxpl = raw_plist; + if(H5FD_read(&fdio_info, H5FD_MEM_DRAW, addr, sizeof(int)*800, data) < 0) + FAIL_STACK_ERROR; + i = 0; + while (i < 800) { + if((i>=100 && i<300) || (i>=600)) { + if(data[i] != i) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + else { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + i++; + } + + /* read elements 0 - 800 using the PB.. this should result in all + * what we have written so far and should get the updates from the PB + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int)*800, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + for (i=0; i < 800; i++) { + if(data[i] != i) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + + /* update elements 400 - 1400 to value 0, this will go to disk but + * also evict existing pages from the PB (page 400 & 1200 that are + * existing). + */ + for(i=0 ; i<1000 ; i++) + data[i] = 0; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*400), sizeof(int)*1000, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + page_count -= 2; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* read elements 0 - 1000.. this should go to disk then update the + * buffer result 200-400 with existing pages + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int)*1000, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + i=0; + while (i < 1000) { + if(i<400) { + if(data[i] != i) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + else { + if(data[i] != 0) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + i++; + } + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return 1; +} /* test_raw_data_handling */ + + +/*------------------------------------------------------------------------- + * Function: test_lru_processing() + * + * Purpose: Basic set of tests verifying expected page buffer LRU + * management. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added base_page_cnt field as supporting code. This allows + * the test to adjust to the number of page buffer pages + * accessed during file open / create. + * + * The test for the value of base_page_cnt just after file + * open exists detect changes in library behavior. Assuming + * any such change is not indicative of other issues, these + * tests can be modified to reflect the change. + * + * JRM -- 2/23/17 + * + * + *------------------------------------------------------------------------- + */ + +static unsigned +test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + size_t base_page_cnt; + size_t page_count = 0; + int i; + int num_elements = 2000; + haddr_t addr = HADDR_UNDEF; + haddr_t search_addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("LRU Processing"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + FAIL_STACK_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + FAIL_STACK_ERROR; + + /* keep 2 pages at max in the page buffer */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*400, 20, 0) < 0) + FAIL_STACK_ERROR; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the number of pages inserted, and verify that it is the + * the expected value. + */ + base_page_cnt = H5SL_count(f->shared->page_buf->slist_ptr); + if(base_page_cnt != 1) + TEST_ERROR; + + /* allocate space for a 2000 elements */ + if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* update the first 100 elements to have values 0-99 - this will be + * a page buffer update with 1 page resulting in the page + * buffer. + */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count ++; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* update elements 300 - 450, with values 300 - 449 - this will + * bring two pages into the page buffer and evict 0. + */ + for(i=0 ; i<150 ; i++) + data[i] = i+300; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*150, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count = 2; + + /* at this point, the page buffer entry created at file open should + * have been evicted -- thus no further need to consider base_page_cnt. + */ + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* The two pages should be the ones with address 100 and 200; 0 + should have been evicted */ + /* Changes: 200, 400 */ + search_addr = addr; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + search_addr = addr + sizeof(int)*400; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* update elements 150-151, this will update existing pages in the + page buffer and move it to the top of the LRU. */ + /* Changes: 300 - 301 */ + for(i=0 ; i<1 ; i++) + data[i] = i+300; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*1, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* read elements 600 - 601, this should read -1 and bring in an + entire page of addr 600, and evict page 200 */ + /* Changes: 1200 - 1201; 1200, 400 */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*1200), sizeof(int)*1, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 1; i++) { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } /* end if */ + } /* end for */ + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* Changes: 400 */ + search_addr = addr + sizeof(int)*400; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 200 */ + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 1200 */ + search_addr = addr + sizeof(int)*1200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + /* read elements 175 - 225, this should move 100 to the top, evict 600 and bring in 200 */ + /* Changes: 350 - 450; 200, 1200, 400 */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 100; i++) { + if(data[i] != i+350) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } /* end if */ + } /* end for */ + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* Changes: 1200 */ + search_addr = addr + sizeof(int)*1200; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 200 */ + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 400 */ + search_addr = addr + sizeof(int)*400; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* update elements 200 - 700 to value 0, this will go to disk but + also discarding existing pages from the PB (page 200). */ + /* Changes: 400 - 1400; 400 */ + for(i=0 ; i<1000 ; i++) + data[i] = 0; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*400), sizeof(int)*1000, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + page_count -= 1; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* Changes: 200 */ + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 400 */ + search_addr = addr + sizeof(int)*400; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return 1; +} /* test_lru_processing */ + + +/*------------------------------------------------------------------------- + * Function: test_min_threshold() + * + * Purpose: Tests verifying observation of minimum and maximum + * raw and metadata page counts in the page buffer. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added the base_raw_cnt and base_meta_cnt fields and + * supporting code. This allows the test to adjust to the + * number of page buffer pages accessed during file open / + * create. + * + * The tests for the values of base_raw_cnt and base_meta_cnt + * just after file open exist detect changes in library + * behavior. Assuming any such change is not indicative of + * other issues, these tests can be modified to reflect the + * change. + * + * JRM -- 2/23/17 + * + *------------------------------------------------------------------------- + */ + +static unsigned +test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + size_t base_raw_cnt = 0; + size_t base_meta_cnt = 0; + size_t page_count = 0; + int i; + int num_elements = 1000; + H5PB_t *page_buf; + haddr_t meta_addr = HADDR_UNDEF; + haddr_t raw_addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("Minimum Metadata threshold Processing"); + HDprintf("\n"); + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + FAIL_STACK_ERROR; + + HDprintf("\tMinimum metadata threshold = 100%%\n"); + + /* keep 5 pages at max in the page buffer and 5 meta page minimum */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 100, 0) < 0) + FAIL_STACK_ERROR; + + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the raw and meta counts now, so we can adjust tests accordingly. + */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->page_buf); + + base_raw_cnt = f->shared->page_buf->raw_count; + base_meta_cnt = f->shared->page_buf->meta_count; + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 5) + TEST_ERROR; + + if(page_buf->min_raw_count != 0) + TEST_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* write all raw data, this would end up in page buffer since there + * is no metadata yet + * + * Not necessarily -- opening the file may may load a metadata page. + */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->raw_count != 5 - base_meta_cnt) + TEST_ERROR; + + /* write all meta data, this would end up in page buffer */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->meta_count != 5) + TEST_ERROR; + + if(page_buf->raw_count != 0) + TEST_ERROR; + + /* write and read more raw data and make sure that they don't end up in + * page buffer since the minimum metadata is actually the entire + * page buffer + */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->meta_count != 5) + TEST_ERROR; + + if(page_buf->raw_count != 0) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + + HDprintf("\tMinimum raw data threshold = 100%%\n"); + + page_count = 0; + + /* keep 5 pages at max in the page buffer and 5 raw page minimum */ + /* Changes: 1000 */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 0, 100) < 0) + TEST_ERROR; + + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the raw and meta counts now, so we can adjust tests accordingly. + */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->page_buf); + + base_raw_cnt = f->shared->page_buf->raw_count; + base_meta_cnt = f->shared->page_buf->meta_count; + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 0) + TEST_ERROR; + if(page_buf->min_raw_count != 5) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + TEST_ERROR; + + /* write all meta data, this would end up in page buffer since there + * is no raw data yet + */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + if(page_buf->meta_count != 5 - base_raw_cnt) + TEST_ERROR; + + /* write/read all raw data, this would end up in page buffer */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->raw_count != 5) + TEST_ERROR; + + if(page_buf->meta_count != 0) + TEST_ERROR; + + /* write and read more meta data and make sure that they don't end up in + * page buffer since the minimum metadata is actually the entire + * page buffer + */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*350), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*750), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*900), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->raw_count != 5) + TEST_ERROR; + + if(page_buf->meta_count != 0) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + + HDprintf("\tMinimum metadata threshold = 40%%, Minimum rawdata threshold = 40%%\n"); + page_count = 0; + /* keep 5 pages at max in the page buffer 2 meta pages, 2 raw pages + * minimum + */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 40, 40) < 0) + TEST_ERROR; + + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * + * However, with the current 1 metadata page inserted into the + * the page buffer, it is not necessary to track the base raw and + * metadata entry counts. + */ + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 2) + TEST_ERROR; + + if(page_buf->min_raw_count != 2) + TEST_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* fill the page buffer with raw data */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 5 - base_meta_cnt) + TEST_ERROR; + + /* add 3 meta entries evicting 3 raw entries */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 3) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 2) + TEST_ERROR; + + /* adding more meta entires should replace meta entries since raw data + * is at its minimum + */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 3) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 2) + TEST_ERROR; + + /* bring existing raw entires up the LRU */ + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* adding 2 raw entries (even with 1 call) should only evict 1 meta + * entry and another raw entry + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 2) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 3) + TEST_ERROR; + + /* adding 2 meta entries should replace 2 entires at the bottom of the LRU */ + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*98), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*242), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 2) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 3) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + HDprintf("\tMinimum metadata threshold = 20%%\n"); + page_count = 0; + /* keep 5 pages at max in the page buffer and 1 meta page minimum */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 39, 0) < 0) + TEST_ERROR; + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 1) + TEST_ERROR; + if(page_buf->min_raw_count != 0) + TEST_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* fill the page buffer with raw data */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* add 2 meta entries evicting 2 raw entries */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 2) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 3) + TEST_ERROR; + + /* bring the rest of the raw entries up the LRU */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* write one more raw entry which replace one meta entry */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 1) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 4) + TEST_ERROR; + + /* write one more raw entry which should replace another raw entry + * keeping min threshold of meta entries + */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 1) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 4) + TEST_ERROR; + + /* write a metadata entry that should replace the metadata entry + * at the bottom of the LRU + */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 1) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 4) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + + HDfree(data); + + PASSED() + + return 0; + +error: + + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + + return 1; + +} /* test_min_threshold */ + + +/*------------------------------------------------------------------------- + * Function: test_stats_collection() + * + * Purpose: Tests verifying correct collection of statistics + * by the page buffer. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added the base_raw_cnt and base_meta_cnt fields and + * supporting code. This allows the test to adjust to the + * number of page buffer pages accessed during file open / + * create. + * + * The tests for the values of base_raw_cnt and base_meta_cnt + * just after file open exist detect changes in library + * behavior. Assuming any such change is not indicative of + * other issues, these tests can be modified to reflect the + * change. + * + * JRM -- 2/23/17 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + int i; + int num_elements = 1000; + size_t base_raw_cnt = 0; + size_t base_meta_cnt = 0; + haddr_t meta_addr = HADDR_UNDEF; + haddr_t raw_addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("Statistics Collection"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + TEST_ERROR; + + /* keep 5 pages at max in the page buffer */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 20, 0) < 0) + TEST_ERROR; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the raw and meta counts now, so we can adjust the expected + * statistics accordingly. + */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->page_buf); + + base_raw_cnt = f->shared->page_buf->raw_count; + base_meta_cnt = f->shared->page_buf->meta_count; + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + + /* reset statistics before we begin the tests */ + if(H5Freset_page_buffering_stats(file_id) < 0) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + for(i=0 ; i<200 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*200, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*182, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*200, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->accesses[0] != 8) + TEST_ERROR; + if(f->shared->page_buf->accesses[1] != 16) + TEST_ERROR; + + if(f->shared->page_buf->bypasses[0] != 3) + TEST_ERROR; + if(f->shared->page_buf->bypasses[1] != 1) + TEST_ERROR; + + if(f->shared->page_buf->hits[0] != 0) + TEST_ERROR; + if(f->shared->page_buf->hits[1] != 4) + TEST_ERROR; + + if(f->shared->page_buf->misses[0] != 8) + TEST_ERROR; + if(f->shared->page_buf->misses[1] != 11) + TEST_ERROR; + + if(f->shared->page_buf->evictions[0] != 5 + base_meta_cnt) + TEST_ERROR; + if(f->shared->page_buf->evictions[1] != 9 + base_raw_cnt) + TEST_ERROR; + + { + unsigned accesses[2]; + unsigned hits[2]; + unsigned misses[2]; + unsigned evictions[2]; + unsigned bypasses[2]; + + if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, evictions, bypasses) < 0) + FAIL_STACK_ERROR; + + if(accesses[0] != 8) + TEST_ERROR; + if(accesses[1] != 16) + TEST_ERROR; + if(bypasses[0] != 3) + TEST_ERROR; + if(bypasses[1] != 1) + TEST_ERROR; + if(hits[0] != 0) + TEST_ERROR; + if(hits[1] != 4) + TEST_ERROR; + if(misses[0] != 8) + TEST_ERROR; + if(misses[1] != 11) + TEST_ERROR; + if(evictions[0] != 5 + base_meta_cnt) + TEST_ERROR; + if(evictions[1] != 9 + base_raw_cnt) + TEST_ERROR; + + if(H5Freset_page_buffering_stats(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, evictions, bypasses) < 0) + FAIL_STACK_ERROR; + + if(accesses[0] != 0) + TEST_ERROR; + if(accesses[1] != 0) + TEST_ERROR; + if(bypasses[0] != 0) + TEST_ERROR; + if(bypasses[1] != 0) + TEST_ERROR; + if(hits[0] != 0) + TEST_ERROR; + if(hits[1] != 0) + TEST_ERROR; + if(misses[0] != 0) + TEST_ERROR; + if(misses[1] != 0) + TEST_ERROR; + if(evictions[0] != 0) + TEST_ERROR; + if(evictions[1] != 0) + TEST_ERROR; + } /* end block */ + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + + return 1; +} /* test_stats_collection */ + + +/*------------------------------------------------------------------------- + * Function: main() + * + * Purpose: Main function for the page buffer tests. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + hid_t fapl = -1; /* File access property list for data files */ + unsigned nerrors = 0; /* Cumulative error count */ + const char *env_h5_drvr = NULL; /* File Driver value from environment */ + +#ifndef H5_HAVE_PARALLEL + + h5_reset(); + + /* Get the VFD to use */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* Temporary skip testing with multi/split drivers: + * Page buffering depends on paged aggregation which is + * currently disabled for multi/split drivers. + */ + if((0 == HDstrcmp(env_h5_drvr, "multi")) || + (0 == HDstrcmp(env_h5_drvr, "split"))) { + + SKIPPED() + HDputs("Skip page buffering test because paged aggregation is disabled for multi/split drivers"); + HDexit(EXIT_SUCCESS); + } /* end if */ + + if((fapl = h5_fileaccess()) < 0) { + nerrors++; + PUTS_ERROR("Can't get VFD-dependent fapl") + } /* end if */ + + nerrors += test_args(fapl, env_h5_drvr); + nerrors += test_raw_data_handling(fapl, env_h5_drvr); + nerrors += test_lru_processing(fapl, env_h5_drvr); + nerrors += test_min_threshold(fapl, env_h5_drvr); + nerrors += test_stats_collection(fapl, env_h5_drvr); + + h5_clean_files(FILENAME, fapl); + + if(nerrors) + goto error; + + HDputs("All Page Buffering tests passed."); + +#else + SKIPPED() + HDputs("Page Buffering is disabled for parallel."); +#endif + + HDexit(EXIT_SUCCESS); + +error: + + HDprintf("***** %d Page Buffering TEST%s FAILED! *****\n", + nerrors, nerrors > 1 ? "S" : ""); + + H5E_BEGIN_TRY { + H5Pclose(fapl); + } H5E_END_TRY; + + HDexit(EXIT_FAILURE); + +} /* main() */ + diff --git a/tools/test/h5diff/testfiles/h5diff_dset_idx1.h5 b/test/paged_nopersist.h5 Binary files differindex 3252303..b6c945a 100644 --- a/tools/test/h5diff/testfiles/h5diff_dset_idx1.h5 +++ b/test/paged_nopersist.h5 diff --git a/test/paged_persist.h5 b/test/paged_persist.h5 Binary files differnew file mode 100644 index 0000000..f0ae836 --- /dev/null +++ b/test/paged_persist.h5 diff --git a/test/set_extent.c b/test/set_extent.c index a992419..17dc6a0 100644 --- a/test/set_extent.c +++ b/test/set_extent.c @@ -139,6 +139,15 @@ int main( void ) unsigned new_format; /* Whether to use the latest file format */ unsigned chunk_cache; /* Whether to enable chunk caching */ int nerrors = 0; + const char *env_h5_drvr; /* File Driver value from environment */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + + + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); /* Initialize random number seed */ HDsrandom((unsigned)HDtime(NULL)); @@ -194,12 +203,15 @@ int main( void ) H5F_LIBVER_LATEST) < 0) TEST_ERROR /* Tests which use chunked datasets */ - nerrors += do_ranks( my_fapl, new_format ) < 0 ? 1 : 0; + if(!new_format || (new_format && contig_addr_vfd)) + nerrors += do_ranks( my_fapl, new_format ) < 0 ? 1 : 0; } /* end for */ /* Tests which do not use chunked datasets */ - nerrors += test_external( fapl ) < 0 ? 1 : 0; - nerrors += do_layouts( fapl ) < 0 ? 1 : 0; + if(!new_format || (new_format && contig_addr_vfd)) { + nerrors += test_external( fapl ) < 0 ? 1 : 0; + nerrors += do_layouts( fapl ) < 0 ? 1 : 0; + } } /* end for */ /* Close 2nd FAPL */ diff --git a/test/stab.c b/test/stab.c index f81bb5f..00aee21 100644 --- a/test/stab.c +++ b/test/stab.c @@ -96,7 +96,7 @@ const char *FILENAME[] = { *------------------------------------------------------------------------- */ static int -test_misc(hid_t fapl, hbool_t new_format) +test_misc(hid_t fcpl, hid_t fapl, hbool_t new_format) { hid_t fid = (-1); /* File ID */ hid_t g1 = (-1), g2 = (-1), g3 = (-1); @@ -110,7 +110,7 @@ test_misc(hid_t fapl, hbool_t new_format) /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR /* Create initial groups for testing, then close */ if((g1 = H5Gcreate2(fid, "test_1a", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR @@ -177,7 +177,7 @@ test_misc(hid_t fapl, hbool_t new_format) *------------------------------------------------------------------------- */ static int -test_long(hid_t fapl, hbool_t new_format) +test_long(hid_t fcpl, hid_t fapl, hbool_t new_format) { hid_t fid = (-1); /* File ID */ hid_t g1 = (-1), g2 = (-1); @@ -192,7 +192,7 @@ test_long(hid_t fapl, hbool_t new_format) /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR /* Group names */ name1 = (char *)HDmalloc((size_t)LONG_NAME_LEN); @@ -252,7 +252,7 @@ error: *------------------------------------------------------------------------- */ static int -test_large(hid_t fapl, hbool_t new_format) +test_large(hid_t fcpl, hid_t fapl, hbool_t new_format) { hid_t fid = (-1); /* File ID */ hid_t cwg = (-1), dir = (-1); /* Group IDs */ @@ -267,7 +267,7 @@ test_large(hid_t fapl, hbool_t new_format) /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR /* * Create a directory that has so many entries that the root @@ -318,7 +318,7 @@ test_large(hid_t fapl, hbool_t new_format) *------------------------------------------------------------------------- */ static int -lifecycle(hid_t fapl2) +lifecycle(hid_t fcpl, hid_t fapl2) { hid_t fid = (-1); /* File ID */ hid_t gid = (-1); /* Group ID */ @@ -341,7 +341,7 @@ lifecycle(hid_t fapl2) /* Create file */ h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl2)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) TEST_ERROR /* Close file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -532,7 +532,7 @@ error: *------------------------------------------------------------------------- */ static int -long_compact(hid_t fapl2) +long_compact(hid_t fcpl, hid_t fapl2) { hid_t fid = (-1); /* File ID */ hid_t gid = (-1); /* Group ID */ @@ -546,7 +546,7 @@ long_compact(hid_t fapl2) /* Create file */ h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl2)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) TEST_ERROR /* Close file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -755,7 +755,7 @@ error: *------------------------------------------------------------------------- */ static int -no_compact(hid_t fapl2) +no_compact(hid_t fcpl, hid_t fapl2) { hid_t fid = (-1); /* File ID */ hid_t gid = (-1); /* Group ID */ @@ -772,7 +772,7 @@ no_compact(hid_t fapl2) /* Create file */ h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl2)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) TEST_ERROR /* Close file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -1165,9 +1165,20 @@ error: int main(void) { - hid_t fapl, fapl2; /* File access property list IDs */ - unsigned new_format; /* Whether to use the new format or not */ - int nerrors = 0; + hid_t fapl, fapl2; /* File access property list IDs */ + hid_t fcpl, fcpl2; /* File creation property list ID */ + unsigned new_format; /* Whether to use the new format or not */ + const char *env_h5_drvr; /* File Driver value from environment */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + int nerrors = 0; + + /* Get the VFD to use */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); /* Reset library */ h5_reset(); @@ -1179,20 +1190,42 @@ main(void) /* Set the "use the latest version of the format" bounds for creating objects in the file */ if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR + /* Set up file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) TEST_ERROR + if((fcpl2 = H5Pcopy(fcpl)) < 0) TEST_ERROR + + /* Set to use paged aggregation strategy and persisting free-space */ + /* Skip testing for multi/split drivers */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)1) < 0) + TEST_ERROR + /* Loop over using new group format */ for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl = fapl; + hid_t my_fcpl = fcpl; + + if(!contig_addr_vfd) + continue; + + if(new_format) { + my_fapl = fapl2; + my_fcpl = fcpl2; /* Set to use paged aggregation and persisting free-space */ + } + /* Perform basic tests, with old & new style groups */ - nerrors += test_misc((new_format ? fapl2 : fapl), new_format); - nerrors += test_long((new_format ? fapl2 : fapl), new_format); - nerrors += test_large((new_format ? fapl2 : fapl), new_format); + nerrors += test_misc(my_fcpl, my_fapl, new_format); + nerrors += test_long(my_fcpl, my_fapl, new_format); + nerrors += test_large(my_fcpl, my_fapl, new_format); } /* end for */ /* New format group specific tests (require new format features) */ - nerrors += lifecycle(fapl2); - nerrors += long_compact(fapl2); - nerrors += read_old(); - nerrors += no_compact(fapl2); - nerrors += gcpl_on_root(fapl2); + if(contig_addr_vfd) { + nerrors += lifecycle(fcpl2, fapl2); + nerrors += long_compact(fcpl2, fapl2); + nerrors += read_old(); + nerrors += no_compact(fcpl2, fapl2); + nerrors += gcpl_on_root(fapl2); + } /* Old group API specific tests */ nerrors += old_api(fapl); @@ -1200,6 +1233,8 @@ main(void) /* Close 2nd FAPL */ H5Pclose(fapl2); + H5Pclose(fcpl); + H5Pclose(fcpl2); /* Verify symbol table messages are cached */ nerrors += (h5_verify_cached_stabs(FILENAME, fapl) < 0 ? 1 : 0); diff --git a/test/testfiles/plist_files/def_fcpl_32be b/test/testfiles/plist_files/def_fcpl_32be Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_32be +++ b/test/testfiles/plist_files/def_fcpl_32be diff --git a/test/testfiles/plist_files/def_fcpl_32le b/test/testfiles/plist_files/def_fcpl_32le Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_32le +++ b/test/testfiles/plist_files/def_fcpl_32le diff --git a/test/testfiles/plist_files/def_fcpl_64be b/test/testfiles/plist_files/def_fcpl_64be Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_64be +++ b/test/testfiles/plist_files/def_fcpl_64be diff --git a/test/testfiles/plist_files/def_fcpl_64le b/test/testfiles/plist_files/def_fcpl_64le Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_64le +++ b/test/testfiles/plist_files/def_fcpl_64le diff --git a/test/testfiles/plist_files/fcpl_32be b/test/testfiles/plist_files/fcpl_32be Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_32be +++ b/test/testfiles/plist_files/fcpl_32be diff --git a/test/testfiles/plist_files/fcpl_32le b/test/testfiles/plist_files/fcpl_32le Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_32le +++ b/test/testfiles/plist_files/fcpl_32le diff --git a/test/testfiles/plist_files/fcpl_64be b/test/testfiles/plist_files/fcpl_64be Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_64be +++ b/test/testfiles/plist_files/fcpl_64be diff --git a/test/testfiles/plist_files/fcpl_64le b/test/testfiles/plist_files/fcpl_64le Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_64le +++ b/test/testfiles/plist_files/fcpl_64le diff --git a/test/testflushrefresh.sh.in b/test/testflushrefresh.sh.in index 83b7134..e7917a0 100644 --- a/test/testflushrefresh.sh.in +++ b/test/testflushrefresh.sh.in @@ -79,11 +79,22 @@ if [ $rc -ne 0 ] ; then exit 0 fi -# ======================== -# Launch the Test Program. -# ======================== -./flushrefresh & -pid_main=$! +# HDF5 has several tests that create and delete signal files to communicate +# between processes, and it seems that even though the names of the files are +# different, occasionally the wrong file is deleted, interrupting the flow of +# the test. Running each of these tests in its own directory should eliminate +# the problem. +mkdir -p flushrefresh_test/.libs +cp flushrefresh flushrefresh_test + +# With the --disable-shared option, flushrefresh is built in the test directory, +# otherwise it is in test/.libs with a wrapper script named flushrefresh in +# the test directory. test/flushrefresh should always be copied, +# .libs/flushrefresh should be copied only if it exists. +if [ -f .libs/flushrefresh ]; then + cp .libs/flushrefresh flushrefresh_test/.libs +fi +cd flushrefresh_test # ================================================= # Set up/initialize some variables to be used later @@ -98,6 +109,12 @@ if [ -e $testfile ]; then rm $testfile fi +# ======================== +# Launch the Test Program. +# ======================== +./flushrefresh & +pid_main=$! + # ======================================= # Run flush verification on test program. # ======================================= diff --git a/test/tfile.c b/test/tfile.c index 6987493..7edc40c 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -104,13 +104,13 @@ /* Declarations for test_filespace_*() */ #define FILENAME_LEN 1024 /* length of file name */ -#define CORE_INCREMENT 1024 /* core file */ -#define FAMILY_SIZE 1024 /* family file */ #define DSETNAME "dset" /* Name of dataset */ #define NELMTS(X) (sizeof(X)/sizeof(X[0])) /* # of elements */ #define READ_OLD_BUFSIZE 1024 /* Buffer for holding file data */ #define FILE5 "tfile5.h5" /* Test file */ #define TEST_THRESHOLD10 10 /* Free space section threshold */ +#define FSP_SIZE_DEF 4096 /* File space page size default */ +#define FSP_SIZE512 512 /* File space page size */ /* Declaration for test_libver_macros2() */ #define FILE6 "tfile6.h5" /* Test file */ @@ -125,17 +125,30 @@ const char *OLD_FILENAME[] = { "filespace_1_6.h5", /* 1.6 HDF5 file */ "filespace_1_8.h5" /* 1.8 HDF5 file */ }; -const char *FILESPACE_NAME[] = { - "tfilespace", - NULL + +/* Files created in 1.10.0 release --used in test_filespace_1.10.0_compatible() */ +/* These files are copied from release 1.10.0 tools/h5format_convert/testfiles */ +const char *OLD_1_10_0_FILENAME[] = { + "h5fc_ext1_i.h5", /* 0 */ + "h5fc_ext1_f.h5", /* 1 */ + "h5fc_ext2_if.h5", /* 2 */ + "h5fc_ext2_sf.h5", /* 3 */ + "h5fc_ext3_isf.h5", /* 4 */ + "h5fc_ext_none.h5" /* 5 */ +}; + +/* Files used in test_filespace_round_compatible() */ +const char *FSPACE_FILENAMES[] = { + "fsm_aggr_nopersist.h5", /* H5F_FILE_SPACE_AGGR, not persisting free-space */ + "fsm_aggr_persist.h5", /* H5F_FILE_SPACE_AGGR, persisting free-space */ + "paged_nopersist.h5", /* H5F_FILE_SPACE_PAGE, not persisting free-space */ + "paged_persist.h5", /* H5F_FILE_SPACE_PAGE, persisting free-space */ + "aggr.h5", /* H5F_FILE_SPACE_AGGR */ + "none.h5" /* H5F_FILE_SPACE_NONE */ }; -const char *FILENAME[] = { - "sec2_tfile", - "split_tfile", - "stdio_tfile", - "core_tfile", - "family_tfile", +const char *FILESPACE_NAME[] = { + "tfilespace", NULL }; @@ -1538,112 +1551,7 @@ test_file_perm2(void) CHECK(ret, FAIL, "H5Sclose"); } /* end test_file_perm2() */ -/**************************************************************** -** -** test_file_freespace(): low-level file test routine. -** This test checks the free space available in a file in various -** situations. -** -** Modifications: -** Vailin Choi; July 2012 -** Remove datasets in reverse order so that all file spaces are shrunk. -** (A change due to H5FD_FLMAP_DICHOTOMY.) -** -*****************************************************************/ -static void -test_file_freespace(void) -{ - hid_t file; /* File opened with read-write permission */ - h5_stat_size_t empty_filesize; /* Size of file when empty */ - h5_stat_size_t mod_filesize; /* Size of file after being modified */ - hssize_t free_space; /* Amount of free space in file */ - hid_t dspace; /* Dataspace ID */ - hid_t dset; /* Dataset ID */ - hid_t dcpl; /* Dataset creation property list */ - int k; /* Local index variable */ - unsigned u; /* Local index variable */ - char name[32]; /* Dataset name */ - herr_t ret; - - /* Output message about test being performed */ - MESSAGE(5, ("Testing Low-Level File Free Space\n")); - - /* Create an "empty" file */ - file = H5Fcreate(FILE1, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - CHECK(file, FAIL, "H5Fcreate"); - - ret = H5Fclose(file); - CHECK_I(ret, "H5Fclose"); - - /* Get the "empty" file size */ - empty_filesize = h5_get_file_size(FILE1, H5P_DEFAULT); - - /* Re-open the file (with read-write permission) */ - file = H5Fopen(FILE1, H5F_ACC_RDWR, H5P_DEFAULT); - CHECK_I(file, "H5Fopen"); - - /* Check that the free space is 0 */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - VERIFY(free_space, 0, "H5Fget_freespace"); - - /* Create dataspace for datasets */ - dspace = H5Screate(H5S_SCALAR); - CHECK(dspace, FAIL, "H5Screate"); - - /* Create a dataset creation property list */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - CHECK(dcpl, FAIL, "H5Pcreate"); - - /* Set the space allocation time to early */ - ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); - CHECK(ret, FAIL, "H5Pset_alloc_time"); - - /* Create datasets in file */ - for(u = 0; u < 10; u++) { - sprintf(name, "Dataset %u", u); - dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); - CHECK(dset, FAIL, "H5Dcreate2"); - - ret = H5Dclose(dset); - CHECK(ret, FAIL, "H5Dclose"); - } /* end for */ - - /* Close dataspace */ - ret = H5Sclose(dspace); - CHECK(ret, FAIL, "H5Sclose"); - - /* Close dataset creation property list */ - ret = H5Pclose(dcpl); - CHECK(ret, FAIL, "H5Pclose"); - - /* Check that there is the right amount of free space in the file */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - VERIFY(free_space, 2360, "H5Fget_freespace"); - /* Delete datasets in file */ - for(k = 9; k >= 0; k--) { - sprintf(name, "Dataset %u", (unsigned)k); - ret = H5Ldelete(file, name, H5P_DEFAULT); - CHECK(ret, FAIL, "H5Ldelete"); - } /* end for */ - - /* Check that there is the right amount of free space in the file */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - VERIFY(free_space, 0, "H5Fget_freespace"); - - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); - - /* Get the file size after modifications*/ - mod_filesize = h5_get_file_size(FILE1, H5P_DEFAULT); - - /* Check that the file reverted to empty size */ - VERIFY(mod_filesize, empty_filesize, "H5Fget_freespace"); -} /* end test_file_freespace() */ /**************************************************************** ** @@ -3185,419 +3093,1227 @@ test_userblock_alignment(void) /**************************************************************** ** -** test_free_sections(): -** This routine does the actual work of checking information for -** free space sections available in a file in various situations. +** test_userblock_alignment_paged(): low-level file test routine. +** This test checks to ensure that files with both a userblock and +** alignment interact properly: +** -- alignment via H5Pset_alignment +** -- alignment via paged aggregation +** +** Programmer: Vailin Choi; March 2013 ** *****************************************************************/ static void -test_free_sections(hid_t fapl, char *fname) +test_userblock_alignment_paged(void) { - hid_t file; /* File ID */ - hid_t fcpl; /* File creation property list template */ - hssize_t free_space; /* Amount of free space in file */ - hid_t dspace; /* Dataspace ID */ - hid_t dset; /* Dataset ID */ - hid_t dcpl; /* Dataset creation property list */ - unsigned u; /* Local index variable */ - char name[32]; /* Dataset name */ - hssize_t nsects; /* # of free-space sections */ - hssize_t saved_nsects; /* saved copy for the # of free-space sections */ - int i; /* local index variable */ - hsize_t total; /* sum of the free-space section sizes */ - hsize_t last_size; /* size of last free-space section */ - H5F_sect_info_t *sect_info; /* array to hold the free-space information */ - H5F_sect_info_t *saved_sect_info; /* array to hold the free-space information */ - herr_t ret; /* return value */ - - /* Create file-creation template */ + hid_t fid; /* File ID */ + hid_t fcpl; /* File creation property list ID */ + hid_t fapl; /* File access property list ID */ + herr_t ret; /* Generic return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("Testing interaction between userblock and alignment (via paged aggregation and H5Pset_alignment)\n")); + + /* + * Case 1: + * Userblock size = 0 + * Alignment in use = 4096 + * Strategy = H5F_FILE_SPACE_PAGE; fsp_size = alignment = 4096 + * Outcome: + * Should succeed: + * userblock is 0 and alignment != 0 + */ + /* Create file creation property list with user block */ fcpl = H5Pcreate(H5P_FILE_CREATE); CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)0); + CHECK(ret, FAIL, "H5Pset_userblock"); - /* Set file space strategy and free space section threshold */ - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Create file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); - /* Create the file */ - file = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); - CHECK(file, FAIL, "H5Fcreate"); + /* Set the "use the latest version of the format" bounds */ + ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); - /* Close the FCPL */ + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ ret = H5Pclose(fcpl); CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - /* Create dataspace for datasets */ - dspace = H5Screate(H5S_SCALAR); - CHECK(dspace, FAIL, "H5Screate"); + /* + * Case 2a: + * Userblock size = 1024 + * Alignment in use = 512 + * Strategy = H5F_FILE_SPACE_PAGE; fsp_size = alignment = 512 + * H5Pset_alignment() is 3 + * Outcome: + * Should succeed: + * userblock (1024) is integral mult. of alignment (512) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); - /* Create a dataset creation property list */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - CHECK(dcpl, FAIL, "H5Pcreate"); + /* Create file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Set the space allocation time to early */ - ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); - CHECK(ret, FAIL, "H5Pset_alloc_time"); + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); - /* Create datasets in file */ - for(u = 0; u < 10; u++) { - sprintf(name, "Dataset %u", u); - dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); - CHECK(dset, FAIL, "H5Dcreate2"); + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - ret = H5Dclose(dset); - CHECK(ret, FAIL, "H5Dclose"); - } /* end for */ + /* + * Case 2b: + * Userblock size = 1024 + * Alignment in use = 3 + * Strategy = H5F_FILE_SPACE_AGGR; fsp_size = 512 + * (via default file creation property) + * H5Pset_alignment() is 3 + * Outcome: + * Should fail at file creation: + * userblock (1024) is non-integral mult. of alignment (3) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); - /* Close dataspace */ - ret = H5Sclose(dspace); - CHECK(ret, FAIL, "H5Sclose"); + /* Create file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Close dataset creation property list */ - ret = H5Pclose(dcpl); + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); CHECK(ret, FAIL, "H5Pclose"); - /* Delete odd-numbered datasets in file */ - for(u = 0; u < 10; u++) { - sprintf(name, "Dataset %u", u); - if(u % 2) { - ret = H5Ldelete(file, name, H5P_DEFAULT); - CHECK(ret, FAIL, "H5Ldelete"); - } /* end if */ - } /* end for */ + /* + * Case 3a: + * Userblock size = 512 + * Alignment in use = 512 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 512 + * H5Pset_alignment() is 3 + * Outcome: + * Should succeed: + * userblock (512) is equal to alignment (512) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Re-open the file with read-only permission */ - file = H5Fopen(fname, H5F_ACC_RDONLY, fapl); - CHECK_I(file, "H5Fopen"); - - /* Get the amount of free space in the file */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - - /* Get the # of free-space sections in the file */ - saved_nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, NULL); - CHECK(saved_nsects, FAIL, "H5Fget_free_sections"); - - /* Allocate storage for the free space section information */ - saved_sect_info = (H5F_sect_info_t *)HDcalloc((size_t)saved_nsects, sizeof(H5F_sect_info_t)); - CHECK(saved_sect_info, NULL, "HDcalloc"); - - /* Should return failure when nsects is 0 with a nonnull sect_info */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, saved_sect_info); - VERIFY(nsects, FAIL, "H5Fget_free_sections"); - - /* Verify the correct # of free-space sections */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)saved_nsects, saved_sect_info); - VERIFY(nsects, saved_nsects, "H5Fget_free_sections"); - - /* Verify the amount of free-space is correct */ - total = 0; - for(i = 0; i < nsects; i++) - total += saved_sect_info[i].size; - VERIFY(free_space, total, "H5Fget_free_sections"); - - /* save the last section's size */ - last_size = saved_sect_info[nsects-1].size; - - /* Allocate storage for -1 free space section information */ - sect_info = (H5F_sect_info_t *)HDcalloc((size_t)(saved_nsects - 1), sizeof(H5F_sect_info_t)); - CHECK(sect_info, NULL, "HDcalloc"); - - /* Retrieve free space info for -1 sections */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(saved_nsects - 1), sect_info); - VERIFY(nsects, saved_nsects, "H5Fget_free_sections"); - - /* Verify the amount of free-space is correct */ - total = 0; - for(i = 0; i < (saved_nsects - 1); i++) { - VERIFY(sect_info[i].addr, saved_sect_info[i].addr, "H5Fget_free_sections"); - VERIFY(sect_info[i].size, saved_sect_info[i].size, "H5Fget_free_sections"); - total += sect_info[i].size; - } + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); - VERIFY(((hsize_t)free_space - last_size), total, "H5Fget_free_sections"); - HDfree(sect_info); + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - /* Allocate storage for +1 free space section information */ - sect_info = (H5F_sect_info_t *)HDcalloc((size_t)(saved_nsects + 1), sizeof(H5F_sect_info_t)); - CHECK(sect_info, NULL, "HDcalloc"); + /* + * Case 3b: + * Userblock size = 512 + * Alignment in use = 3 + * Strategy is H5F_FILE_SPACE_NONE; fsp_size = 512 + * H5Pset_alignment() is 3 + * Outcome: + * Should fail at file creation: + * userblock (512) is non-integral mult. of alignment (3) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); - /* Retrieve free-space info for +1 sections */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(saved_nsects + 1), sect_info); - VERIFY(nsects, saved_nsects, "H5Fget_free_sections"); + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Verify free-space info is correct */ - total = 0; - for(i = 0; i < nsects; i++) { - VERIFY(sect_info[i].addr, saved_sect_info[i].addr, "H5Fget_free_sections"); - VERIFY(sect_info[i].size, saved_sect_info[i].size, "H5Fget_free_sections"); - total += sect_info[i].size; - } + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); - VERIFY(sect_info[nsects].addr, 0, "H5Fget_free_sections"); - VERIFY(sect_info[nsects].size, 0, "H5Fget_free_sections"); - VERIFY(free_space, total, "H5Fget_free_sections"); - HDfree(sect_info); + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - /* Verify that there is no free-space section for this type */ - nsects = H5Fget_free_sections(file, H5FD_MEM_BTREE, (size_t)0, NULL); - VERIFY(nsects, 0, "H5Fget_free_sections"); + /* + * Case 4a: + * Userblock size = 1024 + * Alignment in use = 1023 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 1023 + * H5Pset_alignment() is 16 + * Outcome: + * Should fail at file creation: + * userblock (1024) is non-integral multiple of alignment (1023) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1023); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); - HDfree(saved_sect_info); + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); -} /* end test_free_sections() */ + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 4b: + * Userblock size = 1024 + * Alignment in use = 16 + * Strategy is H5F_FILE_SPACE_FSM_AGGR; fsp_size = 1023 + * H5Pset_alignment() is 16 + * Outcome: + * Should succeed: + * userblock (512) is integral multiple of alignment (16) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1023); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 5a: + * Userblock size = 512 + * Alignment in use = 1024 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 1024 + * H5Pset_alignment() is 16 + * Outcome: + * Should fail at file creation: + * userblock (512) is less than alignment (1024) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 5b: + * Userblock size = 512 + * Alignment in use = 16 + * Strategy is H5F_FILE_SPACE_NONE; fsp_size = 1024 + * H5Pset_alignment() is 16 + * Outcome: + * Should succed: + * userblock (512) is integral multiple of alignment (16) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 6: + * Userblock size = 512 + * Alignment in use = 512 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 512 + * H5Pset_alignment() is 3 + * Reopen the file; H5Pset_alignment() is 1024 + * Outcome: + * Should succed: + * Userblock (512) is the same as alignment (512); + * The H5Pset_alignment() calls have no effect + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + + /* Change alignment in FAPL */ + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper2(fapl, FALSE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + +} /* end test_userblock_alignment_paged() */ /**************************************************************** ** -** test_filespace_sects(): -** This test checks free space section info for -** files created with sec2 and split drivers. +** test_filespace_info(): +** Verify the following public routines retrieve and set file space +** information correctly: +** (1) H5Pget/set_file_space_strategy(): +** Retrieve and set file space strategy, persisting free-space, +** and free-space section threshold as specified +** (2) H5Pget/set_file_space_page_size(): +** Retrieve and set the page size for paged aggregation ** -*****************************************************************/ +****************************************************************/ static void -test_filespace_sects(void) +test_filespace_info(const char *env_h5_drvr) { - hid_t fapl_sec2; /* File access property id with sec2 driver */ - hid_t fapl_split; /* File access property id with split driver */ - hid_t fapl_core; /* File access property id with core driver */ - hid_t fapl_stdio; /* File access property id with stdio driver */ - hid_t fapl_family; /* File access property id with family driver */ - char filename[FILENAME_LEN]; /* Filename to use */ - herr_t ret; /* Return value */ + hid_t fid; /* File IDs */ + hid_t fapl, new_fapl; /* File access property lists */ + hid_t fcpl, fcpl1, fcpl2; /* File creation property lists */ + H5F_fspace_strategy_t strategy; /* File space strategy */ + hbool_t persist; /* Persist free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + unsigned new_format; /* New or old format */ + H5F_fspace_strategy_t fs_strategy; /* File space strategy--iteration variable */ + unsigned fs_persist; /* Persist free-space or not--iteration variable */ + hsize_t fs_threshold; /* Free-space section threshold--iteration variable */ + hsize_t fsp_size; /* File space page size */ + char filename[FILENAME_LEN]; /* Filename to use */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + herr_t ret; /* Return value */ - /* SEC2 */ - MESSAGE(5, ("Testing File free space information for a sec2 file\n")); + /* Output message about test being performed */ + MESSAGE(5, ("Testing file creation public routines: H5Pget/set_file_space_strategy & H5Pget/set_file_space_page_size\n")); - fapl_sec2 = H5Pcreate(H5P_FILE_ACCESS); + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); - ret = H5Pset_fapl_sec2(fapl_sec2); - CHECK(ret, FAIL, "H5Pset_fapl_sec2"); + fapl = h5_fileaccess(); + h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_sec2, filename, sizeof(filename)); + /* Get a copy of the file access property list */ + new_fapl = H5Pcopy(fapl); + CHECK(new_fapl, FAIL, "H5Pcopy"); - /* perform free space information test for file with sec2 driver */ - test_free_sections(fapl_sec2, filename); + /* Set the "use the latest version of the format" bounds */ + ret = H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); - /* close fapl_sec2 and remove the file */ - h5_clean_files(FILENAME, fapl_sec2); + /* + * Case (1) + * Check file space information from a default file creation property list. + * Values expected: + * strategy--H5F_FILE_SPACE_AGGR + * persist--FALSE + * threshold--1 + * file space page size--4096 + */ + /* Create file creation property list template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* SPLIT */ - MESSAGE(5, ("Testing File free space information for a split file\n")); + /* Verify file space information */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); - fapl_split = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_split, FAIL, "h5_fileaccess"); + /* Retrieve file space page size */ + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE_DEF, "H5Pget_file_space_page_size"); - ret = H5Pset_fapl_split(fapl_split, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT); - CHECK(ret, FAIL, "H5Pset_fapl_split"); + /* Close property list */ + H5Pclose(fcpl); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[1], fapl_split, filename, sizeof(filename)); + /* + * Case (2) + * File space page size has a minimum size of 512. + * Setting value less than 512 will return an error; + * --setting file space page size to 0 + * --setting file space page size to 511 + */ + /* Create file creation property list template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); - /* perform free space information test for file with split driver */ - test_free_sections(fapl_split, filename); + /* Setting to 0: should fail */ + H5E_BEGIN_TRY { + ret = H5Pset_file_space_page_size(fcpl, 0); + } H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); - /* close fapl and remove the file */ - h5_clean_files(FILENAME, fapl_split); + /* Setting to 511: should fail */ + H5E_BEGIN_TRY { + ret = H5Pset_file_space_page_size(fcpl, 511); + } H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 512: should succeed */ + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); - /* STDIO */ - MESSAGE(5, ("Testing File free space information for a stdio file\n")); + /* Close property list */ + H5Pclose(fcpl); - fapl_stdio = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_stdio, FAIL, "h5_fileaccess"); + /* + * Case (3) + * Check file space information when creating a file with default properties. + * Values expected: + * strategy--H5F_FILE_SPACE_AGGR + * persist--FALSE + * threshold--1 + * file space page size--4096 + */ + /* Create a file with default file creation and access property lists */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fcreate"); - ret = H5Pset_fapl_stdio(fapl_stdio); - CHECK(ret, FAIL, "H5Pset_fapl_split"); + /* Get the file's creation property list */ + fcpl1 = H5Fget_create_plist(fid); + CHECK(fcpl1, FAIL, "H5Fget_create_plist"); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[2], fapl_stdio, filename, sizeof(filename)); + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl1, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* perform free space information test for file with stdio driver */ - test_free_sections(fapl_stdio, filename); + /* Verify file space information */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); - /* close fapl and remove the file */ - h5_clean_files(FILENAME, fapl_stdio); + /* Retrieve file space page size */ + ret = H5Pget_file_space_page_size(fcpl1, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE_DEF, "H5Pget_file_space_page_size"); + /* Close property lists */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + ret = H5Pclose(fcpl1); + CHECK(ret, FAIL, "H5Pclose"); - /* CORE */ - MESSAGE(5, ("Testing File free space information for a core file\n")); + /* + * Case (4) + * Check file space information when creating a file with the + * latest library format and default properties. + * Values expected: + * strategy--H5F_FILE_SPACE_AGGR + * persist--FALSE + * threshold--1 + * file space page size--4096 + */ + /* Create a file with the latest library format */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, new_fapl); + CHECK(fid, FAIL, "H5Fcreate"); - fapl_core = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_core, FAIL, "h5_fileaccess"); + /* Get the file's creation property */ + fcpl1 = H5Fget_create_plist(fid); + CHECK(fcpl1, FAIL, "H5Fget_create_plist"); - ret = H5Pset_fapl_core(fapl_core, (size_t)CORE_INCREMENT, TRUE); - CHECK(ret, FAIL, "H5Pset_fapl_core"); + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl1, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[3], fapl_core, filename, sizeof(filename)); + /* Verify file space information */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); - /* perform free space information test for file with core driver */ - test_free_sections(fapl_core, filename); + /* Retrieve file space page size */ + ret = H5Pget_file_space_page_size(fcpl1, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE_DEF, "H5Pget_file_space_page_size"); - /* close fapl_ and remove the file */ - h5_clean_files(FILENAME, fapl_core); + /* Close property lists */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + ret = H5Pclose(fcpl1); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case (5) + * Check file space information with the following combinations: + * Create file with -- + * New or old format + * Persist or not persist free-space + * Different sizes for free-space section threshold (0 to 10) + * The four file space strategies: + * H5F_FSPACE_STRATEGY_FSM_AGGR, H5F_FSPACE_STRATEGY_PAGE, + * H5F_FSPACE_STRATEGY_AGGR, H5F_FSPACE_STRATEGY_NONE + * File space page size: set to 512 + * + */ + for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl; + + /* Set the FAPL for the type of format */ + if(new_format) { + MESSAGE(5, ("Testing with new group format\n")); + my_fapl = new_fapl; + } /* end if */ + else { + MESSAGE(5, ("Testing with old group format\n")); + my_fapl = fapl; + } /* end else */ + + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + + /* Test with free-space section threshold size: 0 to 10 */ + for(fs_threshold = 0; fs_threshold <= TEST_THRESHOLD10; fs_threshold++) { + + /* Test with 4 file space strategies */ + for(fs_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_strategy < H5F_FSPACE_STRATEGY_NTYPES; H5_INC_ENUM(H5F_fspace_strategy_t, fs_strategy)) { + + if(!contig_addr_vfd && (fs_strategy == H5F_FSPACE_STRATEGY_PAGE || fs_persist)) + continue; + + /* Create file creation property list template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + + /* Set file space information */ + ret = H5Pset_file_space_strategy(fcpl, fs_strategy, (hbool_t)fs_persist, fs_threshold); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Verify file space information */ + VERIFY(strategy, fs_strategy, "H5Pget_file_space_strategy"); + + if(fs_strategy < H5F_FSPACE_STRATEGY_AGGR) { + VERIFY(persist, (hbool_t)fs_persist, "H5Pget_file_space_strategy"); + VERIFY(threshold, fs_threshold, "H5Pget_file_space_strategy"); + } else { + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + } + + /* Retrieve and verify file space page size */ + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + + /* Create the file with the specified file space info */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, my_fapl); + CHECK(fid, FAIL, "H5Fcreate"); + + /* Get the file's creation property */ + fcpl1 = H5Fget_create_plist(fid); + CHECK(fcpl1, FAIL, "H5Fget_create_plist"); + + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl1, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Verify file space information */ + VERIFY(strategy, fs_strategy, "H5Pget_file_space_strategy"); + + if(fs_strategy < H5F_FSPACE_STRATEGY_AGGR) { + VERIFY(persist, fs_persist, "H5Pget_file_space_strategy"); + VERIFY(threshold, fs_threshold, "H5Pget_file_space_strategy"); + } else { + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + } + + /* Retrieve and verify file space page size */ + ret = H5Pget_file_space_page_size(fcpl1, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Re-open the file */ + fid = H5Fopen(filename, H5F_ACC_RDWR, my_fapl); + CHECK(ret, FAIL, "H5Fopen"); + + /* Get the file's creation property */ + fcpl2 = H5Fget_create_plist(fid); + CHECK(fcpl2, FAIL, "H5Fget_create_plist"); + + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl2, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Verify file space information */ + VERIFY(strategy, fs_strategy, "H5Pget_file_space_strategy"); + if(fs_strategy < H5F_FSPACE_STRATEGY_AGGR) { + VERIFY(persist, fs_persist, "H5Pget_file_space_strategy"); + VERIFY(threshold, fs_threshold, "H5Pget_file_space_strategy"); + } else { + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + } + + /* Retrieve and verify file space page size */ + ret = H5Pget_file_space_page_size(fcpl2, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Release file creation property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fcpl1); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fcpl2); + CHECK(ret, FAIL, "H5Pclose"); + } /* end for file space strategy type */ + } /* end for free-space section threshold */ + } /* end for fs_persist */ + + /* close fapl_ and remove the file */ + h5_clean_files(FILESPACE_NAME, my_fapl); + } /* end for new_format */ + +} /* test_filespace_info() */ +/**************************************************************** +** +** set_multi_split(): +** Internal routine to set up page-aligned address space for multi/split driver +** when testing paged aggregation. +** This is used by test_file_freespace() and test_sects_freespace(). +** +*****************************************************************/ +static int +set_multi_split(hid_t fapl, hsize_t pagesize, hbool_t multi, hbool_t split) +{ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + HDassert(split || multi); + + HDmemset(memb_name, 0, sizeof memb_name); - /* FAMILY */ - MESSAGE(5, ("Testing File free space information for a family file\n")); + /* Get current split settings */ + if(H5Pget_fapl_multi(fapl, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR - fapl_family = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_family, FAIL, "h5_fileaccess"); + if(split) { + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + pagesize - 1) / pagesize) * pagesize; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + pagesize - 1) / pagesize) * pagesize; + } else { + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + memb_addr[mt] = ((memb_addr[mt] + pagesize - 1) / pagesize) * pagesize; + } /* end else */ - ret = H5Pset_fapl_family(fapl_family, (hsize_t)FAMILY_SIZE, H5P_DEFAULT); - CHECK(ret, FAIL, "H5Pset_fapl_family"); + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[4], fapl_family, filename, sizeof(filename)); + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); - /* perform free space information test for file with family driver */ - test_free_sections(fapl_family, filename); + return 0; - /* close fapl and remove the file */ - h5_clean_files(FILENAME, fapl_family); +error: + return(-1); -} /* end test_filespace_sects() */ +} /* set_multi_split() */ /**************************************************************** ** -** test_filespace_info(): -** Verify that the public routines H5Pget/set_file_space() -** retrieve and set the file space strategy and free space -** section threshold as specified. +** test_file_freespace(): +** This routine checks the free space available in a file as +** returned by the public routine H5Fget_freespace(). ** -****************************************************************/ +** Modifications: +** Vailin Choi; July 2012 +** Remove datasets in reverse order so that all file spaces are shrunk. +** (A change due to H5FD_FLMAP_DICHOTOMY.) +** +** Vailin Choi; Dec 2012 +** Add changes due to paged aggregation via new format: +** the amount of freespace is different. +** +*****************************************************************/ static void -test_filespace_info(void) +test_file_freespace(const char *env_h5_drvr) { - hid_t fid1, fid2; /* HDF5 File IDs */ - hid_t fapl, new_fapl; /* File access property */ - hid_t fcpl, fcpl1, fcpl2; /* File creation property */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_file_space_type_t strategy, fs_type, def_type; /* File space handling strategy */ - hsize_t threshold, fs_size, def_size; /* Free space section threshold */ - unsigned new_format; /* new format or old format */ - herr_t ret; /* return value */ + hid_t file; /* File opened with read-write permission */ + h5_stat_size_t empty_filesize; /* Size of file when empty */ + h5_stat_size_t mod_filesize; /* Size of file after being modified */ + hssize_t free_space; /* Amount of free space in file */ + hid_t fcpl; /* File creation property list */ + hid_t fapl, new_fapl; /* File access property list IDs */ + hid_t dspace; /* Dataspace ID */ + hid_t dset; /* Dataset ID */ + hid_t dcpl; /* Dataset creation property list */ + int k; /* Local index variable */ + unsigned u; /* Local index variable */ + char filename[FILENAME_LEN]; /* Filename to use */ + char name[32]; /* Dataset name */ + unsigned new_format; /* To use old or new format */ + hbool_t split_vfd, multi_vfd; /* Indicate multi/split driver */ + hsize_t expected_freespace; /* Freespace expected */ + hsize_t expected_fs_del; /* Freespace expected after delete */ + herr_t ret; /* Return value */ + + split_vfd = !HDstrcmp(env_h5_drvr, "split"); + multi_vfd = !HDstrcmp(env_h5_drvr, "multi"); + + if(!split_vfd && !multi_vfd) { + fapl = h5_fileaccess(); + h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); + + new_fapl = H5Pcopy(fapl); + CHECK(new_fapl, FAIL, "H5Pcopy"); + + /* Set the "use the latest version of the format" bounds */ + ret = H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); - /* Output message about test being performed */ - MESSAGE(5, ("Testing File Space Management public routines: H5Pget/set_file_space()\n")); + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + + /* Test with old & new format */ + for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl; + + /* Set the FAPL for the type of format */ + if(new_format) { + MESSAGE(5, ("Testing with new group format\n")); + + my_fapl = new_fapl; + + if(multi_vfd || split_vfd) { + ret = set_multi_split(new_fapl, FSP_SIZE_DEF, multi_vfd, split_vfd); + CHECK(ret, FAIL, "set_multi_split"); + } + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5P_set_file_space_strategy"); + + expected_freespace = 4534; + if(split_vfd) expected_freespace = 427; + if(multi_vfd) expected_freespace = 248; + expected_fs_del = 0; + } /* end if */ + else { + MESSAGE(5, ("Testing with old group format\n")); + /* Default: non-paged aggregation, non-persistent free-space */ + my_fapl = fapl; + expected_freespace = 2464; + if(split_vfd) expected_freespace = 264; + if(multi_vfd) expected_freespace = 0; + expected_fs_del = 4096; + + } /* end else */ + + /* Create an "empty" file */ + file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, my_fapl); + CHECK(file, FAIL, "H5Fcreate"); + + ret = H5Fclose(file); + CHECK_I(ret, "H5Fclose"); + + /* Get the "empty" file size */ + empty_filesize = h5_get_file_size(filename, H5P_DEFAULT); + + /* Re-open the file (with read-write permission) */ + file = H5Fopen(filename, H5F_ACC_RDWR, my_fapl); + CHECK_I(file, "H5Fopen"); + + /* Check that the free space is 0 */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + VERIFY(free_space, 0, "H5Fget_freespace"); + + /* Create dataspace for datasets */ + dspace = H5Screate(H5S_SCALAR); + CHECK(dspace, FAIL, "H5Screate"); + + /* Create a dataset creation property list */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + CHECK(dcpl, FAIL, "H5Pcreate"); + + /* Set the space allocation time to early */ + ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); + CHECK(ret, FAIL, "H5Pset_alloc_time"); + + /* Create datasets in file */ + for(u = 0; u < 10; u++) { + sprintf(name, "Dataset %u", u); + dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset, FAIL, "H5Dcreate2"); + + ret = H5Dclose(dset); + CHECK(ret, FAIL, "H5Dclose"); + } /* end for */ - fapl = h5_fileaccess(); - h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); + /* Close dataspace */ + ret = H5Sclose(dspace); + CHECK(ret, FAIL, "H5Sclose"); - new_fapl = H5Pcopy(fapl); - CHECK(new_fapl, FAIL, "H5Pcopy"); + /* Close dataset creation property list */ + ret = H5Pclose(dcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Set the "use the latest version of the format" bounds */ - ret = H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); - CHECK(ret, FAIL, "H5Pset_libver_bounds"); + /* Check that there is the right amount of free space in the file */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + VERIFY(free_space, expected_freespace, "H5Fget_freespace"); - /* Create file-creation template */ - fcpl = H5Pcreate(H5P_FILE_CREATE); - CHECK(fcpl, FAIL, "H5Pcreate"); + /* Delete datasets in file */ + for(k = 9; k >= 0; k--) { + sprintf(name, "Dataset %u", (unsigned)k); + ret = H5Ldelete(file, name, H5P_DEFAULT); + CHECK(ret, FAIL, "H5Ldelete"); + } /* end for */ - /* Get default file space information */ - ret = H5Pget_file_space(fcpl, &def_type, &def_size); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Check that there is the right amount of free space in the file */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + if(new_format) + VERIFY(free_space, expected_fs_del, "H5Fget_freespace"); + else + VERIFY(free_space, expected_fs_del, "H5Fget_freespace"); - /* Test with old & new format groups */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { - hid_t my_fapl; + /* Close file */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); + + /* Get the file size after modifications*/ + mod_filesize = h5_get_file_size(filename, H5P_DEFAULT); + + /* Check that the file reverted to empty size */ + VERIFY(mod_filesize, empty_filesize, "H5Fget_freespace"); + + h5_clean_files(FILESPACE_NAME, my_fapl); + + } /* end for */ + } + +} /* end test_file_freespace() */ + +/**************************************************************** +** +** test_sects_freespace(): +** This routine checks free-space section information for the +** file as returned by the public routine H5Fget_free_sections(). +** +*****************************************************************/ +static void +test_sects_freespace(const char *env_h5_drvr, hbool_t new_format) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file; /* File ID */ + hid_t fcpl; /* File creation property list template */ + hid_t fapl; /* File access property list template */ + hssize_t free_space; /* Amount of free-space in the file */ + hid_t dspace; /* Dataspace ID */ + hid_t dset; /* Dataset ID */ + hid_t dcpl; /* Dataset creation property list */ + char name[32]; /* Dataset name */ + hssize_t nsects = 0; /* # of free-space sections */ + hssize_t nall; /* # of free-space sections for all types of data */ + hssize_t nmeta = 0, nraw = 0; /* # of free-space sections for meta/raw/generic data */ + H5F_sect_info_t sect_info[15]; /* Array to hold free-space information */ + H5F_sect_info_t all_sect_info[15]; /* Array to hold free-space information for all types of data */ + H5F_sect_info_t meta_sect_info[15]; /* Array to hold free-space information for metadata */ + H5F_sect_info_t raw_sect_info[15]; /* Array to hold free-space information for raw data */ + hsize_t total = 0; /* sum of the free-space section sizes */ + hsize_t tmp_tot = 0; /* Sum of the free-space section sizes */ + hsize_t last_size; /* Size of last free-space section */ + hsize_t dims[1]; /* Dimension sizes */ + unsigned u; /* Local index variable */ + H5FD_mem_t type; + hbool_t split_vfd = FALSE, multi_vfd = FALSE; + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("Testing H5Fget_free_sections()--free-space section info in the file\n")); + + split_vfd = !HDstrcmp(env_h5_drvr, "split"); + multi_vfd = !HDstrcmp(env_h5_drvr, "multi"); + + if(!split_vfd && !multi_vfd) { + + fapl = h5_fileaccess(); + h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); + + /* Create file-creation template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); - /* Set the FAPL for the type of format */ if(new_format) { - MESSAGE(5, ("Testing with new group format\n")); - my_fapl = new_fapl; - } /* end if */ - else { - MESSAGE(5, ("Testing with old group format\n")); - my_fapl = fapl; - } /* end else */ + ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); + + /* Set to paged aggregation and persistent free-space */ + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Set up paged aligned address space for multi/split driver */ + if(multi_vfd || split_vfd) { + ret = set_multi_split(fapl, FSP_SIZE_DEF, multi_vfd, split_vfd); + CHECK(ret, FAIL, "set_multi_split"); + } + + } else { + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + } + + /* Create the file */ + file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + CHECK(file, FAIL, "H5Fcreate"); + + /* Create a dataset creation property list */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + CHECK(dcpl, FAIL, "H5Pcreate"); + + /* Set the space allocation time to early */ + ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); + CHECK(ret, FAIL, "H5Pset_alloc_time"); + + /* Create 1 large dataset */ + dims[0] = 1200; + dspace = H5Screate_simple(1, dims, NULL); + dset = H5Dcreate2(file, "Dataset_large", H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset, FAIL, "H5Dcreate2"); - /* Test with different sized free space section threshold */ - for(fs_size = 0; fs_size <= TEST_THRESHOLD10; fs_size++) { + /* Close dataset */ + ret = H5Dclose(dset); + CHECK(ret, FAIL, "H5Dclose"); - /* Test with different file space handling strategies */ - for(fs_type = H5F_FILE_SPACE_DEFAULT; fs_type < H5F_FILE_SPACE_NTYPES; H5_INC_ENUM(H5F_file_space_type_t, fs_type)) { + /* Close dataspace */ + ret = H5Sclose(dspace); + CHECK(ret, FAIL, "H5Sclose"); - /* Get a copy of the default file creation property */ - fcpl1 = H5Pcopy(fcpl); - CHECK(fcpl1, FAIL, "H5Pcopy"); + /* Create dataspace for datasets */ + dspace = H5Screate(H5S_SCALAR); + CHECK(dspace, FAIL, "H5Screate"); - /* Set file space strategy and free space section threshold */ - ret = H5Pset_file_space(fcpl1, fs_type, fs_size); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Create datasets in file */ + for(u = 0; u < 10; u++) { + sprintf(name, "Dataset %u", u); + dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset, FAIL, "H5Dcreate2"); - /* Get the file space info from the creation property */ - ret = H5Pget_file_space(fcpl1, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); + ret = H5Dclose(dset); + CHECK(ret, FAIL, "H5Dclose"); + } /* end for */ - /* A 0 value for strategy retains existing strategy in use */ - VERIFY(strategy, (H5F_file_space_type_t)(fs_type ? fs_type : def_type), "H5Pget_file_space"); - /* A 0 value for threshold retains existing threshold in use */ - VERIFY(threshold, (hsize_t)(fs_size ? fs_size : def_size), "H5Pget_file_space"); + /* Close dataspace */ + ret = H5Sclose(dspace); + CHECK(ret, FAIL, "H5Sclose"); - /* Create the file with the specified file space info */ - fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl1, my_fapl); - CHECK(fid1, FAIL, "H5Fcreate"); + /* Close dataset creation property list */ + ret = H5Pclose(dcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Close the file */ - ret = H5Fclose(fid1); - CHECK(ret, FAIL, "H5Fclose"); + /* Delete odd-numbered datasets in file */ + for(u = 0; u < 10; u++) { + sprintf(name, "Dataset %u", u); + if(u % 2) { + ret = H5Ldelete(file, name, H5P_DEFAULT); + CHECK(ret, FAIL, "H5Ldelete"); + } /* end if */ + } /* end for */ - /* Re-open the file */ - fid2 = H5Fopen(filename, H5F_ACC_RDWR, my_fapl); - CHECK(fid2, FAIL, "H5Fopen"); + /* Close file */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); + + /* Re-open the file with read-only permission */ + file = H5Fopen(filename, H5F_ACC_RDONLY, fapl); + CHECK_I(file, "H5Fopen"); + + /* Get the amount of free space in the file */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + + /* Get the total # of free-space sections in the file */ + nall = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, NULL); + CHECK(nall, FAIL, "H5Fget_free_sections"); + + /* Should return failure when nsects is 0 with a nonnull sect_info */ + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, all_sect_info); + VERIFY(nsects, FAIL, "H5Fget_free_sections"); + + /* Retrieve and verify free space info for all the sections */ + HDmemset(all_sect_info, 0, sizeof(all_sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)nall, all_sect_info); + VERIFY(nsects, nall, "H5Fget_free_sections"); + + /* Verify the amount of free-space is correct */ + for(u = 0; u < nall; u++) + total += all_sect_info[u].size; + VERIFY(free_space, total, "H5Fget_free_sections"); + + /* Save the last section's size */ + last_size = all_sect_info[nall-1].size; + + /* Retrieve and verify free space info for -1 sections */ + HDmemset(sect_info, 0, sizeof(sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(nall - 1), sect_info); + VERIFY(nsects, nall, "H5Fget_free_sections"); + + /* Verify the amount of free-space is correct */ + total = 0; + for(u = 0; u < (nall - 1); u++) { + VERIFY(sect_info[u].addr, all_sect_info[u].addr, "H5Fget_free_sections"); + VERIFY(sect_info[u].size, all_sect_info[u].size, "H5Fget_free_sections"); + total += sect_info[u].size; + } + VERIFY(((hsize_t)free_space - last_size), total, "H5Fget_free_sections"); + + /* Retrieve and verify free-space info for +1 sections */ + HDmemset(sect_info, 0, sizeof(sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(nall + 1), sect_info); + VERIFY(nsects, nall, "H5Fget_free_sections"); + + /* Verify amount of free-space is correct */ + total = 0; + for(u = 0; u < nall; u++) { + VERIFY(sect_info[u].addr, all_sect_info[u].addr, "H5Fget_free_sections"); + VERIFY(sect_info[u].size, all_sect_info[u].size, "H5Fget_free_sections"); + total += sect_info[u].size; + } + VERIFY(sect_info[nall].addr, 0, "H5Fget_free_sections"); + VERIFY(sect_info[nall].size, 0, "H5Fget_free_sections"); + VERIFY(free_space, total, "H5Fget_free_sections"); + + HDmemset(meta_sect_info, 0, sizeof(meta_sect_info)); + if(multi_vfd) { + hssize_t ntmp; + + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + continue; + /* Get the # of free-space sections in the file for metadata */ + ntmp = H5Fget_free_sections(file, type, (size_t)0, NULL); + CHECK(ntmp, FAIL, "H5Fget_free_sections"); + + if(ntmp > 0) { + nsects = H5Fget_free_sections(file, type, (size_t)ntmp, &meta_sect_info[nmeta]); + VERIFY(nsects, ntmp, "H5Fget_free_sections"); + nmeta += ntmp; + } + } + } else { + /* Get the # of free-space sections in the file for metadata */ + nmeta = H5Fget_free_sections(file, H5FD_MEM_SUPER, (size_t)0, NULL); + CHECK(nmeta, FAIL, "H5Fget_free_sections"); + + /* Retrieve and verify free-space sections for metadata */ + nsects = H5Fget_free_sections(file, H5FD_MEM_SUPER, (size_t)nmeta, meta_sect_info); + VERIFY(nsects, nmeta, "H5Fget_free_sections"); + } - /* Get the file's creation property */ - fcpl2 = H5Fget_create_plist(fid2); - CHECK(fcpl2, FAIL, "H5Fget_create_plist"); + /* Get the # of free-space sections in the file for raw data */ + nraw = H5Fget_free_sections(file, H5FD_MEM_DRAW, (size_t)0, NULL); + CHECK(nraw, FAIL, "H5Fget_free_sections"); - strategy = H5F_FILE_SPACE_DEFAULT; - threshold = 0; + /* Retrieve and verify free-space sections for raw data */ + HDmemset(raw_sect_info, 0, sizeof(raw_sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DRAW, (size_t)nraw, raw_sect_info); + VERIFY(nsects, nraw, "H5Fget_free_sections"); - /* Get the file space info from the creation property list */ - ret = H5Pget_file_space(fcpl2, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Sum all the free-space sections */ + for(u = 0; u < nmeta; u++) + tmp_tot += meta_sect_info[u].size; - VERIFY(strategy, (H5F_file_space_type_t)(fs_type ? fs_type : def_type), "H5Pget_file_space"); - VERIFY(threshold, (hsize_t)(fs_size ? fs_size : def_size), "H5Pget_file_space"); + for(u = 0; u < nraw; u++) + tmp_tot += raw_sect_info[u].size; - /* Close the file */ - ret = H5Fclose(fid2); - CHECK(ret, FAIL, "H5Fclose"); + /* Verify free-space info */ + VERIFY(nmeta+nraw, nall, "H5Fget_free_sections"); + VERIFY(tmp_tot, total, "H5Fget_free_sections"); - /* Release file-creation template */ - ret = H5Pclose(fcpl1); - CHECK(ret, FAIL, "H5Pclose"); - ret = H5Pclose(fcpl2); - CHECK(ret, FAIL, "H5Pclose"); - } /* end for file space strategy type */ - } /* end for free space threshold */ + /* Closing */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); + ret = H5Pclose(fcpl); + CHECK(fcpl, FAIL, "H5Pclose"); - h5_clean_files(FILESPACE_NAME, my_fapl); + h5_clean_files(FILESPACE_NAME, fapl); + } - } /* end for new/old format */ +} /* end test_sects_freespace() */ - /* Close the file creation property list */ - ret = H5Pclose(fcpl); - CHECK(ret, FAIL, "H5Pclose"); -} /* test_filespace_info() */ /**************************************************************** ** ** test_filespace_compatible(): -** Verify that the branch with file space management enhancement +** Verify that the trunk with the latest file space management ** can open, read and modify 1.6 HDF5 file and 1.8 HDF5 file. -** Also verify the correct file space strategy/threshold in use +** Also verify the correct file space handling information ** and the amount of free space. ** ****************************************************************/ @@ -3614,8 +4330,9 @@ test_filespace_compatible(void) ssize_t nread; /* Number of bytes read in */ unsigned i, j; /* Local index variable */ hssize_t free_space; /* Amount of free-space in the file */ + hbool_t persist; /* Persist free-space or not */ hsize_t threshold; /* Free-space section threshold */ - H5F_file_space_type_t strategy; /* File space handling strategy */ + H5F_fspace_strategy_t strategy; /* File space handling strategy */ herr_t ret; /* Return value */ /* Output message about test being performed */ @@ -3655,14 +4372,16 @@ test_filespace_compatible(void) fcpl = H5Fget_create_plist(fid); CHECK(fcpl, FAIL, "H5Fget_create_plist"); - /* Retrieve the file space handling strategy and threshold */ - ret = H5Pget_file_space(fcpl, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Retrieve the file space info */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* File space handling strategy should be H5F_FILE_SPACE_ALL = 2 */ - /* Free space section threshold should be 1 */ - VERIFY(strategy, 2, "H5Pget_file_space"); - VERIFY(threshold, 1, "H5Pget_file_space"); + /* File space handling strategy should be H5F_FSPACE_STRATEGY_FSM_AGGR */ + /* Persisting free-space should be FALSE */ + /* Free-space section threshold should be 1 */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); /* Generate raw data */ for(i = 0; i < 100; i++) @@ -3715,6 +4434,383 @@ test_filespace_compatible(void) /**************************************************************** ** +** test_filespace_1.10.0_compatible(): +** Verify that the latest file space management can open, read and +** modify 1.10.0 HDF5 files : +** h5fc_ext1_i.h5: H5F_FILE_SPACE_ALL, default threshold; has superblock extension but no fsinfo message +** h5fc_ext1_f.h5: H5F_FILE_SPACE_ALL_PERSIST, default threshold; has superblock extension with fsinfo message +** h5fc_ext2_if.h5: H5F_FILE_SPACE_ALL, non-default threshold; has superblock extension with fsinfo message +** h5fc_ext2_sf.h5: H5F_FILE_SPACE_VFD, default threshold; has superblock extension with fsinfo message +** h5fc_ext3_isf.h5: H5F_FILE_SPACE_AGGR_VFD, default threshold; has superblock extension with fsinfo message +** h5fc_ext_none.h5: H5F_FILE_SPACE_ALL, default threshold; without superblock extension +** The above files are copied from release 1.10.0 tools/h5format_convert/testfiles. +** +****************************************************************/ +static void +test_filespace_1_10_0_compatible(void) +{ + hid_t fid = -1; /* File id */ + hid_t did = -1; /* Dataset id */ + hid_t fcpl; /* File creation property list */ + hbool_t persist; /* Persist free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + H5F_fspace_strategy_t strategy; /* File space handling strategy */ + int wbuf[24]; /* Buffer for dataset data */ + int rdbuf[24]; /* Buffer for dataset data */ + int status; /* Status from copying the existing file */ + unsigned i, j; /* Local index variable */ + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("File space compatibility testing for 1.10.0 files\n")); + + for(j = 0; j < NELMTS(OLD_1_10_0_FILENAME); j++) { + /* Make a copy of the test file */ + status = h5_make_local_copy(OLD_1_10_0_FILENAME[j], FILE5); + CHECK(status, FAIL, "h5_make_local_copy"); + + /* Open the temporary test file */ + fid = H5Fopen(FILE5, H5F_ACC_RDWR, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + /* Get the file's file creation property list */ + fcpl = H5Fget_create_plist(fid); + CHECK(fcpl, FAIL, "H5Fget_create_plist"); + + /* Retrieve the file space info */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + switch(j) { + case 0: + VERIFY(strategy, H5F_FILE_SPACE_STRATEGY_DEF, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/DSET_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 1: + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, TRUE, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/DSET_NDATA_BT2", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 2: + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, 2, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/DSET_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 3: + VERIFY(strategy, H5F_FSPACE_STRATEGY_NONE, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 4: + VERIFY(strategy, H5F_FSPACE_STRATEGY_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_FA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + case 5: + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + default: + break; + } + + /* Close the plist */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Re-Open the file */ + fid = H5Fopen(FILE5, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + switch(j) { + case 0: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/DSET_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 1: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/DSET_NDATA_BT2", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 2: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/DSET_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 3: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 4: + + /* Open and read the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_FA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 5: + + /* Open and read the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + default: + break; + } + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + } /* end for */ + +} /* test_filespace_1_10_0_compatible */ + +/**************************************************************** +** +** test_filespace_round_compatible(): +** Verify that the trunk can open, read and modify these files-- +** 1) They are initially created (via gen_filespace.c) in the trunk +** with combinations of file space strategies, default/non-default +** threshold, and file spacing paging enabled/disbled. +** The library creates the file space info message with +** "mark if unknown" in these files. +** 2) They are copied to the 1.8 branch, and are opened/read/modified +** there via test_filespace_compatible() in test/tfile.c. +** The 1.8 library marks the file space info message as "unknown" +** in these files. +** 3) They are then copied back from the 1.8 branch to the trunk for +** compatibility testing via this routine. +** 4) Upon encountering the file space info message which is marked +** as "unknown", the library will use the default file space management +** from then on: non-persistent free-space managers, default threshold, +** and non-paging file space. +** +****************************************************************/ +static void +test_filespace_round_compatible(void) +{ + hid_t fid = -1; /* File id */ + hid_t fcpl = -1; /* File creation property list ID */ + unsigned j; /* Local index variable */ + H5F_fspace_strategy_t strategy; /* File space strategy */ + hbool_t persist; /* Persist free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + hssize_t free_space; /* Amount of free space in the file */ + int status; /* Status from copying the existing file */ + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("File space compatibility testing for files from trunk to 1_8 to trunk\n")); + + for(j = 0; j < NELMTS(FSPACE_FILENAMES); j++) { + /* Make a copy of the test file */ + status = h5_make_local_copy(FSPACE_FILENAMES[j], FILE5); + CHECK(status, FAIL, "h5_make_local_copy"); + + /* Open the temporary test file */ + fid = H5Fopen(FILE5, H5F_ACC_RDWR, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + /* Get the file's creation property list */ + fcpl = H5Fget_create_plist(fid); + CHECK(fcpl, FAIL, "H5Fget_create_plist"); + + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + + /* There should not be any free space in the file */ + free_space = H5Fget_freespace(fid); + CHECK(free_space, FAIL, "H5Fget_freespace"); + VERIFY(free_space, (hssize_t)0, "H5Fget_freespace"); + + /* Closing */ + ret = H5Fclose(fid); + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Fclose"); + } /* end for */ + +} /* test_filespace_round_compatible */ + + +/**************************************************************** +** ** test_libver_bounds_real(): ** Verify that a file created and modified with the ** specified libver bounds has the specified object header @@ -3951,6 +5047,9 @@ test_deprec(void) { hid_t file; /* File IDs for old & new files */ hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File creation property list */ + hid_t new_fapl; + hsize_t align; unsigned super; /* Superblock version # */ unsigned freelist; /* Free list version # */ unsigned stab; /* Symbol table entry version # */ @@ -4003,17 +5102,25 @@ test_deprec(void) CHECK(fcpl, FAIL, "H5Pcreate"); /* Set a property in the FCPL that will push the superblock version up */ - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_VFD, (hsize_t)0); - CHECK(ret, FAIL, "H5Pset_file_space"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + fapl = H5Pcreate(H5P_FILE_ACCESS); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_alignment"); /* Creating a file with the non-default file creation property list should * create a version 2 superblock */ /* Create file with custom file creation property list */ - file= H5Fcreate(FILE1, H5F_ACC_TRUNC , fcpl, H5P_DEFAULT); + file= H5Fcreate(FILE1, H5F_ACC_TRUNC , fcpl, fapl); CHECK(file, FAIL, "H5Fcreate"); + new_fapl = H5Fget_access_plist(file); + H5Pget_alignment(new_fapl, NULL, &align); + /* Close FCPL */ ret=H5Pclose(fcpl); CHECK(ret, FAIL, "H5Pclose"); @@ -4021,7 +5128,7 @@ test_deprec(void) /* Get the file's version information */ ret = H5Fget_info1(file, &finfo); CHECK(ret, FAIL, "H5Fget_info1"); - VERIFY(finfo.super_ext_size, 40,"H5Fget_info1"); + VERIFY(finfo.super_ext_size, 152,"H5Fget_info1"); VERIFY(finfo.sohm.hdr_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.index_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.heap_size, 0,"H5Fget_info1"); @@ -4053,7 +5160,7 @@ test_deprec(void) /* Get the file's version information */ ret = H5Fget_info1(file, &finfo); CHECK(ret, FAIL, "H5Fget_info1"); - VERIFY(finfo.super_ext_size, 40,"H5Fget_info1"); + VERIFY(finfo.super_ext_size, 152,"H5Fget_info1"); VERIFY(finfo.sohm.hdr_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.index_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.heap_size, 0,"H5Fget_info1"); @@ -4077,6 +5184,81 @@ test_deprec(void) /* Close file */ ret=H5Fclose(file); CHECK(ret, FAIL, "H5Fclose"); + + { /* Test deprecated H5Pget/set_file_space() */ + + H5F_file_space_type_t old_strategy; + hsize_t old_threshold; + hid_t fid; + hid_t ffcpl; + + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL, "H5Pget_file_space"); + VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); + + /* Set file space strategy and free space section threshold */ + ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); + CHECK(ret, FAIL, "H5Pget_file_space"); + + /* Get the file space info from the creation property */ + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); + + ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_DEFAULT, (hsize_t)3); + CHECK(ret, FAIL, "H5Pget_file_space"); + + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); + + /* Create a file */ + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC , fcpl, H5P_DEFAULT); + CHECK(file, FAIL, "H5Fcreate"); + + old_strategy = H5F_FILE_SPACE_DEFAULT; + old_threshold = 0; + ffcpl = H5Fget_create_plist(fid); + ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); + + /* Close file */ + ret=H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + ret = H5Pclose(ffcpl); + CHECK(ret, FAIL, "H5Pclose"); + + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Reopen the file */ + fid = H5Fopen(FILE1, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fcreate"); + + old_strategy = H5F_FILE_SPACE_DEFAULT; + old_threshold = 0; + ffcpl = H5Fget_create_plist(fid); + ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); + + ret = H5Pclose(ffcpl); + CHECK(ret, FAIL, "H5Pclose"); + + ret=H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + } + } /* test_deprec */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ @@ -4088,9 +5270,16 @@ test_deprec(void) void test_file(void) { + const char *env_h5_drvr; /* File Driver value from environment */ + /* Output message about test being performed */ MESSAGE(5, ("Testing Low-Level File I/O\n")); + /* Get the VFD to use */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + test_file_create(); /* Test file creation(also creation templates)*/ test_file_open(); /* Test file opening */ test_file_reopen(); /* Test file reopening */ @@ -4099,7 +5288,6 @@ test_file(void) test_get_obj_ids(); /* Test H5Fget_obj_ids for Jira Issue 8528 */ test_file_perm(); /* Test file access permissions */ test_file_perm2(); /* Test file access permission again */ - test_file_freespace(); /* Test file free space information */ test_file_ishdf5(); /* Test detecting HDF5 files correctly */ test_file_open_dot(); /* Test opening objects with "." for a name */ test_file_open_overlap(); /* Test opening files in an overlapping manner */ @@ -4114,9 +5302,21 @@ test_file(void) test_cached_stab_info(); /* Tests that files are created with cached stab info in the superblock */ test_rw_noupdate(); /* Test to ensure that RW permissions don't write the file unless dirtied */ test_userblock_alignment(); /* Tests that files created with a userblock and alignment interact properly */ - test_filespace_sects(); /* Test file free space section information */ - test_filespace_info(); /* Test file creation public routines:H5Pget/set_file_space */ + test_userblock_alignment_paged(); /* Tests files created with a userblock and alignment (via paged aggregation) interact properly */ + test_filespace_info(env_h5_drvr); /* Test file creation public routines: */ + /* H5Pget/set_file_space_strategy() & H5Pget/set_file_space_page_size() */ + /* Skipped testing for multi/split drivers */ + test_file_freespace(env_h5_drvr); /* Test file public routine H5Fget_freespace() */ + /* Skipped testing for multi/split drivers */ + /* Setup for multi/split drivers are there already */ + test_sects_freespace(env_h5_drvr, TRUE); /* Test file public routine H5Fget_free_sections() for new format */ + /* Skipped testing for multi/split drivers */ + /* Setup for multi/split drivers are there already */ + test_sects_freespace(env_h5_drvr, FALSE); /* Test file public routine H5Fget_free_sections() */ + /* Skipped testing for multi/split drivers */ test_filespace_compatible(); /* Test compatibility for file space management */ + test_filespace_round_compatible(); /* Testing file space compatibility for files from trunk to 1_8 to trunk */ + test_filespace_1_10_0_compatible(); /* Testing file space compatibility for files from release 1.10.0 */ test_libver_bounds(); /* Test compatibility for file space management */ test_libver_macros(); /* Test the macros for library version comparison */ test_libver_macros2(); /* Test the macros for library version comparison */ diff --git a/test/tmisc.c b/test/tmisc.c index f93500d..6012588 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -1819,8 +1819,9 @@ test_misc11(void) unsigned sym_lk; /* Symbol table B-tree leaf 'K' value */ unsigned nindexes; /* Shared message number of indexes */ H5F_info2_t finfo; /* global information about file */ - H5F_file_space_type_t strategy; /* File/free space strategy */ + H5F_fspace_strategy_t strategy; /* File space strategy */ hsize_t threshold; /* Free-space section threshold */ + hbool_t persist; /* To persist free-space or not */ herr_t ret; /* Generic return value */ /* Output message about test being performed */ @@ -1878,11 +1879,11 @@ test_misc11(void) ret=H5Pset_shared_mesg_nindexes(fcpl,MISC11_NINDEXES); CHECK(ret, FAIL, "H5Pset_shared_mesg"); - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_VFD, (hsize_t)0); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); CHECK(ret, FAIL, "H5Pset_file_space"); /* Creating a file with the non-default file creation property list should - * create a version 1 superblock + * create a version 2 superblock */ /* Create file with custom file creation property list */ @@ -1942,10 +1943,11 @@ test_misc11(void) CHECK(ret, FAIL, "H5Pget_shared_mesg_nindexes"); VERIFY(nindexes, MISC11_NINDEXES, "H5Pget_shared_mesg_nindexes"); - ret = H5Pget_file_space(fcpl, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); - VERIFY(strategy, 4, "H5Pget_file_space"); - VERIFY(threshold, 1, "H5Pget_file_space"); + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + VERIFY(strategy, 3, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); /* Close file */ ret=H5Fclose(file); diff --git a/testpar/Makefile.am b/testpar/Makefile.am index 4fe0ba8..3889630 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -25,7 +25,7 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test # Test programs. These are our main targets. # -TEST_PROG_PARA=t_mpi testphdf5 t_cache t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame +TEST_PROG_PARA=t_mpi testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame check_PROGRAMS = $(TEST_PROG_PARA) diff --git a/testpar/t_cache_image.c b/testpar/t_cache_image.c new file mode 100644 index 0000000..7283fa7 --- /dev/null +++ b/testpar/t_cache_image.c @@ -0,0 +1,2161 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: John Mainzer + * 7/13/15 + * + * This file contains tests specific to the cache image + * feature implemented in H5C.c + */ +#include "h5test.h" +#include "testphdf5.h" +#include "testpar.h" +#include "cache_common.h" +#include "genall5.h" + +#define CHUNK_SIZE 10 +#define DSET_SIZE (40 * CHUNK_SIZE) +#define MAX_NUM_DSETS 256 + + +/* global variable declarations: */ + + +const char *FILENAMES[] = { + "t_cache_image_00", + "t_cache_image_01", + NULL +}; + +/* local utility function declarations */ + +static void create_data_sets(hid_t file_id, int min_dset, int max_dset); +static void delete_data_sets(hid_t file_id, int min_dset, int max_dset); + +static void open_hdf5_file(const hbool_t create_file, + const hbool_t mdci_sbem_expected, const hbool_t read_only, + const hbool_t set_mdci_fapl, const hbool_t config_fsm, + const char * hdf_file_name, const unsigned cache_image_flags, + hid_t * file_id_ptr, H5F_t ** file_ptr_ptr, H5C_t ** cache_ptr_ptr, + MPI_Comm comm, MPI_Info info, int l_facc_type, + const hbool_t all_coll_metadata_ops, const hbool_t coll_metadata_write, + const int md_write_strat); + +static void verify_data_sets(hid_t file_id, int min_dset, int max_dset); + +/* local test function declarations */ + +static hbool_t parse_flags(int argc, char * argv[], hbool_t * setup_ptr, + hbool_t display); +static void usage(void); +static unsigned construct_test_file(int test_file_index); + + +/* top level test function declarations */ +static unsigned verify_cache_image_RW(int file_name_id, + int md_write_strat, int mpi_rank); + + +/****************************************************************************/ +/***************************** Utility Functions ****************************/ +/****************************************************************************/ + +/*------------------------------------------------------------------------- + * Function: construct_test_file() + * + * Purpose: This function attempts to mimic the typical "poor man's + * parallel use case in which the file is passed between + * processes, each of which open the file, write some data, + * close the file, and then pass control on to the next + * process. + * + * In this case, we create one group for each process, and + * populate it with a "zoo" of HDF5 objects selected to + * (ideally) exercise all HDF5 on disk data structures. + * + * The end result is a test file used verify that PHDF5 + * can open a file with a cache image. + * + * Cycle of operation + * + * 1) Create a HDF5 file with the cache image FAPL entry. + * + * Verify that the cache is informed of the cache image + * FAPL entry. + * + * Set all cache image flags, forcing full functionality. + * + * 2) Create a data set in the file. + * + * 3) Close the file. + * + * 4) Open the file. + * + * Verify that the metadata cache is instructed to load + * the metadata cache image. + * + * 5) Create a data set in the file. + * + * 6) Close the file. If enough datasets have been created + * goto 7. Otherwise return to 4. + * + * 7) Open the file R/O. + * + * Verify that the file contains a metadata cache image + * superblock extension message. + * + * 8) Verify all data sets. + * + * Verify that the cache image has been loaded. + * + * 9) close the file. + * + * Return: void + * + * Programmer: John Mainzer + * 1/25/17 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +construct_test_file(int test_file_index) +{ + const char * fcn_name = "construct_test_file()"; + char filename[512]; + hbool_t show_progress = FALSE; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + int cp = 0; + int min_dset = 0; + int max_dset = 0; + MPI_Comm dummy_comm = MPI_COMM_WORLD; + MPI_Info dummy_info = MPI_INFO_NULL; + + pass = TRUE; + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + HDassert(FILENAMES[test_file_index]); + + if ( h5_fixname(FILENAMES[test_file_index], H5P_DEFAULT, + filename, sizeof(filename)) + == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 1) Create a HDF5 file with the cache image FAPL entry. + * + * Verify that the cache is informed of the cache image FAPL entry. + * + * Set flags forcing full function of the cache image feature. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ TRUE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ TRUE, + /* config_fsm */ TRUE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ dummy_comm, + /* info */ dummy_info, + /* l_facc_type */ 0, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ 0); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Create a data set in the file. */ + + if ( pass ) { + + create_data_sets(file_id, min_dset++, max_dset++); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(1)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 3) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + while ( ( pass ) && ( max_dset < MAX_NUM_DSETS ) ) + { + + /* 4) Open the file. + * + * Verify that the metadata cache is instructed to load the + * metadata cache image. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ FALSE, + /* set_mdci_fapl */ TRUE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ dummy_comm, + /* info */ dummy_info, + /* l_facc_type */ 0, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ 0); + } + + if ( show_progress ) + HDfprintf(stdout, "%s:L1 cp = %d, max_dset = %d, pass = %d.\n", + fcn_name, cp, max_dset, pass); + + + /* 5) Create a data set in the file. */ + + if ( pass ) { + + create_data_sets(file_id, min_dset++, max_dset++); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded == 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block not loaded(1)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s:L2 cp = %d, max_dset = %d, pass = %d.\n", + fcn_name, cp + 1, max_dset, pass); + + + /* 6) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s:L3 cp = %d, max_dset = %d, pass = %d.\n", + fcn_name, cp + 2, max_dset, pass); + } /* end while */ + cp += 3; + + + /* 7) Open the file R/O. + * + * Verify that the file contains a metadata cache image + * superblock extension message. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ TRUE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ dummy_comm, + /* info */ dummy_info, + /* l_facc_type */ 0, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ 0); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 8) Open and close all data sets. + * + * Verify that the cache image has been loaded. + */ + + if ( pass ) { + + verify_data_sets(file_id, 0, max_dset - 1); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded == 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block not loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 9) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + } + } + + return !pass; + +} /* construct_test_file() */ + + +/*------------------------------------------------------------------------- + * Function: create_data_sets() + * + * Purpose: If pass is TRUE on entry, create the specified data sets + * in the indicated file. + * + * Data sets and their contents must be well know, as we + * will verify that they contain the expected data later. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 7/15/15 + * + * Modifications: + * + * Added min_dset and max_dset parameters and supporting + * code. This allows the caller to specify a range of + * datasets to create. + * JRM -- 8/20/15 + * + *------------------------------------------------------------------------- + */ + +static void +create_data_sets(hid_t file_id, int min_dset, int max_dset) +{ + const char * fcn_name = "create_data_sets()"; + char dset_name[64]; + hbool_t show_progress = FALSE; + hbool_t valid_chunk; + hbool_t verbose = FALSE; + int cp = 0; + int i, j, k, l, m; + int data_chunk[CHUNK_SIZE][CHUNK_SIZE]; + herr_t status; + hid_t dataspace_id = -1; + hid_t filespace_ids[MAX_NUM_DSETS]; + hid_t memspace_id = -1; + hid_t dataset_ids[MAX_NUM_DSETS]; + hid_t properties = -1; + hsize_t dims[2]; + hsize_t a_size[2]; + hsize_t offset[2]; + hsize_t chunk_size[2]; + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + HDassert(0 <= min_dset); + HDassert(min_dset <= max_dset); + HDassert(max_dset < MAX_NUM_DSETS); + + /* create the datasets */ + + if ( pass ) { + + i = min_dset; + + while ( ( pass ) && ( i <= max_dset ) ) + { + /* create a dataspace for the chunked dataset */ + dims[0] = DSET_SIZE; + dims[1] = DSET_SIZE; + dataspace_id = H5Screate_simple(2, dims, NULL); + + if ( dataspace_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Screate_simple() failed."; + } + + /* set the dataset creation plist to specify that the raw data is + * to be partioned into 10X10 element chunks. + */ + + if ( pass ) { + + chunk_size[0] = CHUNK_SIZE; + chunk_size[1] = CHUNK_SIZE; + properties = H5Pcreate(H5P_DATASET_CREATE); + + if ( properties < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pcreate() failed."; + } + } + + if ( pass ) { + + if ( H5Pset_chunk(properties, 2, chunk_size) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_chunk() failed."; + } + } + + /* create the dataset */ + if ( pass ) { + + sprintf(dset_name, "/dset%03d", i); + dataset_ids[i] = H5Dcreate2(file_id, dset_name, H5T_STD_I32BE, + dataspace_id, H5P_DEFAULT, + properties, H5P_DEFAULT); + + if ( dataset_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dcreate() failed."; + } + } + + /* get the file space ID */ + if ( pass ) { + + filespace_ids[i] = H5Dget_space(dataset_ids[i]); + + if ( filespace_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dget_space() failed."; + } + } + + i++; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* create the mem space to be used to read and write chunks */ + if ( pass ) { + + dims[0] = CHUNK_SIZE; + dims[1] = CHUNK_SIZE; + memspace_id = H5Screate_simple(2, dims, NULL); + + if ( memspace_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Screate_simple() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* select in memory hyperslab */ + if ( pass ) { + + offset[0] = 0; /*offset of hyperslab in memory*/ + offset[1] = 0; + a_size[0] = CHUNK_SIZE; /*size of hyperslab*/ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, offset, NULL, + a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sselect_hyperslab() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* initialize all datasets on a round robin basis */ + i = 0; + while ( ( pass ) && ( i < DSET_SIZE ) ) + { + j = 0; + while ( ( pass ) && ( j < DSET_SIZE ) ) + { + m = min_dset; + while ( ( pass ) && ( m <= max_dset ) ) + { + /* initialize the slab */ + for ( k = 0; k < CHUNK_SIZE; k++ ) + { + for ( l = 0; l < CHUNK_SIZE; l++ ) + { + data_chunk[k][l] = (DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l; + } + } + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; /*offset of hyperslab in file*/ + offset[1] = (hsize_t)j; + a_size[0] = CHUNK_SIZE; /*size of hyperslab*/ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk H5Sselect_hyperslab() failed."; + } + + /* write the chunk to file */ + status = H5Dwrite(dataset_ids[m], H5T_NATIVE_INT, memspace_id, + filespace_ids[m], H5P_DEFAULT, data_chunk); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dwrite() failed."; + } + m++; + } + j += CHUNK_SIZE; + } + + i += CHUNK_SIZE; + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* read data from data sets and validate it */ + i = 0; + while ( ( pass ) && ( i < DSET_SIZE ) ) + { + j = 0; + while ( ( pass ) && ( j < DSET_SIZE ) ) + { + m = min_dset; + while ( ( pass ) && ( m <= max_dset ) ) + { + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; /* offset of hyperslab in file */ + offset[1] = (hsize_t)j; + a_size[0] = CHUNK_SIZE; /* size of hyperslab */ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + + /* read the chunk from file */ + if ( pass ) { + + status = H5Dread(dataset_ids[m], H5T_NATIVE_INT, + memspace_id, filespace_ids[m], + H5P_DEFAULT, data_chunk); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + } + + /* validate the slab */ + if ( pass ) { + + valid_chunk = TRUE; + for ( k = 0; k < CHUNK_SIZE; k++ ) + { + for ( l = 0; l < CHUNK_SIZE; l++ ) + { + if ( data_chunk[k][l] + != + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l) ) { + + valid_chunk = FALSE; + + if ( verbose ) { + + HDfprintf(stdout, + "data_chunk[%0d][%0d] = %0d, expect %0d.\n", + k, l, data_chunk[k][l], + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l)); + HDfprintf(stdout, + "m = %d, i = %d, j = %d, k = %d, l = %d\n", + m, i, j, k, l); + } + } + } + } + + if ( ! valid_chunk ) { + + pass = FALSE; + failure_mssg = "slab validation failed."; + + if ( verbose ) { + + fprintf(stdout, + "Chunk (%0d, %0d) in /dset%03d is invalid.\n", + i, j, m); + } + } + } + m++; + } + j += CHUNK_SIZE; + } + i += CHUNK_SIZE; + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* close the file spaces */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Sclose(filespace_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose() failed."; + } + i++; + } + + + /* close the datasets */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Dclose(dataset_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dclose() failed."; + } + i++; + } + + /* close the mem space */ + if ( pass ) { + + if ( H5Sclose(memspace_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose(memspace_id) failed."; + } + } + + return; + +} /* create_data_sets() */ + + +/*------------------------------------------------------------------------- + * Function: delete_data_sets() + * + * Purpose: If pass is TRUE on entry, verify and then delete the + * dataset(s) indicated by min_dset and max_dset in the + * indicated file. + * + * Data sets and their contents must be well know, as we + * will verify that they contain the expected data later. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 10/31/16 + * + * Modifications: + * + * None. + * JRM -- 8/20/15 + * + *------------------------------------------------------------------------- + */ + +static void +delete_data_sets(hid_t file_id, int min_dset, int max_dset) +{ + const char * fcn_name = "delete_data_sets()"; + char dset_name[64]; + hbool_t show_progress = FALSE; + int cp = 0; + int i; + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + HDassert(0 <= min_dset); + HDassert(min_dset <= max_dset); + HDassert(max_dset < MAX_NUM_DSETS); + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* first, verify the contents of the target dataset(s) */ + verify_data_sets(file_id, min_dset, max_dset); + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* now delete the target datasets */ + if ( pass ) { + + i = min_dset; + + while ( ( pass ) && ( i <= max_dset ) ) + { + sprintf(dset_name, "/dset%03d", i); + + if ( H5Ldelete(file_id, dset_name, H5P_DEFAULT) < 0) { + + pass = FALSE; + failure_mssg = "H5Ldelete() failed."; + } + + i++; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + return; + +} /* delete_data_sets() */ + + +/*------------------------------------------------------------------------- + * Function: open_hdf5_file() + * + * Purpose: If pass is true on entry, create or open the specified HDF5 + * and test to see if it has a metadata cache image superblock + * extension message. + * + * Set pass to FALSE and issue a suitable failure + * message if either the file contains a metadata cache image + * superblock extension and mdci_sbem_expected is TRUE, or + * vise versa. + * + * If mdci_sbem_expected is TRUE, also verify that the metadata + * cache has been advised of this. + * + * If read_only is TRUE, open the file read only. Otherwise + * open the file read/write. + * + * If set_mdci_fapl is TRUE, set the metadata cache image + * FAPL entry when opening the file, and verify that the + * metadata cache is notified. + * + * If config_fsm is TRUE, setup the persistant free space + * manager. Note that this flag may only be set if + * create_file is also TRUE. + * + * Return pointers to the cache data structure and file data + * structures. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 7/14/15 + * + * Modifications: + * + * Modified function to handle parallel file creates / opens. + * + * JRM -- 2/1/17 + * + *------------------------------------------------------------------------- + */ + +static void +open_hdf5_file(const hbool_t create_file, + const hbool_t mdci_sbem_expected, + const hbool_t read_only, + const hbool_t set_mdci_fapl, + const hbool_t config_fsm, + const char * hdf_file_name, + const unsigned cache_image_flags, + hid_t * file_id_ptr, + H5F_t ** file_ptr_ptr, + H5C_t ** cache_ptr_ptr, + MPI_Comm comm, + MPI_Info info, + int l_facc_type, + const hbool_t all_coll_metadata_ops, + const hbool_t coll_metadata_write, + const int md_write_strat) +{ + const char * fcn_name = "open_hdf5_file()"; + hbool_t show_progress = FALSE; + hbool_t verbose = FALSE; + int cp = 0; + hid_t fapl_id = -1; + hid_t fcpl_id = -1; + hid_t file_id = -1; + herr_t result; + H5F_t * file_ptr = NULL; + H5C_t * cache_ptr = NULL; + H5C_cache_image_ctl_t image_ctl; + H5AC_cache_image_config_t cache_image_config = { + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, + TRUE, + FALSE, + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; + + if ( pass ) + { + /* opening the file both read only and with a cache image + * requested is a contradiction. We resolve it by ignoring + * the cache image request silently. + */ + if ( ( create_file && mdci_sbem_expected ) || + ( create_file && read_only ) || + ( config_fsm && !create_file ) || + ( hdf_file_name == NULL ) || + ( ( set_mdci_fapl ) && ( cache_image_flags == 0 ) ) || + ( ( set_mdci_fapl ) && + ( (cache_image_flags & ~H5C_CI__ALL_FLAGS) != 0 ) ) || + ( file_id_ptr == NULL ) || + ( file_ptr_ptr == NULL ) || + ( cache_ptr_ptr == NULL ) || + ( l_facc_type != (l_facc_type & (FACC_MPIO)) ) ) { + + failure_mssg = + "Bad param(s) on entry to open_hdf5_file().\n"; + + pass = FALSE; + } else if ( verbose ) { + + HDfprintf(stdout, "%s: HDF file name = \"%s\".\n", + fcn_name, hdf_file_name); + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* create a file access propertly list. */ + if ( pass ) { + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + + if ( fapl_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pcreate() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* call H5Pset_libver_bounds() on the fapl_id */ + if ( pass ) { + + if ( H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) + < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_libver_bounds() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* get metadata cache image config -- verify that it is the default */ + if ( pass ) { + + result = H5Pget_mdc_image_config(fapl_id, &cache_image_config); + + if ( result < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pget_mdc_image_config() failed.\n"; + } + + if ( ( cache_image_config.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( cache_image_config.generate_image != FALSE ) || + ( cache_image_config.save_resize_status != FALSE ) || + ( cache_image_config.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) ) { + + pass = FALSE; + failure_mssg = "Unexpected default cache image config.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* set metadata cache image fapl entry if indicated */ + if ( ( pass ) && ( set_mdci_fapl ) ) { + + /* set cache image config fields to taste */ + cache_image_config.generate_image = TRUE; + cache_image_config.save_resize_status = FALSE; + cache_image_config.entry_ageout = H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE; + + result = H5Pset_mdc_image_config(fapl_id, &cache_image_config); + + if ( result < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_mdc_image_config() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* setup the persistant free space manager if indicated */ + if ( ( pass ) && ( config_fsm ) ) { + + fcpl_id = H5Pcreate(H5P_FILE_CREATE); + + if ( fcpl_id <= 0 ) { + + pass = FALSE; + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed."; + } + } + + if ( ( pass ) && ( config_fsm ) ) { + + if ( H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, + TRUE, (hsize_t)1) == FAIL ) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_strategy() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + /* set Parallel access with communicator */ + if ( H5Pset_fapl_mpio(fapl_id, comm, info) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_fapl_mpio() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + if (H5Pset_all_coll_metadata_ops(fapl_id, all_coll_metadata_ops) < 0) { + + pass = FALSE; + failure_mssg = "H5Pset_all_coll_metadata_ops() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + if ( H5Pset_coll_metadata_write(fapl_id, coll_metadata_write) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_coll_metadata_write() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + /* set the desired parallel metadata write strategy */ + H5AC_cache_config_t mdc_config; + + mdc_config.version = H5C__CURR_AUTO_SIZE_CTL_VER; + + if ( H5Pget_mdc_config(fapl_id, &mdc_config) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pget_mdc_config() failed.\n"; + } + + mdc_config.metadata_write_strategy = md_write_strat; + + if ( H5Pset_mdc_config(fapl_id, &mdc_config) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_mdc_config() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* open the file */ + if ( pass ) { + + if ( create_file ) { + + if ( fcpl_id != -1 ) + + file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, + fcpl_id, fapl_id); + else + + file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, + H5P_DEFAULT, fapl_id); + + } else { + + if ( read_only ) + + file_id = H5Fopen(hdf_file_name, H5F_ACC_RDONLY, fapl_id); + + else + + file_id = H5Fopen(hdf_file_name, H5F_ACC_RDWR, fapl_id); + } + + if ( file_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fcreate() or H5Fopen() failed.\n"; + + } else { + + file_ptr = (struct H5F_t *)H5I_object_verify(file_id, H5I_FILE); + + if ( file_ptr == NULL ) { + + pass = FALSE; + failure_mssg = "Can't get file_ptr."; + + if ( verbose ) { + HDfprintf(stdout, "%s: Can't get file_ptr.\n", fcn_name); + } + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* get a pointer to the files internal data structure and then + * to the cache structure + */ + if ( pass ) { + + if ( file_ptr->shared->cache == NULL ) { + + pass = FALSE; + failure_mssg = "can't get cache pointer(1).\n"; + + } else { + + cache_ptr = file_ptr->shared->cache; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* verify expected metadata cache status */ + + /* get the cache image control structure from the cache, and verify + * that it contains the expected values. + * + * Then set the flags in this structure to the specified value. + */ + if ( pass ) { + + if ( H5C_get_cache_image_config(cache_ptr, &image_ctl) < 0 ) { + + pass = FALSE; + failure_mssg = "error returned by H5C_get_cache_image_config()."; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + if ( set_mdci_fapl ) { + + if ( read_only ) { + + if ( ( image_ctl.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( image_ctl.generate_image != FALSE ) || + ( image_ctl.save_resize_status != FALSE ) || + ( image_ctl.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) || + ( image_ctl.flags != H5C_CI__ALL_FLAGS ) ) { + + pass = FALSE; + failure_mssg = "Unexpected image_ctl values(1).\n"; + } + } else { + + if ( ( image_ctl.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( image_ctl.generate_image != TRUE ) || + ( image_ctl.save_resize_status != FALSE ) || + ( image_ctl.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) || + ( image_ctl.flags != H5C_CI__ALL_FLAGS ) ) { + + pass = FALSE; + failure_mssg = "Unexpected image_ctl values(2).\n"; + } + } + } else { + + if ( ( image_ctl.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( image_ctl.generate_image != FALSE ) || + ( image_ctl.save_resize_status != FALSE ) || + ( image_ctl.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) || + ( image_ctl.flags != H5C_CI__ALL_FLAGS ) ) { + + pass = FALSE; + failure_mssg = "Unexpected image_ctl values(3).\n"; + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( set_mdci_fapl ) ) { + + image_ctl.flags = cache_image_flags; + + if ( H5C_set_cache_image_config(file_ptr, cache_ptr, &image_ctl) < 0 ) { + + pass = FALSE; + failure_mssg = "error returned by H5C_set_cache_image_config()."; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + if ( cache_ptr->close_warning_received == TRUE ) { + + pass = FALSE; + failure_mssg = "Unexpected value of close_warning_received.\n"; + } + + if ( mdci_sbem_expected ) { + + if ( read_only ) { + + if ( ( cache_ptr->load_image != TRUE ) || + ( cache_ptr->delete_image != FALSE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message not present?\n"; + } + } else { + + if ( ( cache_ptr->load_image != TRUE ) || + ( cache_ptr->delete_image != TRUE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message not present?\n"; + } + } + } else { + + if ( ( cache_ptr->load_image == TRUE ) || + ( cache_ptr->delete_image == TRUE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message present?\n"; + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + *file_id_ptr = file_id; + *file_ptr_ptr = file_ptr; + *cache_ptr_ptr = cache_ptr; + } + + if ( show_progress ) { + HDfprintf(stdout, "%s: cp = %d, pass = %d -- exiting.\n", + fcn_name, cp++, pass); + + if ( ! pass ) + HDfprintf(stdout, "%s: failure_mssg = %s\n", + fcn_name, failure_mssg); + } + + return; + +} /* open_hdf5_file() */ + + +/*------------------------------------------------------------------------- + * Function: parse_flags + * + * Purpose: Parse the flags passed to this program, and load the + * values into the supplied field. + * + * Return: Success: 1 + * Failure: 0 + * + * Programmer: J Mainzer + * 4/28/11 + * + *------------------------------------------------------------------------- + */ +hbool_t +parse_flags(int argc, char * argv[], hbool_t * setup_ptr, hbool_t display) +{ + const char * fcn_name = "parse_flags()"; + const char * (ops[]) = {"setup"}; + int success = TRUE; + + if ( setup_ptr == NULL ) { + + success = FALSE; + HDfprintf(stdout, "%s: bad arg(s) on entry.\n", fcn_name); + } + + if ( ( success ) && + ( ( argc < 1 ) || ( argc > 2 ) ) ) { + + success = FALSE; + usage(); + } + + if ( success ) { + + if ( argc == 2 ) { + + if ( strcmp(argv[1], ops[0]) == 0 ) { + + *setup_ptr = TRUE; + + } else { + + success = FALSE; + usage(); + } + } else { + + *setup_ptr = FALSE; + } + } + + if ( ( success ) && ( display ) ) { + + if ( *setup_ptr ) + HDfprintf(stdout, "t_cache_image setup\n"); + else + HDfprintf(stdout, "t_cache_image\n"); + } + + return(success); + +} /* parse_flags() */ + + +/*------------------------------------------------------------------------- + * Function: usage + * + * Purpose: Display a brief message describing the purpose and use + * of the program. + * + * Return: void + * + * Programmer: John Mainzer + * 4/28/11 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void +usage(void) +{ + const char * s[] = + { + "\n", + "t_cache_image:\n", + "\n", + "Run the parallel cache image tests. \n" + "\n" + "In general, this program is run via MPI. However, at present, files\n" + "with cache images can only be constructed by serial processes.\n", + "\n", + "To square this circle, one process in the parallel computation \n" + "forks a serial version of the test program to handle this detail.\n", + "The \"setup\" parameter indicates that t_cache_image is being \n", + "invokde for this purpose.\n", + "\n", + "usage: t_cache_image [setup]\n", + "\n", + "where:\n", + "\n", + " setup parameter forces creation of test file\n", + "\n", + "Returns 0 on success, 1 on failure.\n", + "\n", + NULL, + }; + int i = 0; + + while(s[i] != NULL) { + fprintf(stdout, "%s", s[i]); + i++; + } + + return; +} /* usage() */ + + +/*------------------------------------------------------------------------- + * Function: verify_data_sets() + * + * Purpose: If pass is TRUE on entry, verify that the data sets in the + * file exist and contain the expected data. + * + * Note that these data sets were created by + * create_data_sets() above. Thus any changes in that + * function must be reflected in this function, and + * vise-versa. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 7/15/15 + * + * Modifications: + * + * Added min_dset and max_dset parameters and supporting + * code. This allows the caller to specify a range of + * datasets to verify. + * JRM -- 8/20/15 + * + *------------------------------------------------------------------------- + */ + +static void +verify_data_sets(hid_t file_id, int min_dset, int max_dset) +{ + const char * fcn_name = "verify_data_sets()"; + char dset_name[64]; + hbool_t show_progress = FALSE; + hbool_t valid_chunk; + hbool_t verbose = FALSE; + int cp = 0; + int i, j, k, l, m; + int data_chunk[CHUNK_SIZE][CHUNK_SIZE]; + herr_t status; + hid_t filespace_ids[MAX_NUM_DSETS]; + hid_t memspace_id = -1; + hid_t dataset_ids[MAX_NUM_DSETS]; + hsize_t dims[2]; + hsize_t a_size[2]; + hsize_t offset[2]; + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + HDassert(0 <= min_dset); + HDassert(min_dset <= max_dset); + HDassert(max_dset < MAX_NUM_DSETS); + + /* open the datasets */ + + if ( pass ) { + + i = min_dset; + + while ( ( pass ) && ( i <= max_dset ) ) + { + /* open the dataset */ + if ( pass ) { + + sprintf(dset_name, "/dset%03d", i); + dataset_ids[i] = H5Dopen2(file_id, dset_name, H5P_DEFAULT); + + if ( dataset_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dopen2() failed."; + } + } + + /* get the file space ID */ + if ( pass ) { + + filespace_ids[i] = H5Dget_space(dataset_ids[i]); + + if ( filespace_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dget_space() failed."; + } + } + + i++; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* create the mem space to be used to read and write chunks */ + if ( pass ) { + + dims[0] = CHUNK_SIZE; + dims[1] = CHUNK_SIZE; + memspace_id = H5Screate_simple(2, dims, NULL); + + if ( memspace_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Screate_simple() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* select in memory hyperslab */ + if ( pass ) { + + offset[0] = 0; /*offset of hyperslab in memory*/ + offset[1] = 0; + a_size[0] = CHUNK_SIZE; /*size of hyperslab*/ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, offset, NULL, + a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sselect_hyperslab() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + + /* read data from data sets and validate it */ + i = 0; + while ( ( pass ) && ( i < DSET_SIZE ) ) + { + j = 0; + while ( ( pass ) && ( j < DSET_SIZE ) ) + { + m = min_dset; + while ( ( pass ) && ( m <= max_dset ) ) + { + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; /* offset of hyperslab in file */ + offset[1] = (hsize_t)j; + a_size[0] = CHUNK_SIZE; /* size of hyperslab */ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + + /* read the chunk from file */ + if ( pass ) { + + status = H5Dread(dataset_ids[m], H5T_NATIVE_INT, + memspace_id, filespace_ids[m], + H5P_DEFAULT, data_chunk); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + } + + /* validate the slab */ + if ( pass ) { + + valid_chunk = TRUE; + for ( k = 0; k < CHUNK_SIZE; k++ ) + { + for ( l = 0; l < CHUNK_SIZE; l++ ) + { + if ( data_chunk[k][l] + != + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l) ) { + + valid_chunk = FALSE; + + if ( verbose ) { + + HDfprintf(stdout, + "data_chunk[%0d][%0d] = %0d, expect %0d.\n", + k, l, data_chunk[k][l], + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l)); + HDfprintf(stdout, + "m = %d, i = %d, j = %d, k = %d, l = %d\n", + m, i, j, k, l); + } + } + } + } + + if ( ! valid_chunk ) { + + pass = FALSE; + failure_mssg = "slab validation failed."; + + if ( verbose ) { + + fprintf(stdout, + "Chunk (%0d, %0d) in /dset%03d is invalid.\n", + i, j, m); + } + } + } + m++; + } + j += CHUNK_SIZE; + } + i += CHUNK_SIZE; + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* close the file spaces */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Sclose(filespace_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose() failed."; + } + i++; + } + + + /* close the datasets */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Dclose(dataset_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dclose() failed."; + } + i++; + } + + /* close the mem space */ + if ( pass ) { + + if ( H5Sclose(memspace_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose(memspace_id) failed."; + } + } + + return; + +} /* verify_data_sets() */ + + +/****************************************************************************/ +/******************************* Test Functions *****************************/ +/****************************************************************************/ + +/*------------------------------------------------------------------------- + * Function: verify_cache_image_RW() + * + * Purpose: Verify that a HDF5 file containing a cache image is + * opened and read correctly by PHDF5 with the specified + * metadata write strategy. + * + * Basic cycle of operation is as follows: + * + * 1) Open the test file created at the beginning of this + * test. + * + * Verify that the file contains a cache image. + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. + * + * 2) Verify that the file contains the expected data. + * + * 3) Close the file. + * + * 4) Open the file, and verify that it doesn't contain + * a cache image. + * + * 5) Verify that the file contains the expected data. + * + * 6) Close the file. + * + * 7) Delete the file. + * + * Return: void + * + * Programmer: John Mainzer + * 1/25/17 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank) +{ + const char * fcn_name = "verify_cache_imageRW()"; + char filename[512]; + hbool_t show_progress = FALSE; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + int cp = 0; + int i; + + pass = TRUE; + + if ( mpi_rank == 0 ) { + + switch(md_write_strat) { + + case H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY: + TESTING("parallel CI load test -- proc0 md write -- R/W"); + break; + + case H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED: + TESTING("parallel CI load test -- dist md write -- R/W"); + break; + + default: + TESTING("parallel CI load test -- unknown md write -- R/W"); + pass = FALSE; + break; + } + } + + show_progress = ( ( show_progress ) && ( mpi_rank == 0 ) ); + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + if ( h5_fixname(FILENAMES[file_name_id], H5P_DEFAULT, + filename, sizeof(filename)) == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 1) Open the test file created at the beginning of this test. + * + * Verify that the file contains a cache image. + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ MPI_COMM_WORLD, + /* info */ MPI_INFO_NULL, + /* l_facc_type */ FACC_MPIO, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ md_write_strat); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that only process 0 reads the cache image. */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that all other processes receive the cache image block + * from process 0. + */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Verify that the file contains the expected data. */ + if ( pass ) { + + verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded == 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block not loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + + /* 3) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 4) Open the file, and verify that it doesn't contain a cache image. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ MPI_COMM_WORLD, + /* info */ MPI_INFO_NULL, + /* l_facc_type */ FACC_MPIO, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ md_write_strat); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 5) Verify that the file contains the expected data. */ + + if ( pass ) { + + verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(1)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + + /* 6) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 7) Delete the file. */ + + if ( pass ) { + + /* wait for everyone to close the file */ + MPI_Barrier(MPI_COMM_WORLD); + + if ( ( mpi_rank == 0 ) && ( HDremove(filename) < 0 ) ) { + + pass = FALSE; + failure_mssg = "HDremove() failed.\n"; + } + } + + + /* report results */ + if ( mpi_rank == 0 ) { + + if ( pass ) { + + PASSED(); + + } else { + + H5_FAILED(); + + if ( show_progress ) + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", failure_mssg); + } + } + + + return !pass; + +} /* verify_cache_imageRW() */ + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Run parallel tests on the cache image feature. + * + * At present, cache image is disabled in parallel, and + * thus these tests are restructed to verifying that a + * file with a cache image can be opened in the parallel + * case, and verifying that instructions to create a + * cache image are ignored in the parallel case. + * + * WARNING: This test uses fork() and execve(), and + * therefore will not run on Windows. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: John Mainzer + * 1/25/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +int +main(int argc, char **argv) +{ + hbool_t setup = FALSE; + unsigned nerrs = 0; + int i; + int mpi_size; + int mpi_rank; + + if ( ! parse_flags(argc, argv, &setup, FALSE) ) + exit(1); /* exit now if unable to parse flags */ + + if ( setup ) { /* construct test file and exit */ + + H5open(); + HDfprintf(stdout, "Constructing test files: \n"); + HDfflush(stdout); + + i = 0; + while ( FILENAMES[i] != NULL ) { + + HDfprintf(stdout, " writing %s ... ", FILENAMES[i]); + HDfflush(stdout); + construct_test_file(i); + + if ( pass ) { + + printf("done.\n"); + HDfflush(stdout); + + } else { + + printf("failed.\n"); + exit(1); + } + i++; + } + HDfprintf(stdout, "Test file construction complete.\n"); + exit(0); + } + + HDassert(!setup); + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + /* Attempt to turn off atexit post processing so that in case errors + * happen during the test and the process is aborted, it will not get + * hang in the atexit post processing in which it may try to make MPI + * calls. By then, MPI calls may not work. + */ + if (H5dont_atexit() < 0){ + printf("%d:Failed to turn off atexit processing. Continue.\n", + mpi_rank); + }; + + H5open(); + + if ( mpi_rank == 0 ) { + printf("===================================\n"); + printf("Parallel metadata cache image tests\n"); + printf(" mpi_size = %d\n", mpi_size); + printf("===================================\n"); + } + + if ( mpi_size < 2 ) { + + if ( mpi_rank == 0 ) { + + printf(" Need at least 2 processes. Exiting.\n"); + } + goto finish; + } + + if ( mpi_rank == 0 ) { /* create test files */ + + int child_status; + pid_t child_pid; + + child_pid = fork(); + + if ( child_pid == 0 ) { /* this is the child process */ + + /* fun and games to shutup the compiler */ + char param0[32] = "t_cache_image"; + char param1[32] = "setup"; + char * child_argv[] = {param0, param1, NULL}; + + /* we may need to play with the path here */ + if ( execv("t_cache_image", child_argv) == -1 ) { + + HDfprintf(stdout, + "execl() of setup process failed. errno = %d(%s)\n", + errno, strerror(errno)); + exit(1); + } + + } else if ( child_pid != -1 ) { + /* this is the parent process -- wait until child is done */ + if ( -1 == waitpid(child_pid, &child_status, WUNTRACED)) { + + HDfprintf(stdout, "can't wait on setup process.\n"); + + } else if ( ! WIFEXITED(child_status) ) { + + HDfprintf(stdout, "setup process hasn't exitied.\n"); + + } else if ( WEXITSTATUS(child_status) != 0 ) { + + HDfprintf(stdout, "setup process reports failure.\n"); + + } else { + + HDfprintf(stdout, + "testfile construction complete -- proceeding with tests.\n"); + } + } else { /* fork failed */ + + HDfprintf(stdout, "can't create process to construct test file.\n"); + } + } + + /* can't start test until test file exists */ + MPI_Barrier(MPI_COMM_WORLD); + + nerrs += verify_cache_image_RW(0, + H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, mpi_rank); + nerrs += verify_cache_image_RW(1, + H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, mpi_rank); + +finish: + + /* make sure all processes are finished before final report, cleanup + * and exit. + */ + MPI_Barrier(MPI_COMM_WORLD); + + if ( mpi_rank == 0 ) { /* only process 0 reports */ + sleep(10); + printf("===================================\n"); + if ( nerrs > 0 ) { + printf("***metadata cache image tests detected %d failures***\n", + nerrs); + } + else { + printf("metadata cache image tests finished with no failures\n"); + } + printf("===================================\n"); + } + + /* takedown_derived_types(); */ + + /* close HDF5 library */ + H5close(); + + /* MPI_Finalize must be called AFTER H5close which may use MPI calls */ + MPI_Finalize(); + + /* cannot just return (nerrs) because exit code is limited to 1byte */ + return(nerrs > 0); + +} /* main() */ + diff --git a/testpar/t_file.c b/testpar/t_file.c index b2f1d5e..4a923d4 100644 --- a/testpar/t_file.c +++ b/testpar/t_file.c @@ -19,6 +19,30 @@ #include "testphdf5.h" +#include "H5PBprivate.h" +#include "H5Iprivate.h" + +/* + * This file needs to access private information from the H5F package. + */ +#define H5C_FRIEND /*suppress error about including H5Cpkg */ +#include "H5Cpkg.h" +#define H5AC_FRIEND /*suppress error about including H5ACpkg */ +#include "H5ACpkg.h" +#define H5MF_FRIEND /*suppress error about including H5MFpkg */ +#include "H5MFpkg.h" +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" + +#define NUM_DSETS 5 + +int mpi_size, mpi_rank; + +static int create_file(const char *filename, hid_t fcpl, hid_t fapl, int metadata_write_strategy); +static int open_file(const char *filename, hid_t fapl, int metadata_write_strategy, + hsize_t page_size, size_t page_buffer_size); + /* * test file access by communicator besides COMM_WORLD. * Split COMM_WORLD into two, one (even_comm) contains the original @@ -33,7 +57,6 @@ void test_split_comm_access(void) { - int mpi_size, mpi_rank; MPI_Comm comm; MPI_Info info = MPI_INFO_NULL; int is_old, mrc; @@ -95,6 +118,595 @@ test_split_comm_access(void) } void +test_page_buffer_access(void) +{ + hid_t file_id = -1; /* File ID */ + hid_t fcpl, fapl, fapl_self; + hid_t dxpl_id = H5P_DATASET_XFER_DEFAULT; + size_t page_count = 0; + int i, num_elements = 200; + haddr_t raw_addr, meta_addr; + int *data; + H5F_t *f = NULL; + herr_t ret; /* generic return value */ + const char *filename; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + filename = (const char *)GetTestParameters(); + + if (VERBOSE_MED) + printf("Page Buffer Usage in Parallel %s\n", filename); + + fapl = create_faccess_plist(MPI_COMM_WORLD, MPI_INFO_NULL, facc_type); + VRFY((fapl >= 0), "create_faccess_plist succeeded"); + fcpl = H5Pcreate(H5P_FILE_CREATE); + VRFY((fcpl >= 0), ""); + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0); + VRFY((ret == 0), ""); + ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*100); + VRFY((ret == 0), ""); + ret = H5Pset_page_buffer_size(fapl, sizeof(int)*100000, 0, 0); + VRFY((ret == 0), ""); + + /* This should fail because collective metadata writes are not supported with page buffering */ + H5E_BEGIN_TRY { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VRFY((file_id < 0), "H5Fcreate failed"); + + /* disable collective metadata writes for page buffering to work */ + ret = H5Pset_coll_metadata_write(fapl, FALSE); + VRFY((ret >= 0), ""); + + ret = create_file(filename, fcpl, fapl, H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + VRFY((ret == 0), ""); + ret = open_file(filename, fapl, H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, sizeof(int)*100, sizeof(int)*100000); + VRFY((ret == 0), ""); + + ret = create_file(filename, fcpl, fapl, H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); + VRFY((ret == 0), ""); + ret = open_file(filename, fapl, H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, sizeof(int)*100, sizeof(int)*100000); + VRFY((ret == 0), ""); + + ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*100); + VRFY((ret == 0), ""); + + data = (int *) HDmalloc(sizeof(int)*(size_t)num_elements); + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + /* MSC - why this stopped working ? */ +#if 0 + if(MAINPROCESS) { + hid_t fapl_self; + + fapl_self = create_faccess_plist(MPI_COMM_SELF, MPI_INFO_NULL, facc_type); + + ret = H5Pset_page_buffer_size(fapl_self, sizeof(int)*1000, 0, 0); + VRFY((ret == 0), ""); + /* collective metadata writes do not work with page buffering */ + ret = H5Pset_coll_metadata_write(fapl_self, FALSE); + VRFY((ret >= 0), ""); + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_self); + VRFY((file_id >= 0), ""); + + /* Get a pointer to the internal file object */ + f = (H5F_t *)H5I_object(file_id); + + VRFY((f->shared->page_buf != NULL), "Page Buffer created with 1 process"); + + /* allocate space for 200 raw elements */ + raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((raw_addr != HADDR_UNDEF), ""); + + /* allocate space for 200 metadata elements */ + meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((meta_addr != HADDR_UNDEF), ""); + + page_count = 0; + + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data); + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the first 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + page_count += 2; + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the second 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i+50; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update 100 - 200 */ + for(i=0 ; i<100 ; i++) + data[i] = i+100; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + ret = H5PB_flush(f, dxpl_id, FALSE); + VRFY((ret == 0), ""); + + /* read elements 0 - 200 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*200, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*200, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + + /* read elements 0 - 50 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + + /* close the file */ + ret = H5Fclose(file_id); + VRFY((ret >= 0), "H5Fclose succeeded"); + ret = H5Pclose(fapl_self); + VRFY((ret>=0), "H5Pclose succeeded"); + } +#endif + + MPI_Barrier(MPI_COMM_WORLD); + + if(mpi_size > 1) { + ret = H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 0, 0); + VRFY((ret == 0), ""); + /* collective metadata writes do not work with page buffering */ + ret = H5Pset_coll_metadata_write(fapl, FALSE); + VRFY((ret >= 0), ""); + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + VRFY((file_id >= 0), ""); + + /* Get a pointer to the internal file object */ + f = (H5F_t *)H5I_object(file_id); + + VRFY((f->shared->page_buf != NULL), "Page Buffer created with 1 process"); + + /* allocate space for 200 raw elements */ + raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((raw_addr != HADDR_UNDEF), ""); + /* allocate space for 200 metadata elements */ + meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((meta_addr != HADDR_UNDEF), ""); + + page_count = 0; + + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the first 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the second 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i+50; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update 100 - 200 */ + for(i=0 ; i<100 ; i++) + data[i] = i+100; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + /* read elements 0 - 200 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*200, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*200, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + + /* read elements 0 - 50 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + page_count += 1; + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + + MPI_Barrier(MPI_COMM_WORLD); + /* reset the first 50 elements to -1*/ + for(i=0 ; i<50 ; i++) + data[i] = -1; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* read elements 0 - 50 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == -1), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == -1), "Read different values than written"); + + /* close the file */ + ret = H5Fclose(file_id); + VRFY((ret >= 0), "H5Fclose succeeded"); + } + + ret = H5Pclose(fapl); + VRFY((ret>=0), "H5Pclose succeeded"); + ret = H5Pclose(fcpl); + VRFY((ret>=0), "H5Pclose succeeded"); + + HDfree(data); + data = NULL; + MPI_Barrier(MPI_COMM_WORLD); +} + +static int +create_file(const char *filename, hid_t fcpl, hid_t fapl, int metadata_write_strategy) +{ + hid_t file_id, dset_id, grp_id; + hid_t sid, mem_dataspace; + hsize_t start[RANK]; + hsize_t count[RANK]; + hsize_t stride[RANK]; + hsize_t block[RANK]; + DATATYPE *data_array = NULL; + hsize_t dims[RANK], i; + hsize_t num_elements; + int k; + char dset_name[10]; + H5F_t *f = NULL; + H5C_t *cache_ptr = NULL; + H5AC_cache_config_t config; + herr_t ret; + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + VRFY((file_id >= 0), ""); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + f = (H5F_t *)H5I_object(file_id); + VRFY((f != NULL), ""); + + cache_ptr = f->shared->cache; + VRFY((cache_ptr->magic == H5C__H5C_T_MAGIC), ""); + + cache_ptr->ignore_tags = TRUE; + H5C_stats__reset(cache_ptr); + config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + + ret = H5AC_get_cache_auto_resize_config(cache_ptr, &config); + VRFY((ret == 0), ""); + + config.metadata_write_strategy = metadata_write_strategy; + + ret = H5AC_set_cache_auto_resize_config(cache_ptr, &config); + VRFY((ret == 0), ""); + + grp_id = H5Gcreate2(file_id, "GROUP", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((grp_id >= 0), ""); + + dims[0] = ROW_FACTOR*mpi_size; + dims[1] = COL_FACTOR*mpi_size; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + + /* Each process takes a slabs of rows. */ + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + num_elements = block[0] * block[1]; + /* allocate memory for data buffer */ + data_array = (DATATYPE *)HDmalloc(num_elements*sizeof(DATATYPE)); + VRFY((data_array != NULL), "data_array HDmalloc succeeded"); + /* put some trivial data in the data_array */ + for(i=0 ; i<num_elements; i++) + data_array[i] = mpi_rank + 1; + + ret = H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (1, &num_elements, NULL); + VRFY((mem_dataspace >= 0), ""); + + for(k=0 ; k<NUM_DSETS; k++) { + sprintf(dset_name, "D1dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + sprintf(dset_name, "D2dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + sprintf(dset_name, "D3dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + sprintf(dset_name, "dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + + ret = H5Dwrite(dset_id, H5T_NATIVE_INT, mem_dataspace, sid, H5P_DEFAULT, data_array); + VRFY((ret == 0), ""); + + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + HDmemset(data_array, 0, num_elements*sizeof(DATATYPE)); + dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, mem_dataspace, sid, H5P_DEFAULT, data_array); + VRFY((ret == 0), ""); + + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + for (i=0; i < num_elements; i++) + VRFY((data_array[i] == mpi_rank+1), "Dataset Verify failed"); + + sprintf(dset_name, "D1dset%d", k); + ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); + VRFY((ret == 0), ""); + sprintf(dset_name, "D2dset%d", k); + ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); + VRFY((ret == 0), ""); + sprintf(dset_name, "D3dset%d", k); + ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); + VRFY((ret == 0), ""); + } + + ret = H5Gclose(grp_id); + VRFY((ret == 0), ""); + ret = H5Fclose(file_id); + VRFY((ret == 0), ""); + ret = H5Sclose(sid); + VRFY((ret == 0), ""); + ret = H5Sclose(mem_dataspace); + VRFY((ret == 0), ""); + + MPI_Barrier(MPI_COMM_WORLD); + HDfree(data_array); + return 0; +} /* create_file */ + +static int +open_file(const char *filename, hid_t fapl, int metadata_write_strategy, + hsize_t page_size, size_t page_buffer_size) +{ + hid_t file_id, dset_id, grp_id, grp_id2; + hid_t sid, mem_dataspace; + DATATYPE *data_array = NULL; + hsize_t dims[RANK]; + hsize_t start[RANK]; + hsize_t count[RANK]; + hsize_t stride[RANK]; + hsize_t block[RANK]; + int i, k, ndims; + hsize_t num_elements; + char dset_name[10]; + H5F_t *f = NULL; + H5C_t *cache_ptr = NULL; + H5AC_cache_config_t config; + herr_t ret; + + config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + ret = H5Pget_mdc_config(fapl, &config); + VRFY((ret == 0), ""); + + config.metadata_write_strategy = metadata_write_strategy; + + ret = H5Pget_mdc_config(fapl, &config); + VRFY((ret == 0), ""); + + file_id = H5Fopen(filename, H5F_ACC_RDWR, fapl); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((file_id >= 0), ""); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + f = (H5F_t *)H5I_object(file_id); + VRFY((f != NULL), ""); + + cache_ptr = f->shared->cache; + VRFY((cache_ptr->magic == H5C__H5C_T_MAGIC), ""); + + MPI_Barrier(MPI_COMM_WORLD); + + VRFY((f->shared->page_buf != NULL), ""); + VRFY((f->shared->page_buf->page_size == page_size), ""); + VRFY((f->shared->page_buf->max_size == page_buffer_size), ""); + + grp_id = H5Gopen2(file_id, "GROUP", H5P_DEFAULT); + VRFY((grp_id >= 0), ""); + + dims[0] = ROW_FACTOR*mpi_size; + dims[1] = COL_FACTOR*mpi_size; + + /* Each process takes a slabs of rows. */ + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + num_elements = block[0] * block[1]; + /* allocate memory for data buffer */ + data_array = (DATATYPE *)HDmalloc(num_elements*sizeof(DATATYPE)); + VRFY((data_array != NULL), "data_array HDmalloc succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (1, &num_elements, NULL); + VRFY((mem_dataspace >= 0), ""); + + for(k=0 ; k<NUM_DSETS; k++) { + sprintf(dset_name, "dset%d", k); + dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + + sid = H5Dget_space(dset_id); + VRFY((dset_id >= 0), "H5Dget_space succeeded"); + + ndims = H5Sget_simple_extent_dims(sid, dims, NULL); + VRFY((ndims == 2), "H5Sget_simple_extent_dims succeeded"); + VRFY(dims[0] == ROW_FACTOR*mpi_size, "Wrong dataset dimensions"); + VRFY(dims[1] == COL_FACTOR*mpi_size, "Wrong dataset dimensions"); + + ret = H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, mem_dataspace, sid, H5P_DEFAULT, data_array); + VRFY((ret >= 0), ""); + + ret = H5Dclose(dset_id); + VRFY((ret >= 0), ""); + ret = H5Sclose(sid); + VRFY((ret == 0), ""); + + for (i=0; i < num_elements; i++) + VRFY((data_array[i] == mpi_rank+1), "Dataset Verify failed"); + } + + grp_id2 = H5Gcreate2(file_id, "GROUP/GROUP2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((grp_id2 >= 0), ""); + ret = H5Gclose(grp_id2); + VRFY((ret == 0), ""); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + MPI_Barrier(MPI_COMM_WORLD); + /* flush invalidate each ring, starting from the outermost ring and + * working inward. + */ + for ( i = 0; i < H5C__HASH_TABLE_LEN; i++ ) { + H5C_cache_entry_t * entry_ptr = NULL; + + entry_ptr = cache_ptr->index[i]; + + while ( entry_ptr != NULL ) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(entry_ptr->is_dirty == FALSE); + + if(!entry_ptr->is_pinned && !entry_ptr->is_protected) { + ret = H5AC_expunge_entry(f, H5AC_ind_read_dxpl_id, entry_ptr->type, entry_ptr->addr, 0); + VRFY((ret == 0), ""); + } + + entry_ptr = entry_ptr->ht_next; + } + } + MPI_Barrier(MPI_COMM_WORLD); + + grp_id2 = H5Gopen2(file_id, "GROUP/GROUP2", H5P_DEFAULT); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((grp_id2 >= 0), ""); + ret = H5Gclose(grp_id2); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((ret == 0), ""); + + ret = H5Gclose(grp_id); + VRFY((ret == 0), ""); + ret = H5Fclose(file_id); + VRFY((ret == 0), ""); + ret = H5Sclose(mem_dataspace); + VRFY((ret == 0), ""); + HDfree(data_array); + + return nerrors; +} + +void test_file_properties(void) { hid_t fid; /* HDF5 file ID */ @@ -103,7 +715,6 @@ test_file_properties(void) const char *filename; MPI_Comm comm = MPI_COMM_WORLD; MPI_Info info = MPI_INFO_NULL; - int mpi_size, mpi_rank; herr_t ret; /* Generic return value */ filename = (const char *)GetTestParameters(); diff --git a/testpar/t_mdset.c b/testpar/t_mdset.c index 7077081..39cfbbc 100644 --- a/testpar/t_mdset.c +++ b/testpar/t_mdset.c @@ -510,7 +510,7 @@ void big_dataset(void) /* Check that file of the correct size was created */ file_size = h5_get_file_size(filename, fapl); - VRFY((file_size == 2147485792ULL), "File is correct size(~2GB)"); + VRFY((file_size == 2147485696ULL), "File is correct size(~2GB)"); /* * Create >4GB HDF5 file @@ -539,7 +539,7 @@ void big_dataset(void) /* Check that file of the correct size was created */ file_size = h5_get_file_size(filename, fapl); - VRFY((file_size == 4294969440ULL), "File is correct size(~4GB)"); + VRFY((file_size == 4294969344ULL), "File is correct size(~4GB)"); /* * Create >8GB HDF5 file @@ -568,7 +568,7 @@ void big_dataset(void) /* Check that file of the correct size was created */ file_size = h5_get_file_size(filename, fapl); - VRFY((file_size == 8589936736ULL), "File is correct size(~8GB)"); + VRFY((file_size == 8589936640ULL), "File is correct size(~8GB)"); /* Close fapl */ ret = H5Pclose(fapl); diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index c54cb5e..a58452e 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -351,6 +351,11 @@ int main(int argc, char **argv) AddTest("split", test_split_comm_access, NULL, "dataset using split communicators", PARATESTFILE); +#ifdef PB_OUT /* temporary: disable page buffering when parallel */ + AddTest("page_buffer", test_page_buffer_access, NULL, + "page buffer usage in parallel", PARATESTFILE); +#endif + AddTest("props", test_file_properties, NULL, "Coll Metadata file property settings", PARATESTFILE); diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index 9838673..0cf5857 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -246,6 +246,7 @@ void collective_group_write(void); void independent_group_read(void); void test_fapl_mpio_dup(void); void test_split_comm_access(void); +void test_page_buffer_access(void); void dataset_atomicity(void); void dataset_writeInd(void); void dataset_writeAll(void); diff --git a/tools/src/h5dump/h5dump_ddl.c b/tools/src/h5dump/h5dump_ddl.c index 182d570..c78f325 100644 --- a/tools/src/h5dump/h5dump_ddl.c +++ b/tools/src/h5dump/h5dump_ddl.c @@ -1186,9 +1186,11 @@ dump_fcpl(hid_t fid) hsize_t userblock; /* userblock size retrieved from FCPL */ size_t off_size; /* size of offsets in the file */ size_t len_size; /* size of lengths in the file */ - H5F_file_space_type_t fs_strategy; /* file space strategy */ - hsize_t fs_threshold; /* free-space section threshold */ - H5F_info2_t finfo; /* file information */ + H5F_fspace_strategy_t fs_strategy; /* file space strategy */ + hbool_t fs_persist; /* Persisting free-space or not */ + hsize_t fs_threshold; /* free-space section threshold */ + hsize_t fsp_size; /* file space page size */ + H5F_info2_t finfo; /* file information */ #ifdef SHOW_FILE_DRIVER hid_t fapl; /* file access property list ID */ hid_t fdriver; /* file driver */ @@ -1204,7 +1206,8 @@ dump_fcpl(hid_t fid) H5Pget_sizes(fcpl,&off_size,&len_size); H5Pget_sym_k(fcpl,&sym_ik,&sym_lk); H5Pget_istore_k(fcpl,&istore_ik); - H5Pget_file_space(fcpl, &fs_strategy, &fs_threshold); + H5Pget_file_space_strategy(fcpl, &fs_strategy, &fs_persist, &fs_threshold); + H5Pget_file_space_page_size(fcpl, &fsp_size); H5Pclose(fcpl); #ifdef SHOW_FILE_DRIVER fapl=h5_fileaccess(); @@ -1265,18 +1268,22 @@ dump_fcpl(hid_t fid) PRINTSTREAM(rawoutstream, "%s %u\n","ISTORE_K", istore_ik); indentation(dump_indent + COL); - if(fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_ALL_PERSIST"); - } else if(fs_strategy == H5F_FILE_SPACE_ALL) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_ALL"); - } else if(fs_strategy == H5F_FILE_SPACE_AGGR_VFD) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_AGGR_VFD"); - } else if(fs_strategy == H5F_FILE_SPACE_VFD) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_VFD"); + if(fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_FSM_AGGR"); + } else if(fs_strategy == H5F_FSPACE_STRATEGY_PAGE) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_PAGE"); + } else if(fs_strategy == H5F_FSPACE_STRATEGY_AGGR) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_AGGR"); + } else if(fs_strategy == H5F_FSPACE_STRATEGY_NONE) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_NONE"); } else PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "Unknown strategy"); indentation(dump_indent + COL); - PRINTSTREAM(rawoutstream, "%s %Hu\n","FREE_SPACE_THRESHOLD", fs_threshold); + PRINTSTREAM(rawoutstream, "%s %s\n","FREE_SPACE_PERSIST", fs_persist ? "TRUE" : "FALSE"); + indentation(dump_indent + COL); + PRINTSTREAM(rawoutstream, "%s %Hu\n","FREE_SPACE_SECTION_THRESHOLD", fs_threshold); + indentation(dump_indent + COL); + PRINTSTREAM(rawoutstream, "%s %Hu\n","FILE_SPACE_PAGE_SIZE", fsp_size); /*------------------------------------------------------------------------- * USER_BLOCK diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index c79db1c..c09dee6 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -78,8 +78,7 @@ int h5repack(const char* infile, const char* outfile, pack_opt_t *options) { *------------------------------------------------------------------------- */ int -h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, - H5F_file_space_type_t strategy, hsize_t threshold) +h5repack_init(pack_opt_t *options, int verbose, hbool_t latest) { int k, n; @@ -96,9 +95,6 @@ h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, options->filter_g[n].cd_values[k] = 0; } - options->fs_strategy = strategy; - options->fs_threshold = threshold; - return (options_table_init(&(options->op_tbl))); } diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h index d2ab923..5ad0b80 100644 --- a/tools/src/h5repack/h5repack.h +++ b/tools/src/h5repack/h5repack.h @@ -30,6 +30,13 @@ #define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */ #define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */ +/* File space default information */ +#define FS_PAGESIZE_DEF 4096 +#define FS_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR +#define FS_PERSIST_DEF FALSE +#define FS_THRESHOLD_DEF 1 + + /*------------------------------------------------------------------------- * data structures for command line options *------------------------------------------------------------------------- @@ -115,8 +122,10 @@ typedef struct { hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ hsize_t threshold; /* alignment threshold for H5Pset_alignment */ hsize_t alignment; /* alignment for H5Pset_alignment */ - H5F_file_space_type_t fs_strategy; /* File space handling strategy */ - hsize_t fs_threshold; /* Free space section threshold */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ + int fs_persist; /* Free space section threshold */ + long fs_threshold; /* Free space section threshold */ + long long fs_pagesize; /* File space page size */ } pack_opt_t; @@ -138,8 +147,7 @@ extern "C" { int h5repack(const char* infile, const char* outfile, pack_opt_t *options); int h5repack_addfilter(const char* str, pack_opt_t *options); int h5repack_addlayout(const char* str, pack_opt_t *options); -int h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, - H5F_file_space_type_t strategy, hsize_t threshold); +int h5repack_init(pack_opt_t *options, int verbose, hbool_t latest); int h5repack_end(pack_opt_t *options); int h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options); int h5repack_cmp_pl(const char *fname1, const char *fname2); diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 1519053..45d52bf 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -96,6 +96,14 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) hsize_t ub_size = 0; /* size of user block */ hid_t fcpl = H5P_DEFAULT; /* file creation property list ID */ hid_t fapl = H5P_DEFAULT; /* file access property list ID */ + H5F_fspace_strategy_t set_strategy; /* Strategy to be set in outupt file */ + hbool_t set_persist; /* Persist free-space status to be set in output file */ + hsize_t set_threshold; /* Free-space section threshold to be set in output file */ + hsize_t set_pagesize; /* File space page size to be set in output file */ + H5F_fspace_strategy_t in_strategy; /* Strategy from input file */ + hbool_t in_persist; /* Persist free-space status from input file */ + hsize_t in_threshold; /* Free-space section threshold from input file */ + hsize_t in_pagesize; /* File space page size from input file */ /*------------------------------------------------------------------------- * open input file @@ -106,7 +114,7 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } - /* get user block size and file space strategy/threshold */ + /* get user block size and file space strategy/persist/threshold */ { hid_t fcpl_in; /* file creation property list ID for input file */ @@ -120,19 +128,17 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } - if (!options->fs_strategy) { - if (H5Pget_file_space(fcpl_in, &options->fs_strategy, NULL) < 0) { - error_msg("failed to retrieve file space strategy\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); - } + /* If the -S option is not set, get "strategy" from the input file */ + if(H5Pget_file_space_strategy(fcpl_in, &in_strategy, &in_persist, &in_threshold) < 0) { + error_msg("failed to retrieve file space strategy\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } - if (!options->fs_threshold) { - if (H5Pget_file_space(fcpl_in, NULL, &options->fs_threshold) < 0) { - error_msg("failed to retrieve file space threshold\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); - } - } + /* If the -G option is not set, get "pagesize" from the input file */ + if(H5Pget_file_space_page_size(fcpl_in, &in_pagesize) < 0) { + error_msg("failed to retrieve file space threshold\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } if (H5Pclose(fcpl_in) < 0) { error_msg("failed to close property list\n"); @@ -291,9 +297,9 @@ print_user_block(fnamein, fidin); } /*------------------------------------------------------------------------- - * set free-space strategy options - *------------------------------------------------------------------------- - */ + * Set file space information + *------------------------------------------------------------------------- + */ /* either use the FCPL already created or create a new one */ if (fcpl == H5P_DEFAULT) { @@ -304,12 +310,44 @@ print_user_block(fnamein, fidin); } } - /* set file space strategy and free space threshold */ - if (H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) { - error_msg("failed to set file space strategy & threshold \n"); + /* Set file space info to those from input file */ + set_strategy = in_strategy; + set_persist = in_persist; + set_threshold = in_threshold; + set_pagesize = in_pagesize; + + if(options->fs_strategy == (H5F_fspace_strategy_t)-1) /* A default strategy is specified by user */ + set_strategy = FS_STRATEGY_DEF; + else if(options->fs_strategy != (H5F_fspace_strategy_t)0) /* Set strategy as specified by user */ + set_strategy = options->fs_strategy; + + if(options->fs_persist == -1) /* A default "persist" is specified by user */ + set_persist = FS_PERSIST_DEF; + else if(options->fs_persist != 0) /* Set "persist" as specified by user */ + set_persist = (hbool_t)options->fs_persist; + + if(options->fs_threshold == -1) /* A "0" threshold is specified by user */ + set_threshold = (hsize_t)0; + else if(options->fs_threshold != 0) /* Set threshold as specified by user */ + set_threshold = (hsize_t)options->fs_threshold; + + /* Set file space information as specified */ + if(H5Pset_file_space_strategy(fcpl, set_strategy, set_persist, set_threshold) < 0) { + error_msg("failed to set file space strategy\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } + if(options->fs_pagesize == -1) /* A "0" file space page size is specified by user */ + set_pagesize = (hsize_t)0; + else if(options->fs_pagesize != 0) /* Set file space page size as specified by user */ + set_pagesize = (hsize_t)options->fs_pagesize; + + if(set_pagesize != FS_PAGESIZE_DEF) /* Set non-default file space page size as specified */ + if(H5Pset_file_space_page_size(fcpl, set_pagesize) < 0) { + error_msg("failed to set file space page size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + /*------------------------------------------------------------------------- * create the output file *------------------------------------------------------------------------- diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index 1807056..07c2b7d 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -33,7 +33,7 @@ const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:S:T:E"; +static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:S:P:T:G:E"; static struct long_options l_opts[] = { { "help", no_arg, 'h' }, { "version", no_arg, 'V' }, @@ -55,7 +55,9 @@ static struct long_options l_opts[] = { { "infile", require_arg, 'i' }, /* -i for backward compability */ { "outfile", require_arg, 'o' }, /* -o for backward compability */ { "fs_strategy", require_arg, 'S' }, + { "fs_persist", require_arg, 'P' }, { "fs_threshold", require_arg, 'T' }, + { "fs_pagesize", require_arg, 'G' }, { "enable-error-stack", no_arg, 'E' }, { NULL, 0, '\0' } }; @@ -92,8 +94,10 @@ static void usage(const char *prog) { PRINTVALSTREAM(rawoutstream, " -a A, --alignment=A Alignment value for H5Pset_alignment\n"); PRINTVALSTREAM(rawoutstream, " -f FILT, --filter=FILT Filter type\n"); PRINTVALSTREAM(rawoutstream, " -l LAYT, --layout=LAYT Layout type\n"); - PRINTVALSTREAM(rawoutstream, " -S FS_STRGY, --fs_strategy=FS_STRGY File space management strategy\n"); - PRINTVALSTREAM(rawoutstream, " -T FS_THRD, --fs_threshold=FS_THRD Free-space section threshold\n"); + PRINTVALSTREAM(rawoutstream, " -S FS_STRATEGY, --fs_strategy=FS_STRATEGY File space management strategy for H5Pset_file_space_strategy\n"); + PRINTVALSTREAM(rawoutstream, " -P FS_PERSIST, --fs_persist=FS_PERSIST Persisting or not persisting free-space for H5Pset_file_space_strategy\n"); + PRINTVALSTREAM(rawoutstream, " -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy\n"); + PRINTVALSTREAM(rawoutstream, " -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size\n"); PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " M - is an integer greater than 1, size of dataset in bytes (default is 0) \n"); PRINTVALSTREAM(rawoutstream, " E - is a filename.\n"); @@ -109,18 +113,27 @@ static void usage(const char *prog) { PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they\n"); PRINTVALSTREAM(rawoutstream, " occur.\n"); PRINTVALSTREAM(rawoutstream, "\n"); - PRINTVALSTREAM(rawoutstream, " FS_STRGY is the file space management strategy to use for the output file.\n"); - PRINTVALSTREAM(rawoutstream, " It is a string as listed below:\n"); - PRINTVALSTREAM(rawoutstream, " ALL_PERSIST - Use persistent free-space managers, aggregators and virtual file driver\n"); - PRINTVALSTREAM(rawoutstream, " for file space allocation\n"); - PRINTVALSTREAM(rawoutstream, " ALL - Use non-persistent free-space managers, aggregators and virtual file driver\n"); - PRINTVALSTREAM(rawoutstream, " for file space allocation\n"); - PRINTVALSTREAM(rawoutstream, " AGGR_VFD - Use aggregators and virtual file driver for file space allocation\n"); - PRINTVALSTREAM(rawoutstream, " VFD - Use virtual file driver for file space allocation\n"); + PRINTVALSTREAM(rawoutstream, " FS_STRATEGY is a string indicating the file space strategy used:\n"); + PRINTVALSTREAM(rawoutstream, " FSM_AGGR:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are free-space managers, aggregators and virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " PAGE:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are free-space managers with embedded paged aggregation and virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " AGGR:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are aggregators and virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " NONE:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " The default strategy when not set is FSM_AGGR without persisting free-space.\n"); PRINTVALSTREAM(rawoutstream, "\n"); - PRINTVALSTREAM(rawoutstream, " FS_THRD is the free-space section threshold to use for the output file.\n"); - PRINTVALSTREAM(rawoutstream, " It is the minimum size (in bytes) of free-space sections to be tracked\n"); - PRINTVALSTREAM(rawoutstream, " by the the library's free-space managers.\n"); + PRINTVALSTREAM(rawoutstream, " FS_PERSIST is 1 to persisting free-space or 0 to not persisting free-space.\n"); + PRINTVALSTREAM(rawoutstream, " The default when not set is not persisting free-space.\n"); + PRINTVALSTREAM(rawoutstream, " The value is ignored for AGGR and NONE strategies.\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " FS_THRESHOLD is the minimum size (in bytes) of free-space sections to be tracked by the library.\n"); + PRINTVALSTREAM(rawoutstream, " The default when not set is 1.\n"); + PRINTVALSTREAM(rawoutstream, " The value is ignored for AGGR and NONE strategies.\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " FS_PAGESIZE is the size (in bytes) >=512 that is used by the library when the file space strategy PAGE is used.\n"); + PRINTVALSTREAM(rawoutstream, " The default when not set is 4096.\n"); PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " FILT - is a string with the format:\n"); PRINTVALSTREAM(rawoutstream, "\n"); @@ -521,25 +534,45 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options) char strategy[MAX_NC_NAME]; HDstrcpy(strategy, opt_arg); - if(!HDstrcmp(strategy, "ALL_PERSIST")) - options->fs_strategy = H5F_FILE_SPACE_ALL_PERSIST; - else if(!HDstrcmp(strategy, "ALL")) - options->fs_strategy = H5F_FILE_SPACE_ALL; - else if(!HDstrcmp(strategy, "AGGR_VFD")) - options->fs_strategy = H5F_FILE_SPACE_AGGR_VFD; - else if(!HDstrcmp(strategy, "VFD")) - options->fs_strategy = H5F_FILE_SPACE_VFD; + if(!HDstrcmp(strategy, "FSM_AGGR")) + options->fs_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + else if(!HDstrcmp(strategy, "PAGE")) + options->fs_strategy = H5F_FSPACE_STRATEGY_PAGE; + else if(!HDstrcmp(strategy, "AGGR")) + options->fs_strategy = H5F_FSPACE_STRATEGY_AGGR; + else if(!HDstrcmp(strategy, "NONE")) + options->fs_strategy = H5F_FSPACE_STRATEGY_NONE; else { error_msg("invalid file space management strategy\n", opt_arg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; } + if(options->fs_strategy == (H5F_fspace_strategy_t)0) + /* To distinguish the "specified" zero value */ + options->fs_strategy = (H5F_fspace_strategy_t)-1; } break; + case 'P': + options->fs_persist = HDatoi( opt_arg ); + if(options->fs_persist == 0) + /* To distinguish the "specified" zero value */ + options->fs_persist = -1; + break; + case 'T': - options->fs_threshold = (hsize_t) HDatol( opt_arg ); + options->fs_threshold = HDatol( opt_arg ); + if(options->fs_threshold == 0) + /* To distinguish the "specified" zero value */ + options->fs_threshold = -1; + break; + + case 'G': + options->fs_pagesize = HDstrtoll(opt_arg, NULL, 0); + if(options->fs_pagesize == 0) + /* To distinguish the "specified" zero value */ + options->fs_pagesize = -1; break; case 'E': @@ -611,7 +644,7 @@ int main(int argc, const char **argv) } /* initialize options */ - h5repack_init(&options, 0, FALSE, H5F_FILE_SPACE_DEFAULT, (hsize_t) 0); + h5repack_init(&options, 0, FALSE); if (parse_command_line(argc, argv, &options) < 0) goto done; diff --git a/tools/src/h5repack/h5repack_verify.c b/tools/src/h5repack/h5repack_verify.c index 58dea03..1022d7a 100644 --- a/tools/src/h5repack/h5repack_verify.c +++ b/tools/src/h5repack/h5repack_verify.c @@ -58,8 +58,10 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options int ok = 1; hid_t fcpl_in = -1; /* file creation property for input file */ hid_t fcpl_out = -1; /* file creation property for output file */ - H5F_file_space_type_t in_strat, out_strat; /* file space handling strategy for in/output file */ - hsize_t in_thresh, out_thresh; /* free space section threshold for in/output file */ + H5F_fspace_strategy_t in_strategy, out_strategy; /* file space handling strategy for in/output file */ + hbool_t in_persist, out_persist; /* free-space persist status for in/output file */ + hsize_t in_threshold, out_threshold; /* free-space section threshold for in/output file */ + hsize_t in_pagesize, out_pagesize; /* file space page size for input/output file */ /* open the output file */ if((fidout = H5Fopen(out_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) @@ -196,8 +198,7 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options } /*------------------------------------------------------------------------- - * Verify that file space strategy and free space threshold - * are set as expected + * Verify that file space info are set as expected *------------------------------------------------------------------------- */ @@ -211,12 +212,18 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); } - /* Get file space management info for input file */ - if(H5Pget_file_space(fcpl_in, &in_strat, &in_thresh) < 0) { + /* Get file space info for input file */ + if(H5Pget_file_space_strategy(fcpl_in, &in_strategy, &in_persist, &in_threshold) < 0) { error_msg("failed to retrieve file space strategy & threshold\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_file_space failed"); } + /* Get file space page size for input file */ + if(H5Pget_file_space_page_size(fcpl_in, &in_pagesize) < 0) { + error_msg("failed to retrieve file space page size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); + } + /* Output file is already opened */ /* Get file creation property list for output file */ if((fcpl_out = H5Fget_create_plist(fidout)) < 0) { @@ -224,45 +231,84 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); } - /* Get file space management info for output file */ - if(H5Pget_file_space(fcpl_out, &out_strat, &out_thresh) < 0) { + /* Get file space info for output file */ + if(H5Pget_file_space_strategy(fcpl_out, &out_strategy, &out_persist, &out_threshold) < 0) { error_msg("failed to retrieve file space strategy & threshold\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_file_space failed"); } + /* Get file space page size for output file */ + if(H5Pget_file_space_page_size(fcpl_out, &out_pagesize) < 0) { + error_msg("failed to retrieve file space page size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); + } + /* - * If the strategy option is not set, - * file space handling strategy should be the same for both - * input & output files. - * If the strategy option is set, - * the output file's file space handling strategy should be the same - * as what is set via the strategy option + * If -S option is set, the file space handling strategy should be set as specified. + * If -S option is not set, the file space handling strategy should be + * the same as the input file's strategy. */ - if(!options->fs_strategy && out_strat != in_strat) { - error_msg("file space strategy not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); - + if(options->fs_strategy) { + if(out_strategy != (options->fs_strategy == (-1) ? 0 : options->fs_strategy)) { + error_msg("file space strategy not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { + if(out_strategy != in_strategy) { + error_msg("file space strategy not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } - else if(options->fs_strategy && out_strat!= options->fs_strategy) { - error_msg("file space strategy not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + + /* + * If the -P option is set, the free-space persist status should be set as specified. + * If the -P option is not set, the free-space persist status should be + * the same as the input file's free-space persist status + */ + if(options->fs_persist) { + if(out_persist != (hbool_t)(options->fs_persist == (-1) ? FALSE : options->fs_persist)) { + error_msg("free-space persist status is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { + if(out_persist != in_persist) { + error_msg("free-space persist status is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } /* - * If the threshold option is not set, - * the free space section threshold should be the same for both - * input & output files. - * If the threshold option is set, - * the output file's free space section threshold should be the same - * as what is set via the threshold option. + * If the -T option is set, the threshold size should be set as specified. + * If the -T option is not set, the threshold should be the same as the + * input file's threshold size. */ - if(!options->fs_threshold && out_thresh != in_thresh) { - error_msg("free space threshold not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "free space threshold not set as unexpected"); + if(options->fs_threshold) { + if(out_threshold != (hsize_t)(options->fs_threshold == (-1) ? 0 : options->fs_threshold)) { + error_msg("threshold is not set as unexpectec\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { + if(out_threshold != in_threshold) { + error_msg("threshold is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } - else if(options->fs_threshold && out_thresh != options->fs_threshold) { - error_msg("free space threshold not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "free space threshold not set as unexpected"); + + /* + * If the -G option is set, the file space page size should be set as specified. + * If the -G option is not set, the file space page size should be + * the same as the input file's file space page size. + */ + if(options->fs_pagesize) { + if(out_pagesize != (hsize_t)(options->fs_pagesize == (-1) ? 0 : options->fs_pagesize)) { + error_msg("file space page size is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { /* "-G" is not set */ + if(out_pagesize != in_pagesize) { + error_msg("file space page size is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } /* Closing */ diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index a331014..f99bab0 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -40,11 +40,11 @@ /* File space management strategies: see H5Fpublic.h for declarations */ const char *FS_STRATEGY_NAME[] = { + "H5F_FSPACE_STRATEGY_FSM_AGGR", + "H5F_FSPACE_STRATEGY_PAGE", + "H5F_FSPACE_STRATEGY_AGGR", + "H5F_FSPACE_STRATEGY_NONE", "unknown", - "H5F_FILE_SPACE_ALL_PERSIST", - "H5F_FILE_SPACE_ALL", - "H5F_FILE_SPACE_AGGR_VFD", - "H5F_FILE_SPACE_VFD", NULL }; @@ -106,10 +106,12 @@ typedef struct iter_t { hsize_t super_size; /* superblock size */ hsize_t super_ext_size; /* superblock extension size */ hsize_t ublk_size; /* user block size (if exists) */ - H5F_file_space_type_t fs_strategy; /* File space management strategy */ + H5F_fspace_strategy_t fs_strategy; /* File space management strategy */ + hbool_t fs_persist; /* Free-space persist or not */ hsize_t fs_threshold; /* Free-space section threshold */ - hsize_t free_space; /* amount of freespace in the file */ - hsize_t free_hdr; /* size of free space manager metadata in the file */ + hsize_t fsp_size; /* File space page size */ + hsize_t free_space; /* Amount of freespace in the file */ + hsize_t free_hdr; /* Size of free space manager metadata in the file */ unsigned long num_small_sects[SIZE_SMALL_SECTS]; /* Size of small free-space sections */ unsigned sect_nbins; /* Number of bins for free-space section sizes */ unsigned long *sect_bins; /* Pointer to array of bins for free-space section sizes */ @@ -1501,6 +1503,7 @@ print_freespace_info(const iter_t *iter) unsigned long total; /* Total count for various statistics */ unsigned u; /* Local index variable */ + HDfprintf(stdout, "Free-space persist: %s\n", iter->fs_persist ? "TRUE" : "FALSE"); HDfprintf(stdout, "Free-space section threshold: %Hu bytes\n", iter->fs_threshold); printf("Small size free-space sections (< %u bytes):\n", (unsigned)SIZE_SMALL_SECTS); total = 0; @@ -1551,6 +1554,7 @@ print_storage_summary(const iter_t *iter) double percent = 0.0f; HDfprintf(stdout, "File space management strategy: %s\n", FS_STRATEGY_NAME[iter->fs_strategy]); + HDfprintf(stdout, "File space page size: %Hu bytes\n", iter->fsp_size); printf("Summary of file space information:\n"); total_meta = iter->super_size + iter->super_ext_size + iter->ublk_size + @@ -1772,9 +1776,12 @@ main(int argc, const char *argv[]) if(H5Pget_userblock(fcpl, &iter.ublk_size) < 0) warn_msg("Unable to retrieve userblock size\n"); - if(H5Pget_file_space(fcpl, &iter.fs_strategy, &iter.fs_threshold) < 0) + if(H5Pget_file_space_strategy(fcpl, &iter.fs_strategy, &iter.fs_persist, &iter.fs_threshold) < 0) warn_msg("Unable to retrieve file space information\n"); - HDassert(iter.fs_strategy != 0 && iter.fs_strategy < H5F_FILE_SPACE_NTYPES); + HDassert(iter.fs_strategy >= 0 && iter.fs_strategy < H5F_FSPACE_STRATEGY_NTYPES); + + if(H5Pget_file_space_page_size(fcpl, &iter.fsp_size) < 0) + warn_msg("Unable to retrieve file space page size\n"); /* get information for free-space sections */ if(freespace_stats(fid, &iter) < 0) diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c index 481c6db..b6c6ea3 100644 --- a/tools/src/misc/h5clear.c +++ b/tools/src/misc/h5clear.c @@ -14,11 +14,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: - * - * Purpose: + * Purpose: A tool to clear the status_flags field from the file's superblock via -s option. + * A tool to remove cache image from the file via -m option. + * */ - #include "hdf5.h" #include "H5private.h" #include "h5tools.h" @@ -30,59 +29,171 @@ /* Make this private property (defined in H5Fprivate.h) available to h5clear. */ #define H5F_ACS_CLEAR_STATUS_FLAGS_NAME "clear_status_flags" +static char *fname_g = NULL; +static hbool_t clear_status_flags = FALSE; +static hbool_t remove_cache_image = FALSE; + +/* + * Command-line options: The user can specify short or long-named + * parameters. + */ +static const char *s_opts = "hVsm"; +static struct long_options l_opts[] = { + { "help", no_arg, 'h' }, + { "hel", no_arg, 'h'}, + { "he", no_arg, 'h'}, + { "version", no_arg, 'V' }, + { "version", no_arg, 'V' }, + { "versio", no_arg, 'V' }, + { "versi", no_arg, 'V' }, + { "vers", no_arg, 'V' }, + { "status", no_arg, 's' }, + { "statu", no_arg, 's' }, + { "stat", no_arg, 's' }, + { "sta", no_arg, 's' }, + { "st", no_arg, 's' }, + { "image", no_arg, 'm' }, + { "imag", no_arg, 'm' }, + { "ima", no_arg, 'm' }, + { "im", no_arg, 'm' }, + { NULL, 0, '\0' } +}; + + + /*------------------------------------------------------------------------- - * Function: leave + * Function: usage * - * Purpose: Close the tools library and exit + * Purpose: Prints a usage message * - * Return: Does not return + * Return: void * *------------------------------------------------------------------------- */ -static void -leave(int ret) +static void usage(const char *prog) { - h5tools_close(); - HDexit(ret); -} /* leave() */ + HDfprintf(stdout, "usage: %s [OPTIONS] file_name\n", prog); + HDfprintf(stdout, " OPTIONS\n"); + HDfprintf(stdout, " -h, --help Print a usage message and exit\n"); + HDfprintf(stdout, " -V, --version Print version number and exit\n"); + HDfprintf(stdout, " -s, --status Clear the status_flags field in the file's superblock\n"); + HDfprintf(stdout, " -m, --image Remove the metadata cache image from the file\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "Examples of use:\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "h5clear -s file_name\n"); + HDfprintf(stdout, " Clear the status_flags field in the superblock of the HDF5 file <file_name>.\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "h5clear -m file_name\n"); + HDfprintf(stdout, " Remove the metadata cache image from the HDF5 file <file_name>.\n"); +} /* usage() */ + /*------------------------------------------------------------------------- - * Function: usage + * Function: parse_command_line * - * Purpose: Prints a usage message + * Purpose: Parses command line and sets up global variable to control output + * + * Return: Success: 0 * - * Return: void + * Failure: -1 * *------------------------------------------------------------------------- */ -static void -usage(void) +static int +parse_command_line(int argc, const char **argv) { - HDfprintf(stdout, "usage: h5clear filename\n"); + int opt; -} /* usage() */ + /* no arguments */ + if (argc == 1) { + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto error; + } + + /* parse command line options */ + while((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + switch((char)opt) { + case 'h': + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_SUCCESS); + goto done; + + case 'V': + print_version(h5tools_getprogname()); + h5tools_setstatus(EXIT_SUCCESS); + goto done; + + case 's': + clear_status_flags = TRUE; + break; + + case 'm': + remove_cache_image = TRUE; + break; + + default: + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto error; + } /* end switch */ + } /* end while */ + + /* check for file name to be processed */ + if(argc <= opt_ind) { + error_msg("missing file name\n"); + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto error; + } /* end if */ + + fname_g = HDstrdup(argv[opt_ind]); + +done: + return(0); + +error: + return -1; +} + +/*------------------------------------------------------------------------- + * Function: leave + * + * Purpose: Close the tools library and exit + * + * Return: Does not return + * + *------------------------------------------------------------------------- + */ +static void +leave(int ret) +{ + h5tools_close(); + HDexit(ret); +} /* leave() */ /*------------------------------------------------------------------------- * Function: main * - * Purpose: - * - * Return: Success: - * Failure: + * Purpose: To clear the status_flags field in the file's superblock (-s option). + * To remove the cache image from the file (-m option). * - * Programmer: + * Return: Success: 0 + * Failure: 1 * *------------------------------------------------------------------------- */ int -main (int argc, char *argv[]) +main (int argc, const char *argv[]) { - char *fname; /* File name */ - hbool_t clear = TRUE; /* To clear the status_flags in the file's superblock */ - hid_t fapl = -1; /* File access property list */ + char *fname = NULL; /* File name */ + hid_t fapl = -1; /* File access property list */ hid_t fid = -1; /* File ID */ + haddr_t image_addr; + hsize_t image_len; h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); @@ -93,46 +204,80 @@ main (int argc, char *argv[]) /* initialize h5tools lib */ h5tools_init(); - /* Check for the # of arguments */ - if(argc != 2) { - usage(); - leave(EXIT_FAILURE); + /* Parse command line options */ + if(parse_command_line(argc, argv) < 0) + goto done; + + if(fname_g == NULL) + goto done; + + if(!clear_status_flags && !remove_cache_image) { + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto done; } + /* Duplicate the file name */ + fname = HDstrdup(fname_g); + /* Get a copy of the file access property list */ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { error_msg("H5Pcreate\n"); - leave(EXIT_FAILURE); - } + h5tools_setstatus(EXIT_FAILURE); + goto done; + } - /* Set to clear the status_flags in the file's superblock */ - /* This is a private property used by h5clear only */ - if(H5Pset(fapl, H5F_ACS_CLEAR_STATUS_FLAGS_NAME, &clear) < 0) { - error_msg("H5Pset\n"); - leave(EXIT_FAILURE); + /* -s option */ + if(clear_status_flags) { + /* Set to clear the status_flags in the file's superblock */ + /* This is a private property used by h5clear only */ + if(H5Pset(fapl, H5F_ACS_CLEAR_STATUS_FLAGS_NAME, &clear_status_flags) < 0) { + error_msg("H5Pset\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } } - /* Duplicate the file name */ - fname = HDstrdup(argv[opt_ind]); - if((fid = h5tools_fopen(fname, H5F_ACC_RDWR, fapl, NULL, NULL, (size_t)0)) < 0) { error_msg("h5tools_fopen\n"); - HDfree(fname); - leave(EXIT_FAILURE); + h5tools_setstatus(EXIT_FAILURE); + goto done; } - HDfree(fname); - /* Close the file */ - if(H5Fclose(fid) < 0) { - error_msg("H5Fclose\n"); - leave(EXIT_FAILURE); - } + /* -m option */ + if(remove_cache_image) { + H5AC_cache_image_config_t config; - /* CLose the property list */ - if(H5Pclose(fapl) < 0) { - error_msg("H5Pclose\n"); - leave(EXIT_FAILURE); - } + /* Retrieve cache image config */ + if((fapl = H5Fget_access_plist(fid)) < 0) { + error_msg("H5Fget_access_plist\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + config.version = H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION; + if(H5Pget_mdc_image_config(fapl, &config) < 0) { + error_msg("H5Pget_mdc_image_config\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + + /* Check for image */ + if(!config.generate_image) + warn_msg("No cache image in the file\n"); + } + + h5tools_setstatus(EXIT_SUCCESS); + +done: + if(fname) + HDfree(fname); + if(fname_g) + HDfree(fname_g); + + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Fclose(fid); + } H5E_END_TRY leave(h5tools_getstatus()); } /* main() */ diff --git a/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls b/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls index 0134714..af499a5 100644 --- a/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls +++ b/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls @@ -8,7 +8,7 @@ Opened "./testfiles/h5copy_extlinks_src.out.h5" with sec2 driver. Storage: <details removed for portability> Type: 32-bit little-endian integer /copy1_group Group - Location: 1:4696 + Location: 1:4600 Links: 1 /copy1_group/extlink_datatype External Link {h5copy_extlinks_trg.h5//datatype} /copy1_group/extlink_dset External Link {h5copy_extlinks_trg.h5//simple} @@ -16,24 +16,24 @@ Opened "./testfiles/h5copy_extlinks_src.out.h5" with sec2 driver. /copy1_group/extlink_notyet1 External Link {h5copy_extlinks_trg.h5//notyet} /copy1_group/extlink_notyet2 External Link {notyet_file.h5//notyet} /copy2_dset Dataset {6/6} - Location: 1:4216 + Location: 1:4120 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /copy2_group Group - Location: 1:5128 + Location: 1:5032 Links: 1 /copy2_group/extlink_datatype Type - Location: 1:6328 + Location: 1:6232 Links: 1 - Type: shared-1:6328 32-bit little-endian integer + Type: shared-1:6232 32-bit little-endian integer /copy2_group/extlink_dset Dataset {6/6} - Location: 1:5496 + Location: 1:5400 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /copy2_group/extlink_grp Group - Location: 1:6288 + Location: 1:6192 Links: 1 /copy2_group/extlink_notyet1 External Link {h5copy_extlinks_trg.h5//notyet} /copy2_group/extlink_notyet2 External Link {notyet_file.h5//notyet} diff --git a/tools/test/h5copy/testfiles/h5copy_ref.out.ls b/tools/test/h5copy/testfiles/h5copy_ref.out.ls index d685af2..f5b31a7 100644 --- a/tools/test/h5copy/testfiles/h5copy_ref.out.ls +++ b/tools/test/h5copy/testfiles/h5copy_ref.out.ls @@ -11,21 +11,21 @@ Opened "./testfiles/h5copy_ref.out.h5" with sec2 driver. Storage: <details removed for portability> Type: 32-bit little-endian integer /COPY/Dset2 Dataset {3/3, 16/16} - Location: 1:1960 + Location: 1:4096 Links: 3 Storage: <details removed for portability> Type: 8-bit integer /COPY/Dset_OBJREF Dataset {2/2} - Location: 1:5184 + Location: 1:5224 Links: 1 Storage: <details removed for portability> Type: object reference /COPY/Dset_REGREF Dataset {2/2} - Location: 1:5304 + Location: 1:5344 Links: 1 Storage: <details removed for portability> Type: dataset region reference /COPY/Group Group - Location: 1:2096 + Location: 1:1960 Links: 3 -/~obj_pointed_by_2096 Group, same as /COPY/Group +/~obj_pointed_by_1960 Group, same as /COPY/Group diff --git a/tools/test/h5copy/testfiles/h5copytst.h5 b/tools/test/h5copy/testfiles/h5copytst.h5 Binary files differindex 1d1cbf1..474c3da 100644 --- a/tools/test/h5copy/testfiles/h5copytst.h5 +++ b/tools/test/h5copy/testfiles/h5copytst.h5 diff --git a/tools/test/h5copy/testfiles/h5copytst.out.ls b/tools/test/h5copy/testfiles/h5copytst.out.ls index 4044aaf..629915e 100644 --- a/tools/test/h5copy/testfiles/h5copytst.out.ls +++ b/tools/test/h5copy/testfiles/h5copytst.out.ls @@ -3,57 +3,57 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. Location: 1:96 Links: 1 /A Group - Location: 1:84304 + Location: 1:84208 Links: 1 /A/B1 Group - Location: 1:85008 + Location: 1:84912 Links: 1 /A/B1/simple Dataset {6/6} - Location: 1:84176 + Location: 1:84080 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /A/B2 Group - Location: 1:88544 + Location: 1:88448 Links: 1 /A/B2/simple2 Dataset {6/6} - Location: 1:88416 + Location: 1:88320 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /C Group - Location: 1:91752 + Location: 1:91656 Links: 1 /C/D Group - Location: 1:92456 + Location: 1:92360 Links: 1 /C/D/simple Dataset {6/6} - Location: 1:91624 + Location: 1:91528 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /E Group - Location: 1:106368 + Location: 1:106272 Links: 1 /E/F Group - Location: 1:107072 + Location: 1:106976 Links: 1 /E/F/grp_dsets Group - Location: 1:94568 + Location: 1:94472 Links: 1 /E/F/grp_dsets/chunk Dataset {6/6} - Location: 1:98752 + Location: 1:98656 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /E/F/grp_dsets/compact Dataset {6/6} - Location: 1:99208 + Location: 1:99112 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /E/F/grp_dsets/compound Dataset {2/2} - Location: 1:99344 + Location: 1:99248 Links: 1 Storage: <details removed for portability> Type: struct { @@ -61,60 +61,60 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /E/F/grp_dsets/compressed Dataset {6/6} - Location: 1:101656 + Location: 1:101560 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /E/F/grp_dsets/named_vl Dataset {2/2} - Location: 1:105920 + Location: 1:105824 Links: 1 Storage: <details removed for portability> - Type: shared-1:106048 variable length of + Type: shared-1:105952 variable length of 32-bit little-endian integer /E/F/grp_dsets/nested_vl Dataset {2/2} - Location: 1:106096 + Location: 1:106000 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /E/F/grp_dsets/simple Dataset {6/6} - Location: 1:106240 + Location: 1:106144 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /E/F/grp_dsets/vl Type - Location: 1:106048 + Location: 1:105952 Links: 2 - Type: shared-1:106048 variable length of + Type: shared-1:105952 variable length of 32-bit little-endian integer /G Group - Location: 1:122016 + Location: 1:121920 Links: 1 /G/H Group - Location: 1:122720 + Location: 1:122624 Links: 1 /G/H/grp_nested Group - Location: 1:109096 + Location: 1:109000 Links: 1 /G/H/grp_nested/grp_dsets Group - Location: 1:109888 + Location: 1:109792 Links: 1 /G/H/grp_nested/grp_dsets/chunk Dataset {6/6} - Location: 1:114072 + Location: 1:113976 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/compact Dataset {6/6} - Location: 1:114528 + Location: 1:114432 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/compound Dataset {2/2} - Location: 1:114664 + Location: 1:114568 Links: 1 Storage: <details removed for portability> Type: struct { @@ -122,48 +122,48 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /G/H/grp_nested/grp_dsets/compressed Dataset {6/6} - Location: 1:116976 + Location: 1:116880 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/named_vl Dataset {2/2} - Location: 1:121240 + Location: 1:121144 Links: 1 Storage: <details removed for portability> - Type: shared-1:121368 variable length of + Type: shared-1:121272 variable length of 32-bit little-endian integer /G/H/grp_nested/grp_dsets/nested_vl Dataset {2/2} - Location: 1:121416 + Location: 1:121320 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /G/H/grp_nested/grp_dsets/simple Dataset {6/6} - Location: 1:121560 + Location: 1:121464 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/vl Type - Location: 1:121368 + Location: 1:121272 Links: 2 - Type: shared-1:121368 variable length of + Type: shared-1:121272 variable length of 32-bit little-endian integer /chunk Dataset {6/6} - Location: 1:6312 + Location: 1:6216 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /compact Dataset {6/6} - Location: 1:6440 + Location: 1:6344 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /compound Dataset {2/2} - Location: 1:8624 + Location: 1:8528 Links: 1 Storage: <details removed for portability> Type: struct { @@ -171,28 +171,28 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /compressed Dataset {6/6} - Location: 1:12984 + Location: 1:12888 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_dsets Group - Location: 1:28128 + Location: 1:28032 Links: 1 /grp_dsets/chunk Dataset {6/6} - Location: 1:32312 + Location: 1:32216 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/compact Dataset {6/6} - Location: 1:32768 + Location: 1:32672 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/compound Dataset {2/2} - Location: 1:32904 + Location: 1:32808 Links: 1 Storage: <details removed for portability> Type: struct { @@ -200,62 +200,62 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_dsets/compressed Dataset {6/6} - Location: 1:35216 + Location: 1:35120 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_dsets/named_vl Dataset {2/2} - Location: 1:39480 + Location: 1:39384 Links: 1 Storage: <details removed for portability> - Type: shared-1:39608 variable length of + Type: shared-1:39512 variable length of 32-bit little-endian integer /grp_dsets/nested_vl Dataset {2/2} - Location: 1:39656 + Location: 1:39560 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_dsets/simple Dataset {6/6} - Location: 1:39800 + Location: 1:39704 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/simple_group Dataset {6/6} - Location: 1:55912 + Location: 1:55816 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/vl Type - Location: 1:39608 + Location: 1:39512 Links: 2 - Type: shared-1:39608 variable length of + Type: shared-1:39512 variable length of 32-bit little-endian integer /grp_empty Group - Location: 1:27336 + Location: 1:27240 Links: 1 /grp_nested Group - Location: 1:40592 + Location: 1:40496 Links: 1 /grp_nested/grp_dsets Group - Location: 1:41384 + Location: 1:41288 Links: 1 /grp_nested/grp_dsets/chunk Dataset {6/6} - Location: 1:45568 + Location: 1:45472 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_nested/grp_dsets/compact Dataset {6/6} - Location: 1:46024 + Location: 1:45928 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_nested/grp_dsets/compound Dataset {2/2} - Location: 1:46160 + Location: 1:46064 Links: 1 Storage: <details removed for portability> Type: struct { @@ -263,51 +263,51 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_nested/grp_dsets/compressed Dataset {6/6} - Location: 1:48472 + Location: 1:48376 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_nested/grp_dsets/named_vl Dataset {2/2} - Location: 1:52736 + Location: 1:52640 Links: 1 Storage: <details removed for portability> - Type: shared-1:52864 variable length of + Type: shared-1:52768 variable length of 32-bit little-endian integer /grp_nested/grp_dsets/nested_vl Dataset {2/2} - Location: 1:52912 + Location: 1:52816 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_nested/grp_dsets/simple Dataset {6/6} - Location: 1:53056 + Location: 1:52960 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_nested/grp_dsets/vl Type - Location: 1:52864 + Location: 1:52768 Links: 2 - Type: shared-1:52864 variable length of + Type: shared-1:52768 variable length of 32-bit little-endian integer /grp_rename Group - Location: 1:57120 + Location: 1:57024 Links: 1 /grp_rename/chunk Dataset {6/6} - Location: 1:61304 + Location: 1:61208 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/compact Dataset {6/6} - Location: 1:61760 + Location: 1:61664 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/compound Dataset {2/2} - Location: 1:61896 + Location: 1:61800 Links: 1 Storage: <details removed for portability> Type: struct { @@ -315,28 +315,28 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_rename/compressed Dataset {6/6} - Location: 1:64208 + Location: 1:64112 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_rename/grp_dsets Group - Location: 1:70000 + Location: 1:69904 Links: 1 /grp_rename/grp_dsets/chunk Dataset {6/6} - Location: 1:74184 + Location: 1:74088 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/grp_dsets/compact Dataset {6/6} - Location: 1:74640 + Location: 1:74544 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/grp_dsets/compound Dataset {2/2} - Location: 1:74776 + Location: 1:74680 Links: 1 Storage: <details removed for portability> Type: struct { @@ -344,73 +344,73 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_rename/grp_dsets/compressed Dataset {6/6} - Location: 1:77088 + Location: 1:76992 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_rename/grp_dsets/named_vl Dataset {2/2} - Location: 1:81352 + Location: 1:81256 Links: 1 Storage: <details removed for portability> - Type: shared-1:81480 variable length of + Type: shared-1:81384 variable length of 32-bit little-endian integer /grp_rename/grp_dsets/nested_vl Dataset {2/2} - Location: 1:81528 + Location: 1:81432 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_rename/grp_dsets/simple Dataset {6/6} - Location: 1:81672 + Location: 1:81576 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/grp_dsets/vl Type - Location: 1:81480 + Location: 1:81384 Links: 2 - Type: shared-1:81480 variable length of + Type: shared-1:81384 variable length of 32-bit little-endian integer /grp_rename/named_vl Dataset {2/2} - Location: 1:68472 + Location: 1:68376 Links: 1 Storage: <details removed for portability> - Type: shared-1:68600 variable length of + Type: shared-1:68504 variable length of 32-bit little-endian integer /grp_rename/nested_vl Dataset {2/2} - Location: 1:68648 + Location: 1:68552 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_rename/simple Dataset {6/6} - Location: 1:68792 + Location: 1:68696 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/vl Type - Location: 1:68600 + Location: 1:68504 Links: 2 - Type: shared-1:68600 variable length of + Type: shared-1:68504 variable length of 32-bit little-endian integer /named_vl Dataset {2/2} - Location: 1:17280 + Location: 1:17184 Links: 1 Storage: <details removed for portability> - Type: shared-1:17408 variable length of + Type: shared-1:17312 variable length of 32-bit little-endian integer /nested_vl Dataset {2/2} - Location: 1:21760 + Location: 1:21664 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /rename Dataset {2/2} - Location: 1:26128 + Location: 1:26032 Links: 1 Storage: <details removed for portability> Type: struct { @@ -423,7 +423,7 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. Storage: <details removed for portability> Type: 32-bit little-endian integer /simple_top Dataset {6/6} - Location: 1:23952 + Location: 1:23856 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer diff --git a/tools/test/h5diff/testfiles/h5diff_idx.txt b/tools/test/h5diff/testfiles/h5diff_idx.txt deleted file mode 100644 index 754d3ea..0000000 --- a/tools/test/h5diff/testfiles/h5diff_idx.txt +++ /dev/null @@ -1,14 +0,0 @@ - -file1 file2 ---------------------------------------- - x x / - x x /dset - x x /dset_filter - -group : </> and </> -0 differences found -dataset: </dset> and </dset> -0 differences found -dataset: </dset_filter> and </dset_filter> -0 differences found -EXIT CODE: 0 diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index bf9a698..afcf405 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -289,8 +289,9 @@ typedef struct s1_t { #define F64_DIM1 (F64_ARRAY_BUF_LEN / sizeof(int) + 1) /* File 65 macros */ -#define STRATEGY H5F_FILE_SPACE_AGGR_VFD /* File space handling strategy */ -#define THRESHOLD10 10 /* Free space section threshold */ +#define STRATEGY H5F_FSPACE_STRATEGY_NONE /* File space handling strategy */ +#define THRESHOLD10 10 /* Free-space section threshold */ +#define FSPACE_PAGE_SIZE 8192 /* File space page size */ /* "FILE66" macros and for FILE69 */ #define F66_XDIM 8 @@ -7042,8 +7043,9 @@ gent_extlinks(void) /*------------------------------------------------------------------------- * Function: gent_fs_strategy_threshold * - * Purpose: Generate a file with non-default file space strategy and - * non-default free-space section threshold. + * Purpose: Generate a file with non-default file space strategy, + * non-default free-space section threshold, + * non-default file space page size. *------------------------------------------------------------------------- */ static void @@ -7056,7 +7058,8 @@ gent_fs_strategy_threshold(void) fcpl = H5Pcreate(H5P_FILE_CREATE); /* Set file space information */ - H5Pset_file_space(fcpl, STRATEGY, (hsize_t)THRESHOLD10); + H5Pset_file_space_strategy(fcpl, STRATEGY, TRUE, (hsize_t)THRESHOLD10); + H5Pset_file_space_page_size(fcpl, (hsize_t)FSPACE_PAGE_SIZE); /* Create the file with the specified strategy and threshold */ fid = H5Fcreate(FILE65, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT); diff --git a/tools/test/h5format_convert/h5fc_gentest.c b/tools/test/h5format_convert/h5fc_gentest.c index 9ef8e6e..aea84f8 100644 --- a/tools/test/h5format_convert/h5fc_gentest.c +++ b/tools/test/h5format_convert/h5fc_gentest.c @@ -568,7 +568,7 @@ gen_ext(const char *fname, unsigned new_format, unsigned what) H5Pset_shared_mesg_nindexes(fcpl, 4); break; case 2: - H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); break; case 3: H5Pset_istore_k(fcpl, ISTORE_IK); @@ -576,16 +576,16 @@ gen_ext(const char *fname, unsigned new_format, unsigned what) break; case 4: H5Pset_istore_k(fcpl, ISTORE_IK); - H5Pset_file_space(fcpl, H5F_FILE_SPACE_DEFAULT, (hsize_t)2); + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1); break; case 5: H5Pset_shared_mesg_nindexes(fcpl, 4); - H5Pset_file_space(fcpl, H5F_FILE_SPACE_VFD, (hsize_t)0); + H5Pset_file_space_page_size(fcpl, (hsize_t)512); break; case 6: H5Pset_istore_k(fcpl, ISTORE_IK); H5Pset_shared_mesg_nindexes(fcpl, 4); - H5Pset_file_space(fcpl, H5F_FILE_SPACE_AGGR_VFD, (hsize_t)0); + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); break; default: break; diff --git a/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 b/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 Binary files differindex ac7dbd3..6f92057 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 b/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 Binary files differindex a10e8a4..d1aecc2 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 Binary files differindex 68ba2ac..5f2a01a 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 Binary files differindex 1a58089..df86faf 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 Binary files differindex 26e9b25..e41f51a 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 Binary files differindex e5c5e25..af2cb15 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 Binary files differindex 0e3eca7..5e99ddf 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 Binary files differindex cb15f03..8996bf0 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 Binary files differindex d46cef4..df1db97 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 Binary files differindex defbcb3..c498ead 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 b/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 Binary files differindex 58a340d..2f0f063 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 Binary files differindex 3cbc7f4..b71b31e 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl index d1768c8..d9cc0b7 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 Binary files differindex a2c9187..6a8e5eb 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 Binary files differindex fdf4f33..ebc6919 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 Binary files differindex 6bf0a2f..a4178b3 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 Binary files differindex c0c7ecc..b5cd60a 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 Binary files differindex 055cabf..8e63726 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 Binary files differindex f4caaf4..d581e3c 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 Binary files differindex d0bf344..0e02ca1 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 diff --git a/tools/test/h5repack/h5repacktst.c b/tools/test/h5repack/h5repacktst.c index abdda08..69e576d 100644 --- a/tools/test/h5repack/h5repacktst.c +++ b/tools/test/h5repack/h5repacktst.c @@ -79,6 +79,20 @@ #define FNAME18 "h5repack_layout2.h5" +/* Files for testing file space paging */ +#define FSPACE_OUT "h5repack_fspace_OUT.h5" /* The output file */ +#define NELMTS(X) (sizeof(X)/sizeof(X[0])) /* # of elements */ +const char *H5REPACK_FSPACE_FNAMES[] = { + "h5repack_latest.h5", /* 0 */ + "h5repack_default.h5", /* 1 */ + "h5repack_page_persist.h5", /* 2 */ + "h5repack_fsm_aggr_persist.h5", /* 3 */ + "h5repack_page_threshold.h5", /* 4 */ + "h5repack_fsm_aggr_threshold.h5", /* 5 */ + "h5repack_aggr.h5", /* 6 */ + "h5repack_none.h5" /* 7 */ +}; + #define FNAME_UB "ublock.bin" /* obj and region references */ @@ -169,8 +183,10 @@ int main (void) { pack_opt_t pack_options; diff_opt_t diff_options; - hsize_t fs_size = 0; /* free space section threshold */ - H5F_file_space_type_t fs_type = H5F_FILE_SPACE_DEFAULT; /* file space handling strategy */ + + unsigned j; /* Local index variable for testing file space */ + const char *fname; /* File name for testing file space */ + h5_stat_t file_stat; h5_stat_size_t fsize1, fsize2; /* file sizes */ #if defined (H5_HAVE_FILTER_SZIP) @@ -203,8 +219,161 @@ int main (void) * 2) use the h5diff function to compare the input and output file *------------------------------------------------------------------------- */ + /*------------------------------------------------------------------------- + * Testing file space info setting + *------------------------------------------------------------------------- + */ + TESTING(" files with file space info setting--no options (-S, -P, -T, -G) are set"); + j = 0; /* #0 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + TESTING(" files with file space info setting--all options -S, -P, -T, -G are set"); + ++j; /* #1 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_NONE; + pack_options.fs_persist = -1; /* "FALSE" is set via -P 0 */ + pack_options.fs_threshold = 1; + pack_options.fs_pagesize = 8192; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + + TESTING(" files with file space info setting--options -S and -T are set"); + ++j; /* #2 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + pack_options.fs_strategy = -1; /* "FSM_AGGR" specified via -S FSM_AGGR */ + pack_options.fs_threshold = -1; /* "0" specified via -T 0 */ + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + + TESTING(" files with file space info setting-- options -S and -P are set & -L"); + ++j; /* #3 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_PAGE; /* "PAGE" specified via -S */ + pack_options.fs_persist = TRUE; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -P and -T are set & -L"); + ++j; /* #4 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_persist = -1; /* "FALSE" is set via -P 0 */ + pack_options.fs_threshold = 2; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -S and -G are set & -L"); + ++j; /* #5 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_PAGE; + pack_options.fs_pagesize = 8192; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -S, -P, -T, -G are set"); + ++j; /* #6 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_NONE; + pack_options.fs_persist = -1; /* "FALSE" is set via -P 0 */ + pack_options.fs_threshold = 1; + pack_options.fs_pagesize = 8192; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -S, -T, -G are set & -L"); + ++j; /* #7 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_AGGR; + pack_options.fs_threshold = 1; + pack_options.fs_pagesize = 4096; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + /*------------------------------------------------------------------------- * file with fill values *------------------------------------------------------------------------- @@ -212,8 +381,7 @@ int main (void) TESTING(" copy of datasets (fill values)"); - /* fs_type = 0; fs_size = 0 */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME0,FNAME0OUT,&pack_options) < 0) GOERROR; @@ -233,7 +401,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (all datatypes)"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME1,FNAME1OUT,&pack_options) < 0) GOERROR; @@ -253,7 +421,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (attributes)"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME2,FNAME2OUT,&pack_options) < 0) GOERROR; @@ -272,7 +440,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (hardlinks)"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME3,FNAME3OUT,&pack_options) < 0) GOERROR; @@ -292,7 +460,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of allocation early file"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME5,FNAME5OUT,&pack_options) < 0) GOERROR; @@ -322,7 +490,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset1:GZIP=9",&pack_options) < 0) GOERROR; @@ -349,7 +517,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, TRUE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, TRUE) < 0) GOERROR; if (h5repack_addfilter("dset1:GZIP=9",&pack_options) < 0) GOERROR; @@ -377,7 +545,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -415,7 +583,7 @@ int main (void) */ if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset2:SZIP=8,EC",&pack_options) < 0) GOERROR; @@ -447,7 +615,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -476,7 +644,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset1:SHUF",&pack_options) < 0) GOERROR; @@ -500,8 +668,7 @@ int main (void) TESTING(" addding shuffle filter to all"); - /* fs_type = H5F_FILE_SPACE_ALL_PERSIST; fs_size = 1 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("SHUF",&pack_options) < 0) GOERROR; @@ -526,7 +693,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset1:FLET",&pack_options) < 0) GOERROR; @@ -551,7 +718,7 @@ int main (void) TESTING(" adding checksum filter to all"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("FLET",&pack_options) < 0) GOERROR; @@ -576,7 +743,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options) < 0) GOERROR; @@ -616,7 +783,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -638,7 +805,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, TRUE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, TRUE) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -658,7 +825,7 @@ int main (void) */ TESTING(" adding layout chunked to all"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -679,7 +846,7 @@ int main (void) * test an individual object option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:CONTI",&pack_options) < 0) GOERROR; @@ -700,7 +867,7 @@ int main (void) * test all objects option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -717,7 +884,7 @@ int main (void) * do the same test for a file with filters (chunked) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -739,7 +906,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:COMPA",&pack_options) < 0) GOERROR; @@ -760,8 +927,7 @@ int main (void) *------------------------------------------------------------------------- */ - /* fs_type = H5F_FILE_SPACE_ALL; fs_size = 2 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("COMPA",&pack_options) < 0) GOERROR; @@ -783,7 +949,7 @@ int main (void) * layout compact to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CONTI",&pack_options) < 0) GOERROR; @@ -803,7 +969,7 @@ int main (void) * layout compact to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CHUNK=2x5",&pack_options) < 0) GOERROR; @@ -823,7 +989,7 @@ int main (void) * layout compact to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_compact:COMPA",&pack_options) < 0) GOERROR; @@ -842,7 +1008,7 @@ int main (void) * layout contiguous to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:COMPA",&pack_options) < 0) GOERROR; @@ -861,7 +1027,7 @@ int main (void) * layout contiguous to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CHUNK=3x6",&pack_options) < 0) GOERROR; @@ -881,7 +1047,7 @@ int main (void) * layout contiguous to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CONTI",&pack_options) < 0) GOERROR; @@ -900,7 +1066,7 @@ int main (void) * layout chunked to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:COMPA",&pack_options) < 0) GOERROR; @@ -920,7 +1086,7 @@ int main (void) * layout chunked to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CONTI",&pack_options) < 0) GOERROR; @@ -939,7 +1105,7 @@ int main (void) * layout chunked to chunked conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CHUNK=18x13",&pack_options) < 0) GOERROR; @@ -969,8 +1135,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - /* fs_type = H5F_FILE_SPACE_AGGR_VFD; fs_size = 3 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME7,FNAME7OUT,&pack_options) < 0) GOERROR; @@ -995,7 +1160,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_szip:NONE",&pack_options) < 0) GOERROR; @@ -1020,7 +1185,7 @@ int main (void) TESTING(" copy of deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME8,FNAME8OUT,&pack_options) < 0) GOERROR; @@ -1040,7 +1205,7 @@ int main (void) TESTING(" removing deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0, FALSE, fs_type, ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:NONE",&pack_options) < 0) GOERROR; @@ -1062,7 +1227,7 @@ int main (void) TESTING(" copy of shuffle filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME9,FNAME9OUT,&pack_options) < 0) GOERROR; @@ -1078,7 +1243,7 @@ int main (void) TESTING(" removing shuffle filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_shuffle:NONE",&pack_options) < 0) GOERROR; @@ -1096,7 +1261,7 @@ int main (void) TESTING(" copy of fletcher filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME10,FNAME10OUT,&pack_options) < 0) GOERROR; @@ -1112,7 +1277,7 @@ int main (void) TESTING(" removing fletcher filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_fletcher32:NONE",&pack_options) < 0) GOERROR; @@ -1130,7 +1295,7 @@ int main (void) TESTING(" copy of nbit filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME12,FNAME12OUT,&pack_options) < 0) GOERROR; @@ -1146,7 +1311,7 @@ int main (void) TESTING(" removing nbit filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_nbit:NONE",&pack_options) < 0) GOERROR; @@ -1164,7 +1329,7 @@ int main (void) TESTING(" adding nbit filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_int31:NBIT",&pack_options) < 0) GOERROR; @@ -1182,7 +1347,7 @@ int main (void) TESTING(" copy of scaleoffset filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME13,FNAME13OUT,&pack_options) < 0) GOERROR; @@ -1198,7 +1363,7 @@ int main (void) TESTING(" removing scaleoffset filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_scaleoffset:NONE",&pack_options) < 0) GOERROR; @@ -1216,7 +1381,7 @@ int main (void) TESTING(" adding scaleoffset filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_none:SOFF=31,IN",&pack_options) < 0) GOERROR; @@ -1248,8 +1413,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) if (szip_can_encode) { - /* fs_type = H5F_FILE_SPACE_VFD; fs_size = 4 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -1275,7 +1439,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options) < 0) GOERROR; @@ -1306,7 +1470,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("NONE",&pack_options) < 0) GOERROR; @@ -1330,7 +1494,7 @@ int main (void) */ TESTING(" big file"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME14,FNAME14OUT,&pack_options) < 0) GOERROR; @@ -1347,7 +1511,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" external datasets"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME15,FNAME15OUT,&pack_options) < 0) GOERROR; @@ -1364,7 +1528,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" file with userblock"); - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; if(h5repack(FNAME16, FNAME16OUT, &pack_options) < 0) GOERROR; @@ -1383,7 +1547,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" latest file format options"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; pack_options.latest=1; pack_options.grp_compact=10; @@ -1413,7 +1577,7 @@ int main (void) #if defined (H5_HAVE_FILTER_DEFLATE) - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -1442,7 +1606,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; /* add the options for a user block size and user block filename */ @@ -1475,7 +1639,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; /* add the options for aligment */ @@ -1529,7 +1693,7 @@ int main (void) */ TESTING(" file with committed datatypes"); - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; if(h5repack(FNAME17, FNAME17OUT, &pack_options) < 0) @@ -1556,7 +1720,7 @@ int main (void) /* First run without metadata option. No need to verify the correctness */ /* since this has been verified by earlier tests. Just record the file */ /* size of the output file. */ - if(h5repack_init(&pack_options, 0, FALSE, H5F_FILE_SPACE_DEFAULT, (hsize_t)0) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; if(h5repack(FNAME4, FNAME4OUT, &pack_options) < 0) GOERROR; @@ -1567,7 +1731,7 @@ int main (void) GOERROR; /* run it again with metadata option */ - if(h5repack_init(&pack_options, 0, FALSE, H5F_FILE_SPACE_DEFAULT, (hsize_t)0) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; pack_options.meta_block_size = 8192; if(h5repack(FNAME4, FNAME4OUT, &pack_options) < 0) @@ -1625,6 +1789,9 @@ static int make_testfiles(void) { hid_t fid; + hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File access property list */ + unsigned j; /* Local index variable */ /*------------------------------------------------------------------------- * create a file for general copy test @@ -1865,6 +2032,173 @@ int make_testfiles(void) if(H5Fclose(fid) < 0) return -1; + /*------------------------------------------------------------------------- + * create 8 files with combinations ??? + *------------------------------------------------------------------------- */ + + /* Create file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return -1; + + /* Set to use latest library format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + return -1; + + /* + * #0 -- h5repack_latest.h5 + * default: strategy=FSM_AGGR, persist=FALSE, threshold=1 + * default: inpage=4096 + */ + j = 0; + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[j], H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + + /* + * #1 -- h5repack_default.h5 + * default: strategy=FSM_AGGR, persist=FALSE, threshold=1 + * default: inpage=4096 + */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + + /* + * #2 -- h5repack_page_persist.h5 + * Setting: + * strategy=PAGE, persist=TRUE, threshold=1 + * inpage=512 + * latest format + */ + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)512) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + + /* + * #3 -- h5repack_fsm_aggr_persist.h5 + * Setting: + * strategy=FSM_AGGR, persist=TRUE, threshold=1 + * default: inpage=4096 + */ + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + + /* + * #4 -- h5repack_page_threshold.h5 + * Setting: + * strategy=PAGE, persist=FALSE, threshold=3 + * inpage=8192 + * latest format + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)3) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)8192) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + /* + * #5 -- h5repack_fsm_aggr_threshold.h5 + * Setting: + * strategy=FSM_AGGR, persist=FALSE, threshold=3 + * inpage=4096 + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, FALSE, (hsize_t)3) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)FS_PAGESIZE_DEF) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + /* + * #6 -- h5repack_aggr.h5 + * Setting: + * strategy=AGGR, persist=FALSE, threshold=1 + * latest format + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_AGGR, FALSE, (hsize_t)1) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + /* + * #7 -- h5repack_none.h5 + * Setting: + * strategy=NONE, persist=FALSE, threshold=1 + * inpage=8192 + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)8192) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + if(H5Pclose(fapl) < 0) + return -1; + return 0; out: diff --git a/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl b/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl index c499b35..54bf26c 100644 --- a/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl +++ b/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl @@ -5,7 +5,7 @@ DATASET "vds_dset" { STORAGE_LAYOUT { CONTIGUOUS SIZE 576 - OFFSET 2144 + OFFSET 2048 } FILTERS { NONE diff --git a/tools/test/h5repack/testfiles/h5repack-help.txt b/tools/test/h5repack/testfiles/h5repack-help.txt index 049e27d..69b1644 100644 --- a/tools/test/h5repack/testfiles/h5repack-help.txt +++ b/tools/test/h5repack/testfiles/h5repack-help.txt @@ -19,8 +19,10 @@ usage: h5repack [OPTIONS] file1 file2 -a A, --alignment=A Alignment value for H5Pset_alignment -f FILT, --filter=FILT Filter type -l LAYT, --layout=LAYT Layout type - -S FS_STRGY, --fs_strategy=FS_STRGY File space management strategy - -T FS_THRD, --fs_threshold=FS_THRD Free-space section threshold + -S FS_STRATEGY, --fs_strategy=FS_STRATEGY File space management strategy for H5Pset_file_space_strategy + -P FS_PERSIST, --fs_persist=FS_PERSIST Persisting or not persisting free-space for H5Pset_file_space_strategy + -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy + -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size M - is an integer greater than 1, size of dataset in bytes (default is 0) E - is a filename. @@ -36,18 +38,27 @@ usage: h5repack [OPTIONS] file1 file2 --enable-error-stack Prints messages from the HDF5 error stack as they occur. - FS_STRGY is the file space management strategy to use for the output file. - It is a string as listed below: - ALL_PERSIST - Use persistent free-space managers, aggregators and virtual file driver - for file space allocation - ALL - Use non-persistent free-space managers, aggregators and virtual file driver - for file space allocation - AGGR_VFD - Use aggregators and virtual file driver for file space allocation - VFD - Use virtual file driver for file space allocation - - FS_THRD is the free-space section threshold to use for the output file. - It is the minimum size (in bytes) of free-space sections to be tracked - by the the library's free-space managers. + FS_STRATEGY is a string indicating the file space strategy used: + FSM_AGGR: + The mechanisms used in managing file space are free-space managers, aggregators and virtual file driver. + PAGE: + The mechanisms used in managing file space are free-space managers with embedded paged aggregation and virtual file driver. + AGGR: + The mechanisms used in managing file space are aggregators and virtual file driver. + NONE: + The mechanisms used in managing file space are virtual file driver. + The default strategy when not set is FSM_AGGR without persisting free-space. + + FS_PERSIST is 1 to persisting free-space or 0 to not persisting free-space. + The default when not set is not persisting free-space. + The value is ignored for AGGR and NONE strategies. + + FS_THRESHOLD is the minimum size (in bytes) of free-space sections to be tracked by the library. + The default when not set is 1. + The value is ignored for AGGR and NONE strategies. + + FS_PAGESIZE is the size (in bytes) >=512 that is used by the library when the file space strategy PAGE is used. + The default when not set is 4096. FILT - is a string with the format: diff --git a/tools/test/h5stat/CMakeTests.cmake b/tools/test/h5stat/CMakeTests.cmake index f6735d5..ea602ba 100644 --- a/tools/test/h5stat/CMakeTests.cmake +++ b/tools/test/h5stat/CMakeTests.cmake @@ -25,7 +25,6 @@ h5stat_newgrat.ddl h5stat_newgrat-UG.ddl h5stat_newgrat-UA.ddl - h5stat_idx.ddl h5stat_err1_links.ddl h5stat_links1.ddl h5stat_links2.ddl @@ -46,7 +45,6 @@ h5stat_filters.h5 h5stat_tsohm.h5 h5stat_newgrat.h5 - h5stat_idx.h5 h5stat_threshold.h5 ) @@ -196,8 +194,6 @@ ADD_H5_TEST (h5stat_newgrat 0 h5stat_newgrat.h5) ADD_H5_TEST (h5stat_newgrat-UG 0 -G h5stat_newgrat.h5) ADD_H5_TEST (h5stat_newgrat-UA 0 -A h5stat_newgrat.h5) -# h5stat_idx.h5 is generated by h5stat_gentest.c - ADD_H5_TEST (h5stat_idx 0 h5stat_idx.h5) # # Tests for -l (--links) option on h5stat_threshold.h5: # -l 0 (incorrect threshold value) diff --git a/tools/test/h5stat/h5stat_gentest.c b/tools/test/h5stat/h5stat_gentest.c index b1ab168..804f418 100644 --- a/tools/test/h5stat/h5stat_gentest.c +++ b/tools/test/h5stat/h5stat_gentest.c @@ -77,7 +77,7 @@ gen_newgrat_file(const char *fname) goto error; /* Set file space handling strategy */ - if(H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, 1, (hsize_t)1) < 0) goto error; /* Create file */ diff --git a/tools/test/h5stat/testfiles/h5stat_filters.ddl b/tools/test/h5stat/testfiles/h5stat_filters.ddl index 1a4fd72..9f9e146 100644 --- a/tools/test/h5stat/testfiles/h5stat_filters.ddl +++ b/tools/test/h5stat/testfiles/h5stat_filters.ddl @@ -82,12 +82,14 @@ Small # of attributes (objects with 1 to 10 attributes): Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 37312 bytes Raw data: 8659 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_idx.ddl b/tools/test/h5stat/testfiles/h5stat_idx.ddl index b26f1a4..1b6ae0c 100644 --- a/tools/test/h5stat/testfiles/h5stat_idx.ddl +++ b/tools/test/h5stat/testfiles/h5stat_idx.ddl @@ -79,15 +79,17 @@ Small # of attributes (objects with 1 to 10 attributes): Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 965 bytes Raw data: 110 bytes Amount/Percent of tracked free space: 0 bytes/0.0% - Unaccounted space: 1131 bytes -Total space: 2206 bytes + Unaccounted space: 1083 bytes +Total space: 2158 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_idx.h5 b/tools/test/h5stat/testfiles/h5stat_idx.h5 Binary files differindex 303d1f8..83ebcdb 100644 --- a/tools/test/h5stat/testfiles/h5stat_idx.h5 +++ b/tools/test/h5stat/testfiles/h5stat_idx.h5 diff --git a/tools/test/h5stat/testfiles/h5stat_links2.ddl b/tools/test/h5stat/testfiles/h5stat_links2.ddl index 4622884..9fc82cd 100644 --- a/tools/test/h5stat/testfiles/h5stat_links2.ddl +++ b/tools/test/h5stat/testfiles/h5stat_links2.ddl @@ -91,12 +91,14 @@ Attribute bins: # of objects with 10 - 99 attributes: 3 Total # of objects with attributes: 5 Max. # of attributes to objects: 25 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 16128 bytes Raw data: 0 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_newgrat.ddl b/tools/test/h5stat/testfiles/h5stat_newgrat.ddl index e305f58..130fe2f 100644 --- a/tools/test/h5stat/testfiles/h5stat_newgrat.ddl +++ b/tools/test/h5stat/testfiles/h5stat_newgrat.ddl @@ -9,7 +9,7 @@ File information Max. # of objects in group: 35001 File space information for file metadata (in bytes): Superblock: 48 - Superblock extension: 119 + Superblock extension: 186 User block: 0 Object headers: (total/unused) Groups: 5145147/3220092 @@ -78,6 +78,7 @@ Attribute bins: # of objects with 100 - 999 attributes: 1 Total # of objects with attributes: 1 Max. # of attributes to objects: 100 +Free-space persist: TRUE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): # of sections of size 1: 1 @@ -86,10 +87,11 @@ Free-space section bins: # of sections of size 1 - 9: 1 # of sections of size 10 - 99: 4 Total # of sections: 5 -File space management strategy: H5F_FILE_SPACE_ALL_PERSIST +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: - File metadata: 6362036 bytes + File metadata: 6362103 bytes Raw data: 0 bytes Amount/Percent of tracked free space: 132 bytes/0.0% Unaccounted space: 0 bytes -Total space: 6362168 bytes +Total space: 6362235 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_newgrat.h5 b/tools/test/h5stat/testfiles/h5stat_newgrat.h5 Binary files differindex c919b71..0d68e79 100644 --- a/tools/test/h5stat/testfiles/h5stat_newgrat.h5 +++ b/tools/test/h5stat/testfiles/h5stat_newgrat.h5 diff --git a/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl b/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl index fb5568d..af53776 100644 --- a/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl +++ b/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl @@ -8,7 +8,8 @@ Attribute bins: # of objects with 10 - 99 attributes: 3 Total # of objects with attributes: 5 Max. # of attributes to objects: 25 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 16128 bytes Raw data: 0 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl b/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl index ccb23c1..638781b 100644 --- a/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl +++ b/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl @@ -91,12 +91,14 @@ Attribute bins: # of objects with 10 - 99 attributes: 3 Total # of objects with attributes: 5 Max. # of attributes to objects: 25 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 16128 bytes Raw data: 0 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_tsohm.ddl b/tools/test/h5stat/testfiles/h5stat_tsohm.ddl index 4cf33fc..9369950 100644 --- a/tools/test/h5stat/testfiles/h5stat_tsohm.ddl +++ b/tools/test/h5stat/testfiles/h5stat_tsohm.ddl @@ -76,12 +76,14 @@ Small # of attributes (objects with 1 to 10 attributes): Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 3850 bytes Raw data: 0 bytes diff --git a/tools/test/misc/CMakeLists.txt b/tools/test/misc/CMakeLists.txt index 029d7a9..97a6f0c 100644 --- a/tools/test/misc/CMakeLists.txt +++ b/tools/test/misc/CMakeLists.txt @@ -39,4 +39,6 @@ INCLUDE_DIRECTORIES (${HDF5_TOOLS_DIR}/lib) target_link_libraries (clear_open_chk ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) set_target_properties (clear_open_chk PROPERTIES FOLDER tools) - include (CMakeTests.cmake) + include (CMakeTestsRepart.cmake) + include (CMakeTestsClear.cmake) + include (CMakeTestsMkgrp.cmake) diff --git a/tools/test/misc/CMakeTests.cmake b/tools/test/misc/CMakeTests.cmake deleted file mode 100644 index 9ecdeae..0000000 --- a/tools/test/misc/CMakeTests.cmake +++ /dev/null @@ -1,340 +0,0 @@ - -############################################################################## -############################################################################## -### T E S T I N G ### -############################################################################## -############################################################################## - - # -------------------------------------------------------------------- - # Copy all the HDF5 files from the source directory into the test directory - # -------------------------------------------------------------------- - set (HDF5_REFERENCE_TEST_FILES - family_file00000.h5 - family_file00001.h5 - family_file00002.h5 - family_file00003.h5 - family_file00004.h5 - family_file00005.h5 - family_file00006.h5 - family_file00007.h5 - family_file00008.h5 - family_file00009.h5 - family_file00010.h5 - family_file00011.h5 - family_file00012.h5 - family_file00013.h5 - family_file00014.h5 - family_file00015.h5 - family_file00016.h5 - family_file00017.h5 - ) - - foreach (h5_file ${HDF5_REFERENCE_TEST_FILES}) - HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "h5repart_files") - endforeach () - add_custom_target(h5repart_files ALL COMMENT "Copying files needed by h5repart tests" DEPENDS ${h5repart_files_list}) - - set (HDF5_MKGRP_TEST_FILES - #h5mkgrp_help.txt - #h5mkgrp_version - h5mkgrp_single.ls - h5mkgrp_single_v.ls - h5mkgrp_single_p.ls - h5mkgrp_single_l.ls - h5mkgrp_several.ls - h5mkgrp_several_v.ls - h5mkgrp_several_p.ls - h5mkgrp_several_l.ls - h5mkgrp_nested_p.ls - h5mkgrp_nested_lp.ls - h5mkgrp_nested_mult_p.ls - h5mkgrp_nested_mult_lp.ls - ) - - # make test dir - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - - foreach (h5_mkgrp_file ${HDF5_MKGRP_TEST_FILES}) - HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_mkgrp_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5_mkgrp_file}" "h5mkgrp_files") - endforeach () - - HDFTEST_COPY_FILE("${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_help.txt" "${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_help.txt" "h5mkgrp_files") - add_custom_target(h5mkgrp_files ALL COMMENT "Copying files needed by h5mkgrp tests" DEPENDS ${h5mkgrp_files_list}) - - configure_file (${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_version.txt.in ${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_version.txt @ONLY) - -############################################################################## -############################################################################## -### T H E T E S T S M A C R O S ### -############################################################################## -############################################################################## - - macro (ADD_H5_TEST resultfile resultcode resultoption) - if (NOT HDF5_ENABLE_USING_MEMCHECKER) - add_test ( - NAME H5MKGRP-${resultfile}-clear-objects - COMMAND ${CMAKE_COMMAND} - -E remove - ${resultfile}.h5 - ) - set_tests_properties (H5MKGRP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - endif () - - add_test ( - NAME H5MKGRP-${resultfile} - COMMAND $<TARGET_FILE:h5mkgrp> ${resultoption} ${resultfile}.h5 ${ARGN} - ) - set_tests_properties (H5MKGRP-${resultfile} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - if (HDF5_ENABLE_USING_MEMCHECKER) - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS ${last_test}) - endif () - else () - set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS H5MKGRP-${resultfile}-clear-objects) - add_test ( - NAME H5MKGRP-${resultfile}-h5ls - COMMAND "${CMAKE_COMMAND}" - -D "TEST_PROGRAM=$<TARGET_FILE:h5ls>" - -D "TEST_ARGS:STRING=-v;-r;${resultfile}.h5" - -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" - -D "TEST_OUTPUT=${resultfile}.out" - -D "TEST_EXPECT=${resultcode}" - -D "TEST_MASK_MOD=true" - -D "TEST_REFERENCE=${resultfile}.ls" - -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" - ) - set_tests_properties (H5MKGRP-${resultfile}-h5ls PROPERTIES DEPENDS H5MKGRP-${resultfile}) - endif () - endmacro () - - macro (ADD_H5_CMP resultfile resultcode) - if (HDF5_ENABLE_USING_MEMCHECKER) - add_test (NAME H5MKGRP_CMP-${resultfile} COMMAND $<TARGET_FILE:h5mkgrp> ${ARGN}) - else () - add_test ( - NAME H5MKGRP_CMP-${resultfile}-clear-objects - COMMAND ${CMAKE_COMMAND} - -E remove - ${resultfile}.h5 - ) - set_tests_properties (H5MKGRP_CMP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - add_test ( - NAME H5MKGRP_CMP-${resultfile} - COMMAND "${CMAKE_COMMAND}" - -D "TEST_PROGRAM=$<TARGET_FILE:h5mkgrp>" - -D "TEST_ARGS:STRING=${ARGN}" - -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" - -D "TEST_OUTPUT=${resultfile}.out" - -D "TEST_EXPECT=${resultcode}" - -D "TEST_REFERENCE=${resultfile}.txt" - -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" - ) - set_tests_properties (H5MKGRP_CMP-${resultfile} PROPERTIES DEPENDS H5MKGRP_CMP-${resultfile}-clear-objects) - endif () - endmacro () - -############################################################################## -############################################################################## -### T H E T E S T S ### -############################################################################## -############################################################################## - - ###################### H5REPART ######################### - # Remove any output file left over from previous test run - add_test ( - NAME H5REPART-clearall-objects - COMMAND ${CMAKE_COMMAND} - -E remove - fst_family00000.h5 - scd_family00000.h5 - scd_family00001.h5 - scd_family00002.h5 - scd_family00003.h5 - family_to_sec2.h5 - ) - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5REPART-clearall-objects PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "H5REPART-clearall-objects") - - # repartition family member size to 20,000 bytes. - add_test (NAME H5REPART-h5repart_20K COMMAND $<TARGET_FILE:h5repart> -m 20000 family_file%05d.h5 fst_family%05d.h5) - set_tests_properties (H5REPART-h5repart_20K PROPERTIES DEPENDS H5REPART-clearall-objects) - - # repartition family member size to 5 KB. - add_test (NAME H5REPART-h5repart_5K COMMAND $<TARGET_FILE:h5repart> -m 5k family_file%05d.h5 scd_family%05d.h5) - set_tests_properties (H5REPART-h5repart_5K PROPERTIES DEPENDS H5REPART-clearall-objects) - - # convert family file to sec2 file of 20,000 bytes - add_test (NAME H5REPART-h5repart_sec2 COMMAND $<TARGET_FILE:h5repart> -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5) - set_tests_properties (H5REPART-h5repart_sec2 PROPERTIES DEPENDS H5REPART-clearall-objects) - - # test the output files repartitioned above. - add_test (NAME H5REPART-h5repart_test COMMAND $<TARGET_FILE:h5repart_test>) - set_tests_properties (H5REPART-h5repart_test PROPERTIES DEPENDS "H5REPART-clearall-objects;H5REPART-h5repart_20K;H5REPART-h5repart_5K;H5REPART-h5repart_sec2") - - set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} - h5repart_test - ) - - ###################### H5CLEAR ######################### - # Remove any output file left over from previous test run - add_test ( - NAME H5CLEAR-clearall-objects - COMMAND ${CMAKE_COMMAND} - -E remove - h5clear_log_v3.h5 - h5clear_sec2_v0.h5 - h5clear_sec2_v2.h5 - h5clear_sec2_v3.h5 - latest_h5clear_log_v3.h5 - latest_h5clear_sec2_v3.h5 - ) - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5CLEAR-clearall-objects PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "H5CLEAR-clearall-objects") - - # create the output files to be used. - add_test (NAME H5CLEAR-h5clear_gentest COMMAND $<TARGET_FILE:h5clear_gentest>) - set_tests_properties (H5CLEAR-h5clear_gentest PROPERTIES DEPENDS "H5CLEAR-clearall-objects") - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-sec2_v3_F COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-sec2_v3 COMMAND $<TARGET_FILE:h5clear> h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-h5clear-sec2_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-sec2_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-sec2_v3 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-sec2_v3) - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-log_v3_F COMMAND $<TARGET_FILE:clear_open_chk> h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-log_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-log_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-log_v3 COMMAND $<TARGET_FILE:h5clear> h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-h5clear-log_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-log_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-log_v3 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-log_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-log_v3) - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-latest_sec2_v3_F COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_sec2_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-latest_sec2_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-latest_sec2_v3 COMMAND $<TARGET_FILE:h5clear> latest_h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-h5clear-latest_sec2_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-latest_sec2_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-latest_sec2_v3 COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_sec2_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-latest_sec2_v3) - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-latest_log_v3_F COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_log_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-latest_log_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-latest_log_v3 COMMAND $<TARGET_FILE:h5clear> latest_h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-h5clear-latest_log_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-latest_log_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-latest_log_v3 COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_log_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-latest_log_v3) - - # - # File open succeeds because the library does not check status_flags for file with < v3 superblock - add_test (NAME H5CLEAR-clear_open_chk-sec2_v0_P COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v0.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v0_P PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-sec2_v0 COMMAND $<TARGET_FILE:h5clear> h5clear_sec2_v0.h5) - set_tests_properties (H5CLEAR-h5clear-sec2_v0 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-sec2_v0_P) - add_test (NAME H5CLEAR-clear_open_chk-sec2_v0 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v0.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v0 PROPERTIES DEPENDS H5CLEAR-h5clear-sec2_v0) - - # - # File open succeeds because the library does not check status_flags for file with < v3 superblock - add_test (NAME H5CLEAR-clear_open_chk-sec2_v2_P COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v2.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v2_P PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-sec2_v2 COMMAND $<TARGET_FILE:h5clear> h5clear_sec2_v2.h5) - set_tests_properties (H5CLEAR-h5clear-sec2_v2 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-sec2_v2_P) - add_test (NAME H5CLEAR-clear_open_chk-sec2_v2 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v2.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v2 PROPERTIES DEPENDS H5CLEAR-h5clear-sec2_v2) - - set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} - h5clear_gentest - ) - - ###################### H5MKGRP ######################### - if (HDF5_ENABLE_USING_MEMCHECKER) - add_test ( - NAME H5MKGRP-clearall-objects - COMMAND ${CMAKE_COMMAND} - -E remove - h5mkgrp_help.out - h5mkgrp_help.out.err - h5mkgrp_version.out - h5mkgrp_version.out.err - h5mkgrp_single.h5 - h5mkgrp_single.out - h5mkgrp_single.out.err - h5mkgrp_single_v.h5 - h5mkgrp_single_v.out - h5mkgrp_single_v.out.err - h5mkgrp_single_p.h5 - h5mkgrp_single_p.out - h5mkgrp_single_p.out.err - h5mkgrp_single_l.h5 - h5mkgrp_single_l.out - h5mkgrp_single_l.out.err - h5mkgrp_several.h5 - h5mkgrp_several.out - h5mkgrp_several.out.err - h5mkgrp_several_v.h5 - h5mkgrp_several_v.out - h5mkgrp_several_v.out.err - h5mkgrp_several_p.h5 - h5mkgrp_several_p.out - h5mkgrp_several_p.out.err - h5mkgrp_several_l.h5 - h5mkgrp_several_l.out - h5mkgrp_several_l.out.err - h5mkgrp_nested_p.h5 - h5mkgrp_nested_p.out - h5mkgrp_nested_p.out.err - h5mkgrp_nested_lp.h5 - h5mkgrp_nested_lp.out - h5mkgrp_nested_lp.out.err - h5mkgrp_nested_mult_p.h5 - h5mkgrp_nested_mult_p.out - h5mkgrp_nested_mult_p.out.err - h5mkgrp_nested_mult_lp.h5 - h5mkgrp_nested_mult_lp.out - h5mkgrp_nested_mult_lp.out.err - ) - set_tests_properties (H5MKGRP-clearall-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5MKGRP-clearall-objects PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "H5MKGRP-clearall-objects") - endif () - - # Check that help & version is displayed properly - ADD_H5_CMP (h5mkgrp_help 0 "-h") - ADD_H5_CMP (h5mkgrp_version 0 "-V") - - # Create single group at root level - ADD_H5_TEST (h5mkgrp_single 0 "" single) - ADD_H5_TEST (h5mkgrp_single_v 0 "-v" single) - ADD_H5_TEST (h5mkgrp_single_p 0 "-p" single) - ADD_H5_TEST (h5mkgrp_single_l 0 "-l" latest) - - # Create several groups at root level - ADD_H5_TEST (h5mkgrp_several 0 "" one two) - ADD_H5_TEST (h5mkgrp_several_v 0 "-v" one two) - ADD_H5_TEST (h5mkgrp_several_p 0 "-p" one two) - ADD_H5_TEST (h5mkgrp_several_l 0 "-l" one two) - - # Create various nested groups - ADD_H5_TEST (h5mkgrp_nested_p 0 "-p" /one/two) - ADD_H5_TEST (h5mkgrp_nested_lp 0 "-lp" /one/two) - ADD_H5_TEST (h5mkgrp_nested_mult_p 0 "-p" /one/two /three/four) - ADD_H5_TEST (h5mkgrp_nested_mult_lp 0 "-lp" /one/two /three/four) diff --git a/tools/test/misc/CMakeTestsClear.cmake b/tools/test/misc/CMakeTestsClear.cmake new file mode 100644 index 0000000..a92d523 --- /dev/null +++ b/tools/test/misc/CMakeTestsClear.cmake @@ -0,0 +1,197 @@ + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (HDF5_REFERENCE_TEST_FILES + h5clear_usage.ddl + h5clear_open_fail.ddl + h5clear_missing_file.ddl + h5clear_no_mdc_image.ddl + orig_h5clear_sec2_v0.h5 + orig_h5clear_sec2_v2.h5 + orig_h5clear_sec2_v3.h5 + mod_h5clear_mdc_image.h5 + ) + + foreach (h5_file ${HDF5_REFERENCE_TEST_FILES}) + HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5_file}" "h5clear_files") + endforeach () + # make second copy of mod_h5clear_mdc_image.h5 + HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/mod_h5clear_mdc_image.h5" "${PROJECT_BINARY_DIR}/testfiles/mod_h5clear_mdc_image2.h5" "h5clear_files") + add_custom_target(h5clear_files ALL COMMENT "Copying files needed by h5clear tests" DEPENDS ${h5clear_files_list}) + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_CMP testname resultfile resultcode) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5CLEAR_CMP-${testname}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + ${testname}.out + ${testname}.out.err + ) + add_test ( + NAME H5CLEAR_CMP-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5clear>" + -D "TEST_ARGS:STRING=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5CLEAR_CMP-${testname} PROPERTIES DEPENDS H5CLEAR_CMP-${testname}-clear-objects) + endif () + endmacro () + + macro (ADD_H5_RETTEST testname resultcode) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5CLEAR_RET-${testname} + COMMAND $<TARGET_FILE:h5clear> ${ARGN} + ) + set_tests_properties (H5CLEAR_RET-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set_tests_properties (H5CLEAR_RET-${testname} PROPERTIES WILL_FAIL "${resultcode}") + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR_RET-${testname} PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5CLEAR_RET-${testname}") + endif () + endmacro () + + macro (ADD_H5_TEST testname resultcode) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + # Initial file open fails OR + # File open succeeds because the library does not check status_flags for file with < v3 superblock + add_test (NAME H5CLEAR-clear_open_chk-${testname}_${resultcode} COMMAND $<TARGET_FILE:clear_open_chk> ${testname}.h5) + set_tests_properties (H5CLEAR-clear_open_chk-${testname}_${resultcode} PROPERTIES WILL_FAIL "${resultcode}") + set_tests_properties (H5CLEAR-clear_open_chk-${testname}_${resultcode} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR-clear_open_chk-${testname}_${resultcode} PROPERTIES DEPENDS ${last_test}) + endif () + # After "h5clear" the file, the subsequent file open succeeds + add_test (NAME H5CLEAR-h5clear-${testname} COMMAND $<TARGET_FILE:h5clear> -s ${testname}.h5) + set_tests_properties (H5CLEAR-h5clear-${testname} PROPERTIES DEPENDS H5CLEAR-clear_open_chk-${testname}_${resultcode}) + set_tests_properties (H5CLEAR-h5clear-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + add_test (NAME H5CLEAR-clear_open_chk-${testname} COMMAND $<TARGET_FILE:clear_open_chk> ${testname}.h5) + set_tests_properties (H5CLEAR-clear_open_chk-${testname} PROPERTIES DEPENDS H5CLEAR-h5clear-${testname}) + set_tests_properties (H5CLEAR-clear_open_chk-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set (last_test "H5CLEAR-clear_open_chk-${testname}") + endif () + endmacro () + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## +# +# +# +# The following are tests to verify the status_flags field is cleared properly: + # Remove any output file left over from previous test run + add_test ( + NAME H5CLEAR-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + h5clear_log_v3.h5 + h5clear_mdc_image.h5 + h5clear_sec2_v0.h5 + h5clear_sec2_v2.h5 + h5clear_sec2_v3.h5 + latest_h5clear_log_v3.h5 + latest_h5clear_sec2_v3.h5 + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5CLEAR-clearall-objects") + + # create the output files to be used. + add_test (NAME H5CLEAR-h5clear_gentest COMMAND $<TARGET_FILE:h5clear_gentest>) + set_tests_properties (H5CLEAR-h5clear_gentest PROPERTIES DEPENDS "H5CLEAR-clearall-objects") + set_tests_properties (H5CLEAR-h5clear_gentest PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set (last_test "H5CLEAR-h5clear_gentest") + +# +# +# +# The following are tests to verify the expected output from h5clear +# "h5clear -h" +# "h5clear" (no options, no file) +# "h5clear junk.h5" (no options, nonexisting file) +# "h5clear orig_h5clear_sec2_v3.h5" (no options, existing file) +# "h5clear -m" (valid 1 option, no file) +# "h5clear -s junk.h5" (valid 1 option, nonexisting file) +# "h5clear -m -s junk.h5" (valid 2 options, no file) +# "h5clear -m -s junk.h5" (valid 2 options, nonexisting file) +# "h5clear -m orig_h5clear_sec2_v2.h5" (valid 1 option, existing file, no cache image) +# "h5clear -s -m orig_h5clear_sec2_v0.h5" (valid 2 options, existing file, no cache image) + ADD_H5_CMP (h5clear_usage_h h5clear_usage 0 "-h") + ADD_H5_CMP (h5clear_usage h5clear_usage 1 "") + ADD_H5_CMP (h5clear_usage_junk h5clear_usage 1 "" junk.h5) + ADD_H5_CMP (h5clear_usage_none h5clear_usage 1 "" orig_h5clear_sec2_v3.h5) + ADD_H5_CMP (h5clear_missing_file_m h5clear_missing_file 1 "-m") + ADD_H5_CMP (h5clear_open_fail_s h5clear_open_fail 1 "-s" junk.h5) + ADD_H5_CMP (h5clear_missing_file_ms h5clear_missing_file 1 "-m" "-s") + ADD_H5_CMP (h5clear_open_fail_ms h5clear_open_fail 1 "-m" "-s" junk.h5) + ADD_H5_CMP (h5clear_no_mdc_image_m h5clear_no_mdc_image 0 "-m" orig_h5clear_sec2_v2.h5) + ADD_H5_CMP (h5clear_no_mdc_image_ms h5clear_no_mdc_image 0 "-m" "-s" orig_h5clear_sec2_v0.h5) +# +# +# +# The following are tests to verify the expected exit code from h5clear: +# "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) +# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear -k" (invalid 1 option, no file, fail exit code) +# "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) +# "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) +# "h5clear -m -k" (valid/invalid 2 options, nofile, fail exit code) +# "h5clear -l -m" (invalid/valid 2 options, nofile, fail exit code) +# "h5clear -m -k junk.h5" (valid/invalid 2 options, nonexisting file, fail exit code) +# "h5clear -l -m junk.h5" (invalid/valid 2 options, nonexisting file, fail exit code) +# "h5clear -m -k h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) +# "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) + ADD_H5_RETTEST (h5clear_mdc_image "false" "-m" h5clear_mdc_image.h5) + ADD_H5_RETTEST (h5clear_vers "false" "--vers") + ADD_H5_RETTEST (h5clear_k "true" "-k") + ADD_H5_RETTEST (h5clear_k_junk "true" "-k" junk.h5) + ADD_H5_RETTEST (h5clear_l_sec2 "true" "-l" h5clear_sec2_v2.h5) + ADD_H5_RETTEST (h5clear_mk "true" "-m" "-k") + ADD_H5_RETTEST (h5clear_lm "true" "-l" "-m") + ADD_H5_RETTEST (h5clear_ml_junk "true" "-m" "-l" junk.h5) + ADD_H5_RETTEST (h5clear_lm_junk "true" "-l" "-m" junk.h5) + ADD_H5_RETTEST (h5clear_ml_sec2 "true" "-m" "-l" h5clear_sec2_v2.h5) + ADD_H5_RETTEST (h5clear_lm_sec2 "true" "-l" "-m" h5clear_sec2_v2.h5) +# +# +# +# h5clear_mdc_image.h5 already has cache image removed earlier, verify the expected warning from h5clear: + ADD_H5_CMP (h5clear_mdc_image_m h5clear_no_mdc_image 0 "-m" mod_h5clear_mdc_image.h5) + ADD_H5_CMP (h5clear_mdc_image_sm h5clear_no_mdc_image 0 "-m" "-s" mod_h5clear_mdc_image2.h5) +# +# +# +# The following are tests to verify the status_flags field is cleared properly: + ADD_H5_TEST (h5clear_sec2_v3 "true") + ADD_H5_TEST (h5clear_log_v3 "true") + ADD_H5_TEST (latest_h5clear_sec2_v3 "true") + ADD_H5_TEST (latest_h5clear_log_v3 "true") + ADD_H5_TEST (h5clear_sec2_v0 "false") + ADD_H5_TEST (h5clear_sec2_v2 "false") + + set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} + h5clear_gentest + ) diff --git a/tools/test/misc/CMakeTestsMkgrp.cmake b/tools/test/misc/CMakeTestsMkgrp.cmake new file mode 100644 index 0000000..2a54a0e --- /dev/null +++ b/tools/test/misc/CMakeTestsMkgrp.cmake @@ -0,0 +1,188 @@ + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (HDF5_MKGRP_TEST_FILES + #h5mkgrp_help.txt + #h5mkgrp_version + h5mkgrp_single.ls + h5mkgrp_single_v.ls + h5mkgrp_single_p.ls + h5mkgrp_single_l.ls + h5mkgrp_several.ls + h5mkgrp_several_v.ls + h5mkgrp_several_p.ls + h5mkgrp_several_l.ls + h5mkgrp_nested_p.ls + h5mkgrp_nested_lp.ls + h5mkgrp_nested_mult_p.ls + h5mkgrp_nested_mult_lp.ls + ) + + # make test dir + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + + foreach (h5_mkgrp_file ${HDF5_MKGRP_TEST_FILES}) + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_mkgrp_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5_mkgrp_file}" "h5mkgrp_files") + endforeach () + + HDFTEST_COPY_FILE("${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_help.txt" "${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_help.txt" "h5mkgrp_files") + add_custom_target(h5mkgrp_files ALL COMMENT "Copying files needed by h5mkgrp tests" DEPENDS ${h5mkgrp_files_list}) + + configure_file (${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_version.txt.in ${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_version.txt @ONLY) + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_TEST resultfile resultcode resultoption) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5MKGRP-${resultfile}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + ${resultfile}.h5 + ) + set_tests_properties (H5MKGRP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + endif () + + add_test ( + NAME H5MKGRP-${resultfile} + COMMAND $<TARGET_FILE:h5mkgrp> ${resultoption} ${resultfile}.h5 ${ARGN} + ) + set_tests_properties (H5MKGRP-${resultfile} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (HDF5_ENABLE_USING_MEMCHECKER) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS ${last_test}) + endif () + else (HDF5_ENABLE_USING_MEMCHECKER) + set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS H5MKGRP-${resultfile}-clear-objects) + add_test ( + NAME H5MKGRP-${resultfile}-h5ls + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5ls>" + -D "TEST_ARGS:STRING=-v;-r;${resultfile}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_MASK_MOD=true" + -D "TEST_REFERENCE=${resultfile}.ls" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5MKGRP-${resultfile}-h5ls PROPERTIES DEPENDS H5MKGRP-${resultfile}) + endif () + endmacro () + + macro (ADD_H5_CMP resultfile resultcode) + if (HDF5_ENABLE_USING_MEMCHECKER) + add_test (NAME H5MKGRP_CMP-${resultfile} COMMAND $<TARGET_FILE:h5mkgrp> ${ARGN}) + else () + add_test ( + NAME H5MKGRP_CMP-${resultfile}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + ${resultfile}.h5 + ) + set_tests_properties (H5MKGRP_CMP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + add_test ( + NAME H5MKGRP_CMP-${resultfile} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5mkgrp>" + -D "TEST_ARGS:STRING=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.txt" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5MKGRP_CMP-${resultfile} PROPERTIES DEPENDS H5MKGRP_CMP-${resultfile}-clear-objects) + endif () + endmacro () + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + if (HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5MKGRP-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + h5mkgrp_help.out + h5mkgrp_help.out.err + h5mkgrp_version.out + h5mkgrp_version.out.err + h5mkgrp_single.h5 + h5mkgrp_single.out + h5mkgrp_single.out.err + h5mkgrp_single_v.h5 + h5mkgrp_single_v.out + h5mkgrp_single_v.out.err + h5mkgrp_single_p.h5 + h5mkgrp_single_p.out + h5mkgrp_single_p.out.err + h5mkgrp_single_l.h5 + h5mkgrp_single_l.out + h5mkgrp_single_l.out.err + h5mkgrp_several.h5 + h5mkgrp_several.out + h5mkgrp_several.out.err + h5mkgrp_several_v.h5 + h5mkgrp_several_v.out + h5mkgrp_several_v.out.err + h5mkgrp_several_p.h5 + h5mkgrp_several_p.out + h5mkgrp_several_p.out.err + h5mkgrp_several_l.h5 + h5mkgrp_several_l.out + h5mkgrp_several_l.out.err + h5mkgrp_nested_p.h5 + h5mkgrp_nested_p.out + h5mkgrp_nested_p.out.err + h5mkgrp_nested_lp.h5 + h5mkgrp_nested_lp.out + h5mkgrp_nested_lp.out.err + h5mkgrp_nested_mult_p.h5 + h5mkgrp_nested_mult_p.out + h5mkgrp_nested_mult_p.out.err + h5mkgrp_nested_mult_lp.h5 + h5mkgrp_nested_mult_lp.out + h5mkgrp_nested_mult_lp.out.err + ) + set_tests_properties (H5MKGRP-clearall-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5MKGRP-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5MKGRP-clearall-objects") + endif () + + # Check that help & version is displayed properly + ADD_H5_CMP (h5mkgrp_help 0 "-h") + ADD_H5_CMP (h5mkgrp_version 0 "-V") + + # Create single group at root level + ADD_H5_TEST (h5mkgrp_single 0 "" single) + ADD_H5_TEST (h5mkgrp_single_v 0 "-v" single) + ADD_H5_TEST (h5mkgrp_single_p 0 "-p" single) + ADD_H5_TEST (h5mkgrp_single_l 0 "-l" latest) + + # Create several groups at root level + ADD_H5_TEST (h5mkgrp_several 0 "" one two) + ADD_H5_TEST (h5mkgrp_several_v 0 "-v" one two) + ADD_H5_TEST (h5mkgrp_several_p 0 "-p" one two) + ADD_H5_TEST (h5mkgrp_several_l 0 "-l" one two) + + # Create various nested groups + ADD_H5_TEST (h5mkgrp_nested_p 0 "-p" /one/two) + ADD_H5_TEST (h5mkgrp_nested_lp 0 "-lp" /one/two) + ADD_H5_TEST (h5mkgrp_nested_mult_p 0 "-p" /one/two /three/four) + ADD_H5_TEST (h5mkgrp_nested_mult_lp 0 "-lp" /one/two /three/four) diff --git a/tools/test/misc/CMakeTestsRepart.cmake b/tools/test/misc/CMakeTestsRepart.cmake new file mode 100644 index 0000000..a1549ec --- /dev/null +++ b/tools/test/misc/CMakeTestsRepart.cmake @@ -0,0 +1,85 @@ + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (HDF5_REFERENCE_TEST_FILES + family_file00000.h5 + family_file00001.h5 + family_file00002.h5 + family_file00003.h5 + family_file00004.h5 + family_file00005.h5 + family_file00006.h5 + family_file00007.h5 + family_file00008.h5 + family_file00009.h5 + family_file00010.h5 + family_file00011.h5 + family_file00012.h5 + family_file00013.h5 + family_file00014.h5 + family_file00015.h5 + family_file00016.h5 + family_file00017.h5 + ) + + foreach (h5_file ${HDF5_REFERENCE_TEST_FILES}) + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "h5repart_files") + endforeach () + add_custom_target(h5repart_files ALL COMMENT "Copying files needed by h5repart tests" DEPENDS ${h5repart_files_list}) + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + + # Remove any output file left over from previous test run + add_test ( + NAME H5REPART-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + fst_family00000.h5 + scd_family00000.h5 + scd_family00001.h5 + scd_family00002.h5 + scd_family00003.h5 + family_to_sec2.h5 + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5REPART-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5REPART-clearall-objects") + + # repartition family member size to 20,000 bytes. + add_test (NAME H5REPART-h5repart_20K COMMAND $<TARGET_FILE:h5repart> -m 20000 family_file%05d.h5 fst_family%05d.h5) + set_tests_properties (H5REPART-h5repart_20K PROPERTIES DEPENDS H5REPART-clearall-objects) + + # repartition family member size to 5 KB. + add_test (NAME H5REPART-h5repart_5K COMMAND $<TARGET_FILE:h5repart> -m 5k family_file%05d.h5 scd_family%05d.h5) + set_tests_properties (H5REPART-h5repart_5K PROPERTIES DEPENDS H5REPART-clearall-objects) + + # convert family file to sec2 file of 20,000 bytes + add_test (NAME H5REPART-h5repart_sec2 COMMAND $<TARGET_FILE:h5repart> -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5) + set_tests_properties (H5REPART-h5repart_sec2 PROPERTIES DEPENDS H5REPART-clearall-objects) + + # test the output files repartitioned above. + add_test (NAME H5REPART-h5repart_test COMMAND $<TARGET_FILE:h5repart_test>) + set_tests_properties (H5REPART-h5repart_test PROPERTIES DEPENDS "H5REPART-clearall-objects;H5REPART-h5repart_20K;H5REPART-h5repart_5K;H5REPART-h5repart_sec2") + + set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} + h5repart_test + ) diff --git a/tools/test/misc/h5clear_gentest.c b/tools/test/misc/h5clear_gentest.c index 0f20c35..f8b6a70 100644 --- a/tools/test/misc/h5clear_gentest.c +++ b/tools/test/misc/h5clear_gentest.c @@ -25,6 +25,103 @@ const char *FILENAME[] = { #define KB 1024U +#define CACHE_IMAGE_FILE "h5clear_mdc_image.h5" +#define DSET "DSET" + +/*------------------------------------------------------------------------- + * Function: gen_cache_image_file + * + * Purpose: To create a file with cache image feature enabled. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; March 2017 + * + *------------------------------------------------------------------------- + */ +static int +gen_cache_image_file(const char *fname) +{ + hid_t fid = -1; /* File ID */ + hid_t did = -1, sid = -1; /* Dataset ID, dataspace ID */ + hid_t fapl = -1; /* File access property list */ + hid_t dcpl = -1; /* Dataset creation property list */ + hsize_t dims[2]; /* Dimension sizes */ + hsize_t chunks[2]; /* Chunked dimension sizes */ + int buf[50][100]; /* Buffer for data to write */ + int i, j; /* Local index variables */ + H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */ + { H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, + TRUE, FALSE, + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; + + /* Create a copy of file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + + /* Enable latest format in fapl */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + goto error; + + /* Enable metadata cache image in fapl */ + if(H5Pset_mdc_image_config(fapl, &cache_image_config) < 0) + goto error; + + /* Create the file */ + if((fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + goto error; + + /* Create dataspace */ + dims[0] = 50; + dims[1] = 100; + if((sid = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + + /* Initialize buffer for writing to dataset */ + for(i = 0; i < 50; i++) + for(j = 0; j < 100; j++) + buf[i][j] = i * j; + + /* Set up to create a chunked dataset */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + chunks[0] = 5; + chunks[1] = 10; + if(H5Pset_chunk(dcpl, 2, chunks) < 0) + goto error; + if((did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Write to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + goto error; + + /* Closing */ + if(H5Dclose(did) < 0) + goto error; + if(H5Pclose(dcpl) < 0) + goto error; + if(H5Pclose(fapl) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Fclose(fid) < 0) + goto error; + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(sid); + H5Dclose(did); + H5Fclose(fid); + H5Pclose(fapl); + H5Pclose(dcpl); + } H5E_END_TRY; + return 1; +} + /*------------------------------------------------------------------------- * Function: main * @@ -60,6 +157,14 @@ main(void) char fname[512]; /* File name */ unsigned new_format; /* To use latest library format or not */ + /* Generate a file with cache image feature enabled */ + if(gen_cache_image_file(CACHE_IMAGE_FILE) < 0) + goto error; + + /* + * Generate files with invalid status_flags + */ + /* Create a copy of the file access property list */ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; diff --git a/tools/test/misc/testfiles/h5clear_missing_file.ddl b/tools/test/misc/testfiles/h5clear_missing_file.ddl new file mode 100644 index 0000000..1e5150c --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_missing_file.ddl @@ -0,0 +1,15 @@ +usage: h5clear [OPTIONS] file_name + OPTIONS + -h, --help Print a usage message and exit + -V, --version Print version number and exit + -s, --status Clear the status_flags field in the file's superblock + -m, --image Remove the metadata cache image from the file + +Examples of use: + +h5clear -s file_name + Clear the status_flags field in the superblock of the HDF5 file <file_name>. + +h5clear -m file_name + Remove the metadata cache image from the HDF5 file <file_name>. +h5clear error: missing file name diff --git a/tools/test/misc/testfiles/h5clear_no_mdc_image.ddl b/tools/test/misc/testfiles/h5clear_no_mdc_image.ddl new file mode 100644 index 0000000..f5acd71 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_no_mdc_image.ddl @@ -0,0 +1 @@ +h5clear warning: No cache image in the file diff --git a/tools/test/misc/testfiles/h5clear_open_fail.ddl b/tools/test/misc/testfiles/h5clear_open_fail.ddl new file mode 100644 index 0000000..895ecd4 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_open_fail.ddl @@ -0,0 +1 @@ +h5clear error: h5tools_fopen diff --git a/tools/test/misc/testfiles/h5clear_usage.ddl b/tools/test/misc/testfiles/h5clear_usage.ddl new file mode 100644 index 0000000..a399ae7 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_usage.ddl @@ -0,0 +1,14 @@ +usage: h5clear [OPTIONS] file_name + OPTIONS + -h, --help Print a usage message and exit + -V, --version Print version number and exit + -s, --status Clear the status_flags field in the file's superblock + -m, --image Remove the metadata cache image from the file + +Examples of use: + +h5clear -s file_name + Clear the status_flags field in the superblock of the HDF5 file <file_name>. + +h5clear -m file_name + Remove the metadata cache image from the HDF5 file <file_name>. diff --git a/tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 b/tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 Binary files differnew file mode 100644 index 0000000..310134b --- /dev/null +++ b/tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 diff --git a/tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 b/tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 Binary files differnew file mode 100644 index 0000000..571fd46 --- /dev/null +++ b/tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 diff --git a/tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 b/tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 Binary files differnew file mode 100644 index 0000000..92833e6 --- /dev/null +++ b/tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 diff --git a/tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 b/tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 Binary files differnew file mode 100644 index 0000000..9f1e6ce --- /dev/null +++ b/tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 diff --git a/tools/test/misc/testh5clear.sh.in b/tools/test/misc/testh5clear.sh.in index a6836d4..71d5cad 100644 --- a/tools/test/misc/testh5clear.sh.in +++ b/tools/test/misc/testh5clear.sh.in @@ -20,11 +20,19 @@ TESTNAME=h5clear EXIT_SUCCESS=0 EXIT_FAILURE=1 -H5CLEAR=../../src/misc/h5clear -H5CLEAR_BIN=`pwd`/$H5CLEAR # The path of the tool binary +H5CLEAR=../../src/misc/h5clear # The tool name +H5CLEAR_BIN=`pwd`/$H5CLEAR # The path of the tool binary -OPENCHK=clear_open_chk # Try opening the test file -OPENCHK_BIN=`pwd`/$OPENCHK # The path to the binary +OPENCHK=clear_open_chk # Program to check whether the file can be opened +OPENCHK_BIN=`pwd`/$OPENCHK # The path to the binary + +RM='rm -rf' +CMP='cmp -s' +DIFF='diff -c' +CP='cp' +DIRNAME='dirname' +LS='ls' +AWK='awk' SUCCEED=0 FAIL=1 @@ -32,6 +40,83 @@ FAIL=1 nerrors=0 verbose=yes +# source dirs +SRC_TOOLS="$srcdir/../.." + +SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" +# testfiles source dirs for tools +SRC_H5CLEAR_TESTFILES="$SRC_TOOLS/test/misc/testfiles" + +TESTDIR=./testh5clear +test -d $TESTDIR || mkdir -p $TESTDIR + +###################################################################### +# test files +# -------------------------------------------------------------------- +# All the test files copy from source directory to test directory +# NOTE: Keep this framework to add/remove test files. +# Any test files from other tools can be used in this framework. +# This list are also used for checking exist. +# Comment '#' without space can be used. +# -------------------------------------------------------------------- + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES=" +$SRC_H5CLEAR_TESTFILES/h5clear_usage.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_open_fail.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_missing_file.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_no_mdc_image.ddl +$SRC_H5CLEAR_TESTFILES/orig_h5clear_sec2_v0.h5 +$SRC_H5CLEAR_TESTFILES/orig_h5clear_sec2_v2.h5 +$SRC_H5CLEAR_TESTFILES/orig_h5clear_sec2_v3.h5 +$SRC_H5CLEAR_TESTFILES/mod_h5clear_mdc_image.h5 +" + +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + # skip cp if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=`$DIRNAME $tstfile` + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $CP -f $tstfile $TESTDIR + if [ $? -ne 0 ]; then + echo "Error: FAILED to copy $tstfile ." + + # Comment out this to CREATE expected file + exit $EXIT_FAILURE + fi + fi + fi + done +} + +CLEAN_TESTFILES_AND_TESTDIR() +{ + # skip rm if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=$SRC_H5CLEAR_TESTFILES + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $RM $TESTDIR + fi +} + +# +# # Print a line-line message left justified in a field of 70 characters # beginning with the word "Testing". # @@ -40,68 +125,197 @@ TESTING() { echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' } +# $1 is the expected output +# $2 is the output from testing +COMPARE_OUT() { + if $CMP $1 $2; then + echo "PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi +} + +# +# +# +# Run h5clear and verify the expected output +# $1: the test file name +# $2: option 1 passed to the tool +# $3: option 2 passed to the tool +# $4: the expected output +TOOLTEST_OUT() { + fname=$1 + option1=$2 + option2=$3 + expected=$4 + # Prepare expected and actual output + expect="$TESTDIR/$expected" + actual="$TESTDIR/`basename $expected .ddl`.out" + actual_err="$TESTDIR/`basename $expected .ddl`.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + + # Run test. + TESTING $H5CLEAR $option1 $option2 $fname + ( + $RUNSERIAL $H5CLEAR_BIN $option1 $option2 $fname + ) >$actual 2>$actual_err + cp $actual $actual_sav + cp $actual_err $actual_err_sav + cat $actual_err >> $actual + + # Compare output + COMPARE_OUT $expect $actual + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + $RM $actual $actual_err $actual_sav $actual_err_sav + fi +} -# (1)Use "clear_open_chk" to check if the file open succeeds or fails # $1 is the filename to open -# $2 is the expected return from "clear_open_chk" -# (2) Use "h5clear" to clear the status_flags in the test file -# (3) Open the test file via "clear_open_chk" -TOOLTEST() { +# $2 is the expected return from the open/check program +OPEN_CHK() { fname=$1 expected=$2 - # (1) $OPENCHK_BIN $fname 2>/dev/null actual=$? if test $actual -ne $expected; then - echo "Unexpected return from $OPENCHK" - nerrors=`expr $nerrors + 1` + echo "Unexpected return from $OPENCHK" + nerrors=`expr $nerrors + 1` fi - # (2) - TESTING $H5CLEAR $1 +} + +# Run h5clear and verify the expected exit code +# $1 is the filename to open +# $2 is option 1 +# $3 is option 2 +# $4 is the expected exit code from the tool +TOOLTEST() { fname=$1 - # Use "h5clear" to clear the status_flags in the test file - $RUNSERIAL $H5CLEAR_BIN $fname - if test $? -ne $SUCCEED; then - echo ".....$H5CLEAR: should succeed" - nerrors=`expr $nerrors + 1` - else - # (3) Open the test file via "clear_open_chk" - $OPENCHK_BIN $fname - if test $? -ne $SUCCEED; then - echo "......$OPENCHK: should succeed" + option1=$2 + option2=$3 + ret_expected=$4 + # + TESTING $H5CLEAR $option1 $option2 $fname + fname=$1 + # h5clear $option1 $option2 $fname + $RUNSERIAL $H5CLEAR_BIN $option1 $option2 $fname 2>/dev/null 1>/dev/null + if test $? -ne $ret_expected; then + echo "*FAILED*" nerrors=`expr $nerrors + 1` else echo "PASSED" fi - fi } ############################################################################## ############################################################################## -### T H E T E S T S ### +### T H E T E S T S ### ############################################################################## ############################################################################## +# prepare for testing +COPY_TESTFILES_TO_TESTDIR +# +# +# +# The following are tests to verify the expected output from h5clear +# "h5clear -h" +# "h5clear" (no options, no file) +# "h5clear junk.h5" (no options, nonexisting file) +# "h5clear orig_h5clear_sec2_v3.h5" (no options, existing file) +# "h5clear -m" (valid 1 option, no file) +# "h5clear -s junk.h5" (valid 1 option, nonexisting file) +# "h5clear -m -s junk.h5" (valid 2 options, no file) +# "h5clear -m -s junk.h5" (valid 2 options, nonexisting file) +# "h5clear -m orig_h5clear_sec2_v2.h5" (valid 1 option, existing file, no cache image) +# "h5clear -s -m orig_h5clear_sec2_v0.h5" (valid 2 options, existing file, no cache image) +TOOLTEST_OUT "" -h "" h5clear_usage.ddl +TOOLTEST_OUT "" "" "" h5clear_usage.ddl +TOOLTEST_OUT junk.h5 "" "" h5clear_usage.ddl +TOOLTEST_OUT $TESTDIR/orig_h5clear_sec2_v3.h5 "" "" h5clear_usage.ddl +TOOLTEST_OUT "" -m "" h5clear_missing_file.ddl +TOOLTEST_OUT junk.h5 -s "" h5clear_open_fail.ddl +TOOLTEST_OUT "" -m -s h5clear_missing_file.ddl +TOOLTEST_OUT junk.h5 -m -s h5clear_open_fail.ddl +TOOLTEST_OUT $TESTDIR/orig_h5clear_sec2_v2.h5 -m "" h5clear_no_mdc_image.ddl +TOOLTEST_OUT $TESTDIR/orig_h5clear_sec2_v0.h5 -s -m h5clear_no_mdc_image.ddl # -# Initial file open fails -# After "h5clear" the file, the subsequent file open succeeds -TOOLTEST h5clear_sec2_v3.h5 $FAIL # -TOOLTEST h5clear_log_v3.h5 $FAIL # -TOOLTEST latest_h5clear_sec2_v3.h5 $FAIL +# The following are tests to verify the expected exit code from h5clear: +# "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) +# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear -k" (invalid 1 option, no file, fail exit code) +# "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) +# "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) +# "h5clear -m -k" (valid/invalid 2 options, nofile, fail exit code) +# "h5clear -l -m" (invalid/valid 2 options, nofile, fail exit code) +# "h5clear -m -k junk.h5" (valid/invalid 2 options, nonexisting file, fail exit code) +# "h5clear -l -m junk.h5" (invalid/valid 2 options, nonexisting file, fail exit code) +# "h5clear -m -k h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) +# "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) +TOOLTEST h5clear_mdc_image.h5 -m "" $SUCCEED +TOOLTEST "" --vers "" $SUCCEED +TOOLTEST "" -k "" $FAIL +TOOLTEST junk.h5 -k "" $FAIL +TOOLTEST h5clear_sec2_v2.h5 -l "" $FAIL +TOOLTEST "" -m -k $FAIL +TOOLTEST "" -l -m $FAIL +TOOLTEST junk.h5 -m -l $FAIL +TOOLTEST junk.h5 -l -m $FAIL +TOOLTEST h5clear_sec2_v0.h5 -m -l $FAIL +TOOLTEST h5clear_sec2_v0.h5 -l -m $FAIL # -TOOLTEST latest_h5clear_log_v3.h5 $FAIL # # -# File open succeeds because the library does not check status_flags for file with < v3 superblock -TOOLTEST h5clear_sec2_v0.h5 $SUCCEED -TOOLTEST h5clear_sec2_v2.h5 $SUCCEED +# h5clear_mdc_image.h5 already has cache image removed earlier, verify the expected warning from h5clear: +TOOLTEST_OUT $TESTDIR/mod_h5clear_mdc_image.h5 -m "" h5clear_no_mdc_image.ddl +TOOLTEST_OUT $TESTDIR/mod_h5clear_mdc_image.h5 -s -m h5clear_no_mdc_image.ddl +# +# +# +# The following are tests to verify the status_flags field is cleared properly: +OPEN_CHK h5clear_sec2_v3.h5 $FAIL +TOOLTEST h5clear_sec2_v3.h5 -s "" $SUCCEED +OPEN_CHK h5clear_sec2_v3.h5 $SUCCEED +# +OPEN_CHK h5clear_log_v3.h5 $FAIL +TOOLTEST h5clear_log_v3.h5 -s "" $SUCCEED +OPEN_CHK h5clear_log_v3.h5 $SUCCEED +# +OPEN_CHK latest_h5clear_sec2_v3.h5 $FAIL +TOOLTEST latest_h5clear_sec2_v3.h5 -s "" $SUCCEED +OPEN_CHK latest_h5clear_sec2_v3.h5 $SUCCEED +# +OPEN_CHK latest_h5clear_log_v3.h5 $FAIL +TOOLTEST latest_h5clear_log_v3.h5 -s "" $SUCCEED +OPEN_CHK latest_h5clear_log_v3.h5 $SUCCEED +# +# File open succeeds for the following 2 test files because the +# library does not check status_flags for files with < v3 superblock: +OPEN_CHK h5clear_sec2_v0.h5 $SUCCEED +TOOLTEST h5clear_sec2_v0.h5 -s "" $SUCCEED +OPEN_CHK h5clear_sec2_v0.h5 $SUCCEED +# +OPEN_CHK h5clear_sec2_v2.h5 $SUCCEED +TOOLTEST h5clear_sec2_v2.h5 -s "" $SUCCEED +OPEN_CHK h5clear_sec2_v2.h5 $SUCCEED +# +# # # Clean up test files if test -z "$HDF5_NOCLEANUP"; then rm -f h5clear_*.h5 latest_h5clear*.h5 fi +# +# +# +# Clean up temporary files/directories +CLEAN_TESTFILES_AND_TESTDIR if test $nerrors -eq 0 ; then echo "All $TESTNAME tests passed." diff --git a/tools/test/perform/sio_engine.c b/tools/test/perform/sio_engine.c index abf1fa6..2a5efb6 100644 --- a/tools/test/perform/sio_engine.c +++ b/tools/test/perform/sio_engine.c @@ -1064,6 +1064,7 @@ done: do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) { int ret_code = SUCCESS; + hid_t fcpl; switch (param->io_type) { case POSIXIO: @@ -1088,9 +1089,17 @@ do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) GOTOERROR(FAIL); } + fcpl = H5Pcreate(H5P_FILE_CREATE); + if(param->page_size) { + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1); + H5Pset_file_space_page_size(fcpl, param->page_size); + if(param->page_buffer_size) + H5Pset_page_buffer_size(fapl, param->page_buffer_size, 0, 0); + } + /* create the parallel file */ if (flags & (SIO_CREATE | SIO_WRITE)) { - fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); } else { fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, fapl); } diff --git a/tools/test/perform/sio_perf.c b/tools/test/perform/sio_perf.c index 5bae832..2201b99 100644 --- a/tools/test/perform/sio_perf.c +++ b/tools/test/perform/sio_perf.c @@ -100,7 +100,7 @@ static const char *progname = "h5perf_serial"; * It seems that only the options that accept additional information * such as dataset size (-e) require the colon next to it. */ -static const char *s_opts = "a:A:B:c:Cd:D:e:F:ghi:Imno:p:P:r:stT:v:wx:X:"; +static const char *s_opts = "a:A:B:b:c:Cd:D:e:F:G:ghi:Imno:p:P:r:stT:v:wx:X:"; static struct long_options l_opts[] = { { "align", require_arg, 'a' }, { "alig", require_arg, 'a' }, @@ -292,7 +292,8 @@ struct options { int h5_extendable; /* Perform the write tests only */ int verify; /* Verify data correctness */ vfdtype vfd; /* File driver */ - + size_t page_buffer_size; + size_t page_size; }; typedef struct _minmax { @@ -401,6 +402,8 @@ run_test_loop(struct options *opts) parms.h5_write_only = opts->h5_write_only; parms.verify = opts->verify; parms.vfd = opts->vfd; + parms.page_buffer_size = opts->page_buffer_size; + parms.page_size = opts->page_size; /* load multidimensional options */ parms.num_bytes = 1; @@ -865,6 +868,16 @@ report_parameters(struct options *opts) recover_size_and_print((long long)opts->buf_size[i], " "); HDfprintf(output, "\n"); + if(opts->page_size) { + HDfprintf(output, "Page Aggregation Enabled. Page size = %ld\n", opts->page_size); + if(opts->page_buffer_size) + HDfprintf(output, "Page Buffering Enabled. Page Buffer size = %ld\n", opts->page_buffer_size); + else + HDfprintf(output, "Page Buffering Disabled\n"); + } + else + HDfprintf(output, "Page Aggregation Disabled\n"); + HDfprintf(output, "Dimension access order="); for (i=0; i<rank; i++) recover_size_and_print((long long)opts->order[i], " "); @@ -941,6 +954,9 @@ parse_command_line(int argc, char *argv[]) cl_opts = (struct options *)HDmalloc(sizeof(struct options)); + cl_opts->page_buffer_size = 0; + cl_opts->page_size = 0; + cl_opts->output_file = NULL; cl_opts->io_types = 0; /* will set default after parsing options */ cl_opts->num_iters = 1; @@ -975,6 +991,12 @@ parse_command_line(int argc, char *argv[]) case 'a': cl_opts->h5_alignment = parse_size_directive(opt_arg); break; + case 'G': + cl_opts->page_size = parse_size_directive(opt_arg); + break; + case 'b': + cl_opts->page_buffer_size = parse_size_directive(opt_arg); + break; case 'A': { const char *end = opt_arg; diff --git a/tools/test/perform/sio_perf.h b/tools/test/perform/sio_perf.h index 7a7d708..5d9358d 100644 --- a/tools/test/perform/sio_perf.h +++ b/tools/test/perform/sio_perf.h @@ -67,6 +67,8 @@ typedef struct parameters_ { int h5_extendable; /* Make HDF5 dataset chunked */ int h5_write_only; /* Perform the write tests only */ int verify; /* Verify data correctness */ + size_t page_size; + size_t page_buffer_size; } parameters; typedef struct results_ { diff --git a/tools/testfiles/file_space.ddl b/tools/testfiles/file_space.ddl index bed5349..6b8aa47 100644 --- a/tools/testfiles/file_space.ddl +++ b/tools/testfiles/file_space.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_AGGR_VFD - FREE_SPACE_THRESHOLD 10 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_NONE + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 8192 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/file_space.h5 b/tools/testfiles/file_space.h5 Binary files differindex 425d0c2..d9e2890 100644 --- a/tools/testfiles/file_space.h5 +++ b/tools/testfiles/file_space.h5 diff --git a/tools/testfiles/tboot1.ddl b/tools/testfiles/tboot1.ddl index 4758948..73ae4e5 100644 --- a/tools/testfiles/tboot1.ddl +++ b/tools/testfiles/tboot1.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tboot2.ddl b/tools/testfiles/tboot2.ddl index d83b92b..504a634 100644 --- a/tools/testfiles/tboot2.ddl +++ b/tools/testfiles/tboot2.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tboot2A.ddl b/tools/testfiles/tboot2A.ddl index d83b92b..504a634 100644 --- a/tools/testfiles/tboot2A.ddl +++ b/tools/testfiles/tboot2A.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tboot2B.ddl b/tools/testfiles/tboot2B.ddl index d83b92b..504a634 100644 --- a/tools/testfiles/tboot2B.ddl +++ b/tools/testfiles/tboot2B.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tdset_idx.ddl b/tools/testfiles/tdset_idx.ddl deleted file mode 100644 index 65d9f44..0000000 --- a/tools/testfiles/tdset_idx.ddl +++ /dev/null @@ -1,61 +0,0 @@ -############################# -Expected output for 'h5dump -p -H tdset_idx.h5' -############################# -HDF5 "tdset_idx.h5" { -GROUP "/" { - DATASET "dset_btree" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 20, 10 ) / ( 200, 100 ) } - STORAGE_LAYOUT { - CHUNKED ( 5, 5 ) - SIZE 800 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE 0 - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_filter" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } - STORAGE_LAYOUT { - CHUNKED ( 5, 5 ) - SIZE 200 (4.000:1 COMPRESSION) - } - FILTERS { - COMPRESSION DEFLATE { LEVEL 9 } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE 0 - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_fixed" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } - STORAGE_LAYOUT { - CHUNKED ( 5, 5 ) - SIZE 800 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE 0 - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } -} -} |