diff options
| author | Larry Knox <lrknox@hdfgroup.org> | 2018-03-13 18:04:03 (GMT) |
|---|---|---|
| committer | Larry Knox <lrknox@hdfgroup.org> | 2018-03-13 18:04:03 (GMT) |
| commit | f30873136be7cb444166d019c57d3508ff79fc76 (patch) | |
| tree | 559406eb3860a27fe1bfc6fcc7b5febe8454cba7 | |
| parent | 8a77c7a3e4cd49cc0de824476b04b4926582e1ed (diff) | |
| parent | 9247388c3d14510341ea3f07847ce2cbb6e44a8e (diff) | |
| download | hdf5-f30873136be7cb444166d019c57d3508ff79fc76.zip hdf5-f30873136be7cb444166d019c57d3508ff79fc76.tar.gz hdf5-f30873136be7cb444166d019c57d3508ff79fc76.tar.bz2 | |
Merge pull request #943 in HDFFV/hdf5 from develop to hdf5_1_10
* commit '9247388c3d14510341ea3f07847ce2cbb6e44a8e':
Rework new tests into one macro
Added a "won't fix" RELEASE.txt entry for HDFFV-10356.
Fix CMake test names
Fixed typo
Add release note for HDFFFV-10397.
Added a fix for HDFFV-10358.
Fixed documentation Description: - Updated the description of copy constructor for clarification. - Removed unnecessary comments. Platforms tested: Linux/64 (jelly)
Enhancement to the tool h5clear (HDFFV-10360)
Fix for HDFFV-10209 VDS SWMR test failure Free the object header when there are chksum retries.
96 files changed, 1366 insertions, 256 deletions
@@ -1488,15 +1488,38 @@ ./tools/test/misc/testh5mkgrp.sh.in ./tools/test/misc/testh5repart.sh.in ./tools/test/misc/talign.c +./tools/test/misc/testfiles/h5clear_equal_after_size.ddl +./tools/test/misc/testfiles/h5clear_equal_before_size.ddl +./tools/test/misc/testfiles/h5clear_greater_after_size.ddl +./tools/test/misc/testfiles/h5clear_greater_before_size.ddl +./tools/test/misc/testfiles/h5clear_less_after_size.ddl +./tools/test/misc/testfiles/h5clear_less_before_size.ddl ./tools/test/misc/testfiles/h5clear_missing_file.ddl +./tools/test/misc/testfiles/h5clear_noclose_after_size.ddl +./tools/test/misc/testfiles/h5clear_noclose_before_size.ddl ./tools/test/misc/testfiles/h5clear_no_mdc_image.ddl ./tools/test/misc/testfiles/h5clear_open_fail.ddl +./tools/test/misc/testfiles/h5clear_status_noclose_after_size.ddl ./tools/test/misc/testfiles/h5clear_usage.ddl +./tools/test/misc/testfiles/h5clear_user_equal_after_size.ddl +./tools/test/misc/testfiles/h5clear_user_equal_before_size.ddl +./tools/test/misc/testfiles/h5clear_user_greater_after_size.ddl +./tools/test/misc/testfiles/h5clear_user_greater_before_size.ddl +./tools/test/misc/testfiles/h5clear_user_less_after_size.ddl +./tools/test/misc/testfiles/h5clear_user_less_before_size.ddl +./tools/test/misc/testfiles/h5clear_fsm_persist_equal.h5 +./tools/test/misc/testfiles/h5clear_fsm_persist_greater.h5 +./tools/test/misc/testfiles/h5clear_fsm_persist_less.h5 +./tools/test/misc/testfiles/h5clear_fsm_persist_noclose.h5 +./tools/test/misc/testfiles/h5clear_fsm_persist_user_equal.h5 +./tools/test/misc/testfiles/h5clear_fsm_persist_user_greater.h5 +./tools/test/misc/testfiles/h5clear_fsm_persist_user_less.h5 ./tools/test/misc/testfiles/h5clear_log_v3.h5 ./tools/test/misc/testfiles/h5clear_mdc_image.h5 ./tools/test/misc/testfiles/h5clear_sec2_v0.h5 ./tools/test/misc/testfiles/h5clear_sec2_v2.h5 ./tools/test/misc/testfiles/h5clear_sec2_v3.h5 +./tools/test/misc/testfiles/h5clear_status_noclose.h5 ./tools/test/misc/testfiles/latest_h5clear_log_v3.h5 ./tools/test/misc/testfiles/latest_h5clear_sec2_v3.h5 ./tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 diff --git a/c++/src/H5AbstractDs.h b/c++/src/H5AbstractDs.h index 16bef66..399e033 100644 --- a/c++/src/H5AbstractDs.h +++ b/c++/src/H5AbstractDs.h @@ -74,18 +74,6 @@ class H5_DLLCPP AbstractDs { // Default constructor AbstractDs(); - // *** Deprecation warning *** - // The following two constructors are no longer appropriate after the - // data member "id" had been moved to the sub-classes. - // The copy constructor is a noop and is removed in 1.8.15 and the - // other will be removed from 1.10 release, and then from 1.8 if its - // removal does not raise any problems in two 1.10 releases. - - // Mar 2016 -BMR, AbstractDs(const hid_t h5_id); - - // Copy constructor - // AbstractDs( const AbstractDs& original ); - private: // This member function is implemented by DataSet and Attribute - pure virtual. virtual hid_t p_get_type() const = 0; diff --git a/c++/src/H5ArrayType.cpp b/c++/src/H5ArrayType.cpp index 49c31d2..836c837 100644 --- a/c++/src/H5ArrayType.cpp +++ b/c++/src/H5ArrayType.cpp @@ -46,7 +46,7 @@ ArrayType::ArrayType(const hid_t existing_id) : DataType(existing_id) {} //-------------------------------------------------------------------------- // Function: ArrayType copy constructor -///\brief Copy constructor: makes a copy of the original ArrayType object. +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - May 2004 //-------------------------------------------------------------------------- ArrayType::ArrayType(const ArrayType& original) : DataType(original) {} diff --git a/c++/src/H5ArrayType.h b/c++/src/H5ArrayType.h index 5443ae6..d99b936 100644 --- a/c++/src/H5ArrayType.h +++ b/c++/src/H5ArrayType.h @@ -50,7 +50,7 @@ class H5_DLLCPP ArrayType : public DataType { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("ArrayType"); } - // Copy constructor: makes copy of the original object. + // Copy constructor: same as the original ArrayType. ArrayType(const ArrayType& original); // Constructor that takes an existing id diff --git a/c++/src/H5AtomType.cpp b/c++/src/H5AtomType.cpp index 5c24cf9..a9f6f36 100644 --- a/c++/src/H5AtomType.cpp +++ b/c++/src/H5AtomType.cpp @@ -47,7 +47,7 @@ AtomType::AtomType(const hid_t existing_id) : DataType(existing_id) {} //-------------------------------------------------------------------------- // Function: AtomType copy constructor -///\brief Copy constructor: makes a copy of the original AtomType object. +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- AtomType::AtomType(const AtomType& original) : DataType(original) {} diff --git a/c++/src/H5AtomType.h b/c++/src/H5AtomType.h index 7ce66a0..80271ac 100644 --- a/c++/src/H5AtomType.h +++ b/c++/src/H5AtomType.h @@ -60,7 +60,7 @@ class H5_DLLCPP AtomType : public DataType { virtual H5std_string fromClass () const { return("AtomType"); } #ifndef DOXYGEN_SHOULD_SKIP_THIS - // Copy constructor - makes copy of the original object + // Copy constructor: same as the original AtomType. AtomType(const AtomType& original); // Noop destructor diff --git a/c++/src/H5Attribute.cpp b/c++/src/H5Attribute.cpp index d90857f..ccba623 100644 --- a/c++/src/H5Attribute.cpp +++ b/c++/src/H5Attribute.cpp @@ -51,7 +51,7 @@ Attribute::Attribute() : AbstractDs(), H5Location(), id(H5I_INVALID_HID) {} //-------------------------------------------------------------------------- // Function: Attribute copy constructor -///\brief Copy constructor: makes a copy of the original Attribute object. +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: Original Attribute object to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5Attribute.h b/c++/src/H5Attribute.h index 823947e..c6fdaec 100644 --- a/c++/src/H5Attribute.h +++ b/c++/src/H5Attribute.h @@ -29,7 +29,7 @@ namespace H5 { class H5_DLLCPP Attribute : public AbstractDs, public H5Location { public: - // Copy constructor: makes a copy of an existing Attribute object. + // Copy constructor: same as the original Attribute. Attribute(const Attribute& original); // Default constructor diff --git a/c++/src/H5CompType.cpp b/c++/src/H5CompType.cpp index 3731fd4..8252d76 100644 --- a/c++/src/H5CompType.cpp +++ b/c++/src/H5CompType.cpp @@ -40,7 +40,7 @@ CompType::CompType() : DataType() {} //-------------------------------------------------------------------------- // Function: CompType copy constructor -///\brief Copy constructor: makes copy of the original CompType object +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: Original CompType instance // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5CompType.h b/c++/src/H5CompType.h index a811b4a..d85e2ed 100644 --- a/c++/src/H5CompType.h +++ b/c++/src/H5CompType.h @@ -36,7 +36,7 @@ class H5_DLLCPP CompType : public DataType { // Gets the compound datatype of the specified dataset CompType(const DataSet& dataset); // H5Dget_type - // Copy constructor - makes a copy of original object + // Copy constructor - same as the original CompType. CompType(const CompType& original); // Constructors that open a compound datatype, given a location. diff --git a/c++/src/H5DataSet.cpp b/c++/src/H5DataSet.cpp index fb9b57e..fbddd8d 100644 --- a/c++/src/H5DataSet.cpp +++ b/c++/src/H5DataSet.cpp @@ -68,7 +68,7 @@ DataSet::DataSet(const hid_t existing_id) : H5Object(), AbstractDs(), id(existin //-------------------------------------------------------------------------- // Function: DataSet copy constructor -///\brief Copy constructor: makes a copy of the original DataSet object. +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: DataSet instance to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5DataSet.h b/c++/src/H5DataSet.h index f9d2ec3..104ccea 100644 --- a/c++/src/H5DataSet.h +++ b/c++/src/H5DataSet.h @@ -93,7 +93,7 @@ class H5_DLLCPP DataSet : public H5Object, public AbstractDs { // Default constructor. DataSet(); - // Copy constructor. + // Copy constructor - same as the original DataSet. DataSet(const DataSet& original); // Creates a copy of an existing DataSet using its id. diff --git a/c++/src/H5DataSpace.cpp b/c++/src/H5DataSpace.cpp index b563da4..e8e5712 100644 --- a/c++/src/H5DataSpace.cpp +++ b/c++/src/H5DataSpace.cpp @@ -133,7 +133,7 @@ DataSpace::DataSpace(const hid_t existing_id) : IdComponent(), id(existing_id) //-------------------------------------------------------------------------- // Function: DataSpace copy constructor -///\brief Copy constructor: makes a copy of the original DataSpace object. +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: DataSpace object to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5DataSpace.h b/c++/src/H5DataSpace.h index ed141a3..54c68ac 100644 --- a/c++/src/H5DataSpace.h +++ b/c++/src/H5DataSpace.h @@ -36,7 +36,7 @@ class H5_DLLCPP DataSpace : public IdComponent { // Creates a DataSpace object using an existing dataspace id. DataSpace(const hid_t space_id); - // Copy constructor: makes a copy of the original DataSpace object. + // Copy constructor - same as the original DataSpace. DataSpace(const DataSpace& original); // Assignment operator diff --git a/c++/src/H5DataType.cpp b/c++/src/H5DataType.cpp index 032937d..a6b8c24 100644 --- a/c++/src/H5DataType.cpp +++ b/c++/src/H5DataType.cpp @@ -125,7 +125,7 @@ DataType::DataType(const H5Location& loc, const void* ref, H5R_type_t ref_type, //-------------------------------------------------------------------------- // Function: DataType copy constructor -///\brief Copy constructor: makes a copy of the original DataType object +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- DataType::DataType(const DataType& original) : H5Object(), id(original.id), encoded_buf(NULL), buf_size(0) diff --git a/c++/src/H5DataType.h b/c++/src/H5DataType.h index 906ccef..52fd4de 100644 --- a/c++/src/H5DataType.h +++ b/c++/src/H5DataType.h @@ -30,7 +30,7 @@ class H5_DLLCPP DataType : public H5Object { // Creates a datatype given its class and size DataType(const H5T_class_t type_class, size_t size); - // Copy constructor: makes a copy of the original object + // Copy constructor - same as the original DataType. DataType(const DataType& original); // Creates a copy of a predefined type diff --git a/c++/src/H5DcreatProp.cpp b/c++/src/H5DcreatProp.cpp index 6ed77e3..0c2a8c0 100644 --- a/c++/src/H5DcreatProp.cpp +++ b/c++/src/H5DcreatProp.cpp @@ -95,7 +95,7 @@ DSetCreatPropList::DSetCreatPropList() : ObjCreatPropList(H5P_DATASET_CREATE) {} //-------------------------------------------------------------------------- // Function: DSetCreatPropList copy constructor -///\brief Copy constructor: makes a copy of the original +///\brief Copy constructor: same HDF5 object as \a original /// DSetCreatPropList object // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5DcreatProp.h b/c++/src/H5DcreatProp.h index 89318e8..9f54e8d 100644 --- a/c++/src/H5DcreatProp.h +++ b/c++/src/H5DcreatProp.h @@ -122,7 +122,7 @@ class H5_DLLCPP DSetCreatPropList : public ObjCreatPropList { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("DSetCreatPropList"); } - // Copy constructor: creates a copy of a DSetCreatPropList object. + // Copy constructor - same as the original DSetCreatPropList. DSetCreatPropList(const DSetCreatPropList& orig); // Creates a copy of an existing dataset creation property list diff --git a/c++/src/H5DxferProp.cpp b/c++/src/H5DxferProp.cpp index 21e18e7..90ecf88 100644 --- a/c++/src/H5DxferProp.cpp +++ b/c++/src/H5DxferProp.cpp @@ -101,7 +101,7 @@ DSetMemXferPropList::DSetMemXferPropList(const char* exp) : PropList(H5P_DATASET //-------------------------------------------------------------------------- // Function DSetMemXferPropList copy constructor -///\brief Copy constructor: makes a copy of the original +///\brief Copy constructor: same HDF5 object as \a original /// DSetMemXferPropList object ///\param original - IN: Original dataset memory and transfer property /// list object to copy diff --git a/c++/src/H5DxferProp.h b/c++/src/H5DxferProp.h index 75205cf..6955778 100644 --- a/c++/src/H5DxferProp.h +++ b/c++/src/H5DxferProp.h @@ -102,7 +102,7 @@ class H5_DLLCPP DSetMemXferPropList : public PropList { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("DSetMemXferPropList"); } - // Copy constructor: makes a copy of a DSetMemXferPropList object. + // Copy constructor - same as the original DSetMemXferPropList. DSetMemXferPropList(const DSetMemXferPropList& orig); // Creates a copy of an existing dataset memory and transfer diff --git a/c++/src/H5EnumType.cpp b/c++/src/H5EnumType.cpp index 91866d7..00b726c 100644 --- a/c++/src/H5EnumType.cpp +++ b/c++/src/H5EnumType.cpp @@ -53,7 +53,7 @@ EnumType::EnumType(const hid_t existing_id) : DataType( existing_id ) {} //-------------------------------------------------------------------------- // Function: EnumType copy constructor -///\brief Copy constructor: makes a copy of the original EnumType object. +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- EnumType::EnumType(const EnumType& original) : DataType( original ) {} diff --git a/c++/src/H5EnumType.h b/c++/src/H5EnumType.h index e0d5b88..4797ee7 100644 --- a/c++/src/H5EnumType.h +++ b/c++/src/H5EnumType.h @@ -75,7 +75,7 @@ class H5_DLLCPP EnumType : public DataType { // Creates an enumeration datatype using an existing id EnumType(const hid_t existing_id); - // Copy constructor: makes a copy of the original EnumType object. + // Copy constructor: same as the original EnumType. EnumType(const EnumType& original); virtual ~EnumType(); diff --git a/c++/src/H5Exception.cpp b/c++/src/H5Exception.cpp index c52a279..cf9e577 100644 --- a/c++/src/H5Exception.cpp +++ b/c++/src/H5Exception.cpp @@ -39,7 +39,7 @@ Exception::Exception(const H5std_string& func, const H5std_string& message) : de //-------------------------------------------------------------------------- // Function: Exception copy constructor -///\brief Copy constructor: makes a copy of the original Exception object. +///\brief Copy constructor: same HDF5 object as \a original ///\param orig - IN: Exception instance to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5FaccProp.cpp b/c++/src/H5FaccProp.cpp index 9e1098b..286f6e7 100644 --- a/c++/src/H5FaccProp.cpp +++ b/c++/src/H5FaccProp.cpp @@ -88,7 +88,7 @@ FileAccPropList::FileAccPropList() : PropList(H5P_FILE_ACCESS) {} //-------------------------------------------------------------------------- // Function: FileAccPropList copy constructor -///\brief Copy Constructor: makes a copy of the original +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: FileAccPropList instance to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5FaccProp.h b/c++/src/H5FaccProp.h index 1df8fc8..58f049e 100644 --- a/c++/src/H5FaccProp.h +++ b/c++/src/H5FaccProp.h @@ -136,7 +136,7 @@ class H5_DLLCPP FileAccPropList : public PropList { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("FileAccPropList"); } - // Copy constructor: creates a copy of a FileAccPropList object. + // Copy constructor: same as the original FileAccPropList. FileAccPropList(const FileAccPropList& original); // Creates a copy of an existing file access property list diff --git a/c++/src/H5FcreatProp.cpp b/c++/src/H5FcreatProp.cpp index 98a35cb..9b0fecc 100644 --- a/c++/src/H5FcreatProp.cpp +++ b/c++/src/H5FcreatProp.cpp @@ -86,7 +86,7 @@ FileCreatPropList::FileCreatPropList() : PropList(H5P_FILE_CREATE) {} //-------------------------------------------------------------------------- // Function: FileCreatPropList copy constructor -///\brief Copy constructor: makes a copy of the original +///\brief Copy constructor: same HDF5 object as \a original /// FileCreatPropList object. ///\param original - IN: FileCreatPropList instance to copy // Programmer Binh-Minh Ribler - 2000 diff --git a/c++/src/H5FcreatProp.h b/c++/src/H5FcreatProp.h index 31a944c..cc28920 100644 --- a/c++/src/H5FcreatProp.h +++ b/c++/src/H5FcreatProp.h @@ -79,7 +79,7 @@ class H5_DLLCPP FileCreatPropList : public PropList { ///\brief Returns this class name. virtual H5std_string fromClass() const { return("FileCreatPropList"); } - // Copy constructor: creates a copy of a FileCreatPropList object. + // Copy constructor: same as the original FileCreatPropList. FileCreatPropList(const FileCreatPropList& orig); // Creates a copy of an existing file create property list diff --git a/c++/src/H5File.cpp b/c++/src/H5File.cpp index 2c7ac51..f5246f9 100644 --- a/c++/src/H5File.cpp +++ b/c++/src/H5File.cpp @@ -46,7 +46,7 @@ namespace H5 { //-------------------------------------------------------------------------- // Function H5File default constructor ///\brief Default constructor: creates a stub H5File object. -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- H5File::H5File() : Group(), id(H5I_INVALID_HID) {} @@ -78,7 +78,7 @@ H5File::H5File() : Group(), id(H5I_INVALID_HID) {} // Notes With a PGI compiler (~2012-2013,) the exception thrown by // p_get_file could not be caught in the applications. Added try // block here to catch then re-throw it. -BMR 2013/03/21 -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- H5File::H5File(const char* name, unsigned int flags, const FileCreatPropList& create_plist, const FileAccPropList& access_plist) : Group(), id(H5I_INVALID_HID) { @@ -103,7 +103,7 @@ H5File::H5File(const char* name, unsigned int flags, const FileCreatPropList& cr // Notes With a PGI compiler (~2012-2013,) the exception thrown by // p_get_file could not be caught in the applications. Added try // block here to catch then re-throw it. -BMR 2013/03/21 -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- H5File::H5File(const H5std_string& name, unsigned int flags, const FileCreatPropList& create_plist, const FileAccPropList& access_plist) : Group(), id(H5I_INVALID_HID) { @@ -118,7 +118,6 @@ H5File::H5File(const H5std_string& name, unsigned int flags, const FileCreatProp //-------------------------------------------------------------------------- // This function is private and contains common code between the // constructors taking a string or a char* -// Programmer Binh-Minh Ribler - 2000 // Modification // - removed H5F_ACC_CREAT because H5Fcreate will fail with // H5F_ACC_CREAT. - BMR, Sep 17, 2014 @@ -153,7 +152,6 @@ void H5File::p_get_file(const char* name, unsigned int flags, const FileCreatPro // Function: H5File overloaded constructor ///\brief Creates an H5File object using an existing file id. ///\param existing_id - IN: Id of an existing file -// Programmer Binh-Minh Ribler - 2015 // Description // Mar 29, 2015 // Added in responding to a request from user Jason Newton. @@ -172,10 +170,10 @@ H5File::H5File(hid_t existing_id) : Group() //-------------------------------------------------------------------------- // Function: H5File copy constructor -///\brief Copy constructor: makes a copy of the original +///\brief Copy constructor: same HDF5 object as \a original /// H5File object. ///\param original - IN: H5File instance to copy -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- H5File::H5File(const H5File& original) : Group() { @@ -189,7 +187,7 @@ H5File::H5File(const H5File& original) : Group() ///\param name - IN: Name of the file ///\return true if the file is in HDF5 format, and false, otherwise ///\exception H5::FileIException -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- bool H5File::isHdf5(const char* name) { @@ -211,7 +209,7 @@ bool H5File::isHdf5(const char* name) ///\brief This is an overloaded member function, provided for convenience. /// It takes an \c H5std_string for \a name. (Static) ///\param name - IN: Name of the file - \c H5std_string -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- bool H5File::isHdf5(const H5std_string& name) { @@ -234,7 +232,7 @@ bool H5File::isHdf5(const H5std_string& name) /// /// H5F_ACC_RDONLY: Open with read only access. - default /// -// Programmer Binh-Minh Ribler - Oct, 2005 +// October 2005 //-------------------------------------------------------------------------- void H5File::openFile(const char* name, unsigned int flags, const FileAccPropList& access_plist) { @@ -261,7 +259,7 @@ void H5File::openFile(const char* name, unsigned int flags, const FileAccPropLis ///\param flags - IN: File access flags ///\param access_plist - IN: File access property list. Default to /// FileAccPropList::DEFAULT -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- void H5File::openFile(const H5std_string& name, unsigned int flags, const FileAccPropList& access_plist) { @@ -276,7 +274,6 @@ void H5File::openFile(const H5std_string& name, unsigned int flags, const FileAc // Description // If this object has represented another HDF5 file, the previous // HDF5 file need to be closed first. -// Programmer Binh-Minh Ribler - 2000 // Note: This wrapper doesn't seem right regarding the 'id' and should // be investigated. BMR - 2/20/2005 // Modification @@ -303,10 +300,10 @@ void H5File::reOpen() //-------------------------------------------------------------------------- // Function: H5File::getCreatePlist -///\brief Returns the creation property list of this file +///\brief Returns a copy of the creation property list of this file ///\return FileCreatPropList object ///\exception H5::FileIException -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- FileCreatPropList H5File::getCreatePlist() const { @@ -327,10 +324,10 @@ FileCreatPropList H5File::getCreatePlist() const //-------------------------------------------------------------------------- // Function: H5File::getAccessPlist -///\brief Returns the access property list of this file +///\brief Returns a copy of the access property list of this file ///\return FileAccPropList object ///\exception H5::FileIException -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- FileAccPropList H5File::getAccessPlist() const { @@ -357,7 +354,7 @@ FileAccPropList H5File::getAccessPlist() const ///\par Description /// The retrieved information may include information about /// superblock extension, free space management, and shared object -// Programmer Binh-Minh Ribler - February 2017 +// February 2017 //-------------------------------------------------------------------------- void H5File::getFileInfo(H5F_info2_t& file_info) const { @@ -373,7 +370,7 @@ void H5File::getFileInfo(H5F_info2_t& file_info) const ///\brief Returns the amount of free space in the file. ///\return Amount of free space ///\exception H5::FileIException -// Programmer Binh-Minh Ribler - May 2004 +// May 2004 //-------------------------------------------------------------------------- hssize_t H5File::getFreeSpace() const { @@ -405,7 +402,7 @@ hssize_t H5File::getFreeSpace() const /// | \c H5F_OBJ_DATATYPE | \c H5F_OBJ_ATTR ///\par /// Multiple object types can be combined with the logical OR operator (|). -// Programmer Binh-Minh Ribler - May 2004 +// May 2004 //-------------------------------------------------------------------------- ssize_t H5File::getObjCount(unsigned types) const { @@ -440,7 +437,7 @@ ssize_t H5File::getObjCount(unsigned types) const /// Multiple object types can be combined with the logical OR operator (|). // // Notes: will do the overload for this one after hearing from Quincey??? -// Programmer Binh-Minh Ribler - May 2004 +// May 2004 //-------------------------------------------------------------------------- void H5File::getObjIDs(unsigned types, size_t max_objs, hid_t *oid_list) const { @@ -468,7 +465,7 @@ void H5File::getObjIDs(unsigned types, size_t max_objs, hid_t *oid_list) const /// The obtained file handle is dynamic and is valid only while /// the file remains open; it will be invalid if the file is /// closed and reopened or opened during a subsequent session. -// Programmer Binh-Minh Ribler - May 2004 +// May 2004 //-------------------------------------------------------------------------- void H5File::getVFDHandle(const FileAccPropList& fapl, void **file_handle) const { @@ -489,7 +486,7 @@ void H5File::getVFDHandle(const FileAccPropList& fapl, void **file_handle) const // Param file_handle - Pointer to the file handle being used by // the low-level virtual file driver // Exception H5::FileIException -// Programmer Binh-Minh Ribler - May 2004 +// May 2004 // Modification // Planned for removal. -BMR, 2014/04/16 // Removed from documentation. -BMR, 2016/03/07 1.8.17 and 1.10.0 @@ -508,7 +505,7 @@ void H5File::getVFDHandle(const FileAccPropList& fapl, void **file_handle) const ///\param file_handle - Pointer to the file handle being used by /// the low-level virtual file driver ///\exception H5::FileIException -// Programmer Binh-Minh Ribler - May 2004 +// May 2004 //-------------------------------------------------------------------------- void H5File::getVFDHandle(void **file_handle) const { @@ -550,7 +547,7 @@ hsize_t H5File::getFileSize() const // AbstractDS and Attribute are moved out of H5Object. In // addition, member IdComponent::id is moved into subclasses, and // IdComponent::getId now becomes pure virtual function. -// Programmer Binh-Minh Ribler - May, 2008 +// May, 2008 //-------------------------------------------------------------------------- hid_t H5File::getId() const { @@ -564,7 +561,7 @@ hid_t H5File::getId() const // Exception H5::FileIException // Description // This function is replaced by the above function reOpen. -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- void H5File::reopen() { @@ -577,7 +574,7 @@ void H5File::reopen() // Description // This function is a redefinition of CommonFG::getLocId. It // is used by CommonFG member functions to get the file id. -// Programmer Binh-Minh Ribler - 2000 +// December 2000 // Deprecated: // Aug 18, 2016 -BMR // After HDFFV-9920, the Group's methods can use getId() and @@ -598,7 +595,7 @@ hid_t H5File::getLocId() const // The underlaying reference counting in the C library ensures // that the current valid id of this object is properly closed. // Then the object's id is reset to the new id. -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- void H5File::p_setId(const hid_t new_id) { @@ -619,7 +616,7 @@ void H5File::p_setId(const hid_t new_id) ///\brief Closes this HDF5 file. /// ///\exception H5::FileIException -// Programmer Binh-Minh Ribler - Mar 9, 2005 +// March 2005 //-------------------------------------------------------------------------- void H5File::close() { @@ -646,7 +643,7 @@ void H5File::close() // proper exception can be thrown for file or group. The // "H5File::" will be inserted to indicate the function called is // an implementation of H5File. -// Programmer Binh-Minh Ribler - 2000 +// December 2000 //-------------------------------------------------------------------------- void H5File::throwException(const H5std_string& func_name, const H5std_string& msg) const { @@ -658,7 +655,7 @@ void H5File::throwException(const H5std_string& func_name, const H5std_string& m //-------------------------------------------------------------------------- // Function: H5File destructor ///\brief Properly terminates access to this file. -// Programmer Binh-Minh Ribler - 2000 +// December 2000 // Modification // - Replaced resetIdComponent() with decRefCount() to use C // library ID reference counting mechanism - BMR, Feb 20, 2005 diff --git a/c++/src/H5File.h b/c++/src/H5File.h index 214feb0..473428a 100644 --- a/c++/src/H5File.h +++ b/c++/src/H5File.h @@ -41,10 +41,10 @@ class H5_DLLCPP H5File : public Group { // Close this file. virtual void close(); - // Gets the access property list of this file. + // Gets a copy of the access property list of this file. FileAccPropList getAccessPlist() const; - // Gets the creation property list of this file. + // Gets a copy of the creation property list of this file. FileCreatPropList getCreatePlist() const; // Gets general information about this file. @@ -97,7 +97,7 @@ class H5_DLLCPP H5File : public Group { // Default constructor H5File(); - // Copy constructor: makes a copy of the original H5File object. + // Copy constructor: same as the original H5File. H5File(const H5File& original); // Gets the HDF5 file id. diff --git a/c++/src/H5FloatType.cpp b/c++/src/H5FloatType.cpp index 9703149..0a2c107 100644 --- a/c++/src/H5FloatType.cpp +++ b/c++/src/H5FloatType.cpp @@ -66,7 +66,7 @@ FloatType::FloatType(const hid_t existing_id) : AtomType( existing_id ) {} //-------------------------------------------------------------------------- // Function: FloatType copy constructor -///\brief Copy constructor: makes a copy of the original FloatType object. +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- FloatType::FloatType(const FloatType& original) : AtomType( original ){} diff --git a/c++/src/H5FloatType.h b/c++/src/H5FloatType.h index d26df5c..f74a9ae 100644 --- a/c++/src/H5FloatType.h +++ b/c++/src/H5FloatType.h @@ -71,7 +71,7 @@ class H5_DLLCPP FloatType : public AtomType { // Creates a floating-point datatype using an existing id. FloatType(const hid_t existing_id); - // Copy constructor: makes a copy of the original FloatType object. + // Copy constructor: same as the original FloatType. FloatType(const FloatType& original); // Noop destructor. diff --git a/c++/src/H5Group.cpp b/c++/src/H5Group.cpp index c4c7017..25e67d3 100644 --- a/c++/src/H5Group.cpp +++ b/c++/src/H5Group.cpp @@ -53,7 +53,7 @@ Group::Group() : H5Object(), CommonFG(), id(H5I_INVALID_HID) {} //-------------------------------------------------------------------------- // Function: Group copy constructor -///\brief Copy constructor: makes a copy of the original Group object. +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: Original group to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5Group.h b/c++/src/H5Group.h index b3a9007..70b6bff 100644 --- a/c++/src/H5Group.h +++ b/c++/src/H5Group.h @@ -53,7 +53,7 @@ class H5_DLLCPP Group : public H5Object, public CommonFG { // default constructor Group(); - // Copy constructor: makes a copy of the original object + // Copy constructor: same as the original Group. Group(const Group& original); // Gets the group id. diff --git a/c++/src/H5IdComponent.h b/c++/src/H5IdComponent.h index baf939e..b4070b0 100644 --- a/c++/src/H5IdComponent.h +++ b/c++/src/H5IdComponent.h @@ -60,21 +60,8 @@ class H5_DLLCPP IdComponent { // Sets the identifier of this object to a new value. void setId(const hid_t new_id); - // *** Deprecation warning *** - // The following two constructors are no longer appropriate after the - // data member "id" had been moved to the sub-classes. - // The copy constructor is a noop and is removed in 1.8.15 and the - // other will be removed from 1.10 release, and then from 1.8 if its - // removal does not raise any problems in two 1.10 releases. - - // Creates an object to hold an HDF5 identifier. - // IdComponent(const hid_t h5_id); - removed from 1.10.1 - #ifndef DOXYGEN_SHOULD_SKIP_THIS - // Copy constructor: makes copy of the original IdComponent object. - // IdComponent(const IdComponent& original); - removed from 1.8.15 - // Gets the identifier of this object. virtual hid_t getId () const = 0; diff --git a/c++/src/H5IntType.cpp b/c++/src/H5IntType.cpp index 38191bc..dd9d042 100644 --- a/c++/src/H5IntType.cpp +++ b/c++/src/H5IntType.cpp @@ -43,7 +43,7 @@ IntType::IntType() {} //-------------------------------------------------------------------------- // Function: IntType copy constructor -///\brief Copy constructor: makes a copy of the original IntType object. +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- IntType::IntType(const IntType& original) : AtomType( original ) {} diff --git a/c++/src/H5IntType.h b/c++/src/H5IntType.h index a9ebad0..496872b 100644 --- a/c++/src/H5IntType.h +++ b/c++/src/H5IntType.h @@ -53,7 +53,7 @@ class H5_DLLCPP IntType : public AtomType { // Creates a integer datatype using an existing id IntType(const hid_t existing_id); - // Copy constructor: makes copy of IntType object + // Copy constructor: same as the original IntType. IntType(const IntType& original); // Noop destructor. diff --git a/c++/src/H5LaccProp.cpp b/c++/src/H5LaccProp.cpp index 1267286..49ffa2b 100644 --- a/c++/src/H5LaccProp.cpp +++ b/c++/src/H5LaccProp.cpp @@ -88,7 +88,7 @@ LinkAccPropList::LinkAccPropList() : PropList(H5P_LINK_ACCESS) {} //-------------------------------------------------------------------------- // Function: LinkAccPropList copy constructor -///\brief Copy Constructor: makes a copy of the original +///\brief Copy Constructor: same HDF5 object as \a original ///\param original - IN: LinkAccPropList instance to copy // Programmer Binh-Minh Ribler - December, 2016 //-------------------------------------------------------------------------- diff --git a/c++/src/H5LaccProp.h b/c++/src/H5LaccProp.h index 6c86a85..70890b3 100644 --- a/c++/src/H5LaccProp.h +++ b/c++/src/H5LaccProp.h @@ -36,7 +36,7 @@ class H5_DLLCPP LinkAccPropList : public PropList { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("LinkAccPropList"); } - // Copy constructor: creates a copy of a LinkAccPropList object. + // Copy constructor: same as the original LinkAccPropList. LinkAccPropList(const LinkAccPropList& original); // Creates a copy of an existing file access property list diff --git a/c++/src/H5LcreatProp.cpp b/c++/src/H5LcreatProp.cpp index 4f8bffc..8bece44 100644 --- a/c++/src/H5LcreatProp.cpp +++ b/c++/src/H5LcreatProp.cpp @@ -88,7 +88,7 @@ LinkCreatPropList::LinkCreatPropList() : PropList(H5P_LINK_CREATE) {} //-------------------------------------------------------------------------- // Function: LinkCreatPropList copy constructor -///\brief Copy Constructor: makes a copy of the original +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: LinkCreatPropList instance to copy // December, 2016 //-------------------------------------------------------------------------- diff --git a/c++/src/H5LcreatProp.h b/c++/src/H5LcreatProp.h index 4ac2191..12cb479 100644 --- a/c++/src/H5LcreatProp.h +++ b/c++/src/H5LcreatProp.h @@ -36,7 +36,7 @@ class H5_DLLCPP LinkCreatPropList : public PropList { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("LinkCreatPropList"); } - // Copy constructor: creates a copy of a LinkCreatPropList object. + // Copy constructor: same as the original LinkCreatPropList. LinkCreatPropList(const LinkCreatPropList& original); // Creates a copy of an existing file access property list diff --git a/c++/src/H5Object.cpp b/c++/src/H5Object.cpp index 27881c4..5d102d7 100644 --- a/c++/src/H5Object.cpp +++ b/c++/src/H5Object.cpp @@ -58,35 +58,6 @@ extern "C" herr_t userAttrOpWrpr(hid_t loc_id, const char *attr_name, H5Object::H5Object() : H5Location() {} //-------------------------------------------------------------------------- -// Function: H5Object overloaded constructor (protected) -// Purpose Creates an H5Object object using the id of an existing HDF5 -// object. -// Parameters object_id - IN: Id of an existing HDF5 object -// Programmer Binh-Minh Ribler - 2000 -// *** Deprecation warning *** -// This constructor is no longer appropriate because the data member "id" had -// been moved to the sub-classes. It will be removed in 1.10 release. If its -// removal does not raise any problems in 1.10, it will be removed from 1.8 in -// subsequent releases. -// Removed in 1.10.1 - Aug 2016 -//-------------------------------------------------------------------------- -//H5Object::H5Object(const hid_t object_id) : H5Location() {} - -//-------------------------------------------------------------------------- -// Function: H5Object copy constructor -///\brief Copy constructor: makes a copy of the original H5Object -/// instance. -///\param original - IN: H5Object instance to copy -// Programmer Binh-Minh Ribler - 2000 -// *** Deprecation warning *** -// This constructor is no longer appropriate because the data member "id" had -// been moved to the sub-classes. It is removed from 1.8.15 because it is -// a noop and it can be generated by the compiler if needed. -// Removed in 1.10.1 - Aug 2016 -//-------------------------------------------------------------------------- -// H5Object::H5Object(const H5Object& original) : H5Location() {} - -//-------------------------------------------------------------------------- // Function: f_Attribute_setId - friend // Purpose: This function is friend to class H5::Attribute so that it // can set Attribute::id in order to work around a problem diff --git a/c++/src/H5Object.h b/c++/src/H5Object.h index 3772e08..10b3865 100644 --- a/c++/src/H5Object.h +++ b/c++/src/H5Object.h @@ -104,19 +104,6 @@ class H5_DLLCPP H5Object : public H5Location { // Default constructor H5Object(); - // *** Deprecation warning *** - // The following two constructors are no longer appropriate after the - // data member "id" had been moved to the sub-classes. - // The copy constructor is a noop and is removed in 1.8.15 and the - // other will be removed from 1.10 release, and then from 1.8 if its - // removal does not raise any problems in two 1.10 releases. - - // Creates a copy of an existing object giving the object id - // H5Object(const hid_t object_id); - - // Copy constructor: makes copy of an H5Object object. - // H5Object(const H5Object& original); - // Sets the identifier of this object to a new value. - this one // doesn't increment reference count virtual void p_setId(const hid_t new_id) = 0; diff --git a/c++/src/H5OcreatProp.cpp b/c++/src/H5OcreatProp.cpp index 121198c..0542d5e 100644 --- a/c++/src/H5OcreatProp.cpp +++ b/c++/src/H5OcreatProp.cpp @@ -88,7 +88,7 @@ ObjCreatPropList::ObjCreatPropList() : PropList(H5P_OBJECT_CREATE) {} //-------------------------------------------------------------------------- // Function: ObjCreatPropList copy constructor -///\brief Copy Constructor: makes a copy of the original +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: ObjCreatPropList instance to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5OcreatProp.h b/c++/src/H5OcreatProp.h index d1eae86..1d5dd55 100644 --- a/c++/src/H5OcreatProp.h +++ b/c++/src/H5OcreatProp.h @@ -46,7 +46,7 @@ class H5_DLLCPP ObjCreatPropList : public PropList { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("ObjCreatPropList"); } - // Copy constructor: creates a copy of a ObjCreatPropList object. + // Copy constructor: same as the original ObjCreatPropList. ObjCreatPropList(const ObjCreatPropList& original); // Creates a copy of an existing object creation property list diff --git a/c++/src/H5PredType.cpp b/c++/src/H5PredType.cpp index 704a617..3f153e5 100644 --- a/c++/src/H5PredType.cpp +++ b/c++/src/H5PredType.cpp @@ -55,7 +55,7 @@ PredType::PredType() : AtomType() {} //-------------------------------------------------------------------------- // Function: PredType copy constructor -///\brief Copy constructor: makes a copy of the original PredType object. +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: PredType instance to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5PredType.h b/c++/src/H5PredType.h index c631fb9..0978384 100644 --- a/c++/src/H5PredType.h +++ b/c++/src/H5PredType.h @@ -34,7 +34,7 @@ class H5_DLLCPP PredType : public AtomType { // id in the left hand side object. PredType& operator=(const PredType& rhs); - // Copy constructor - makes copy of the original object + // Copy constructor: same as the original PredType. PredType(const PredType& original); // Noop destructor diff --git a/c++/src/H5PropList.cpp b/c++/src/H5PropList.cpp index 16a6316..ef9e16d 100644 --- a/c++/src/H5PropList.cpp +++ b/c++/src/H5PropList.cpp @@ -94,7 +94,7 @@ PropList::PropList() : IdComponent(), id(H5P_DEFAULT) {} //-------------------------------------------------------------------------- // Function: PropList copy constructor -///\brief Copy constructor +///\brief Copy constructor: same HDF5 object as \a original ///\param original - IN: The original property list to copy // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- diff --git a/c++/src/H5PropList.h b/c++/src/H5PropList.h index 6e1e287..e0244c1 100644 --- a/c++/src/H5PropList.h +++ b/c++/src/H5PropList.h @@ -106,7 +106,7 @@ class H5_DLLCPP PropList : public IdComponent { // Default constructor: creates a stub PropList object. PropList(); - // Copy constructor: creates a copy of a PropList object. + // Copy constructor: same as the original PropList. PropList(const PropList& original); // Gets the property list id. diff --git a/c++/src/H5StrType.cpp b/c++/src/H5StrType.cpp index 4c0b3d2..a067d6c 100644 --- a/c++/src/H5StrType.cpp +++ b/c++/src/H5StrType.cpp @@ -120,7 +120,7 @@ StrType::StrType(const hid_t existing_id) : AtomType( existing_id ) {} //-------------------------------------------------------------------------- // Function: StrType copy constructor -///\brief Copy constructor: makes a copy of the original StrType object. +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - 2000 //-------------------------------------------------------------------------- StrType::StrType(const StrType& original) : AtomType ( original ) {} diff --git a/c++/src/H5StrType.h b/c++/src/H5StrType.h index 24c9ca3..38974da 100644 --- a/c++/src/H5StrType.h +++ b/c++/src/H5StrType.h @@ -65,7 +65,7 @@ class H5_DLLCPP StrType : public AtomType { // Creates a string datatype using an existing id StrType(const hid_t existing_id); - // Copy constructor - makes a copy of the original object + // Copy constructor: same as the original StrType. StrType(const StrType& original); // Noop destructor. diff --git a/c++/src/H5VarLenType.cpp b/c++/src/H5VarLenType.cpp index 188ee41..e70d42f 100644 --- a/c++/src/H5VarLenType.cpp +++ b/c++/src/H5VarLenType.cpp @@ -45,7 +45,7 @@ VarLenType::VarLenType(const hid_t existing_id) : DataType(existing_id) {} //-------------------------------------------------------------------------- // Function: VarLenType copy constructor -///\brief Copy constructor: makes a copy of the original VarLenType object. +///\brief Copy constructor: same HDF5 object as \a original // Programmer Binh-Minh Ribler - May, 2004 //-------------------------------------------------------------------------- VarLenType::VarLenType(const VarLenType& original) : DataType(original) {} diff --git a/c++/src/H5VarLenType.h b/c++/src/H5VarLenType.h index 078b534..ab3c14c 100644 --- a/c++/src/H5VarLenType.h +++ b/c++/src/H5VarLenType.h @@ -38,7 +38,7 @@ class H5_DLLCPP VarLenType : public DataType { ///\brief Returns this class name. virtual H5std_string fromClass () const { return("VarLenType"); } - // Copy constructor: makes copy of the original object. + // Copy constructor: same as the original VarLenType. VarLenType(const VarLenType& original); // Constructor that takes an existing id diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 13c4b5a..72afd84 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -533,6 +533,26 @@ Bug Fixes since HDF5-1.10.1 release (DER - 2018/02/26, HDFFV-10355) + - If an HDF5 file contains a malformed compound datatype with a + suitably large offset, the type conversion code can run off + the end of the type conversion buffer, causing a segmentation + fault. + + This issue was reported to The HDF Group as issue #CVE-2017-17507. + + NOTE: The HDF5 C library cannot produce such a file. This condition + should only occur in a corrupt (or deliberately altered) file + or a file created by third-party software. + + THE HDF GROUP WILL NOT FIX THIS BUG AT THIS TIME + + Fixing this problem would involve updating the publicly visible + H5T_conv_t function pointer typedef and versioning the API calls + which use it. We normally only modify the public API during + major releases, so this bug will not be fixed at this time. + + (DER - 2018/02/26, HDFFV-10356) + - If an HDF5 file contains a malformed compound type which contains a member of size zero, a division by zero error will occur while processing the type. @@ -548,6 +568,23 @@ Bug Fixes since HDF5-1.10.1 release (DER - 2018/02/26, HDFFV-10357) + - If an HDF5 file contains a malformed symbol table node that declares + it contains more symbols than it actually contains, the library + can run off the end of the metadata cache buffer while processing + the symbol table node. + + This issue was reported to The HDF Group as issue #CVE-2017-17509. + + NOTE: The HDF5 C library cannot produce such a file. This condition + should only occur in a corrupt (or deliberately altered) file + or a file created by third-party software. + + Performing bounds checks on the buffer while processing fixes the + problem. Instead of the segmentation fault, the normal HDF5 error + handling is invoked. + + (DER - 2018/03/12, HDFFV-10358) + Configuration ------------- - CMake @@ -858,11 +895,20 @@ Bug Fixes since HDF5-1.10.1 release C++ APIs -------- - - + - Removal of memory leaks + + A private function was inadvertently called, causing memory leaks. This + is now fixed. + + (BMR - 2018/03/12 - User's reported in email) Testing ------- - - + - Memory for three variables in testphdf5's coll_write_test was malloced + but not freed, leaking memory when running the test. The variables' + memory is now freed. + + (LRK - 2018/03/12, HDFFV-10397) Supported Platforms =================== @@ -845,8 +845,6 @@ herr_t H5Fget_filesize(hid_t file_id, hsize_t *size) { H5F_t *file; /* File object for file ID */ - haddr_t eof; /* End of file address */ - haddr_t eoa; /* End of allocation address */ haddr_t max_eof_eoa; /* Maximum of the EOA & EOF */ haddr_t base_addr; /* Base address for the file */ herr_t ret_value = SUCCEED; /* Return value */ @@ -859,11 +857,9 @@ H5Fget_filesize(hid_t file_id, hsize_t *size) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") /* Go get the actual file size */ - eof = H5FD_get_eof(file->shared->lf, H5FD_MEM_DEFAULT); - eoa = H5FD_get_eoa(file->shared->lf, H5FD_MEM_DEFAULT); - max_eof_eoa = MAX(eof, eoa); - if(HADDR_UNDEF == max_eof_eoa) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "file get eof/eoa requests failed") + if(H5F__get_max_eof_eoa(file, &max_eof_eoa) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "file can't get max eof/eoa ") + base_addr = H5FD_get_base_addr(file->shared->lf); if(size) @@ -1949,3 +1945,87 @@ done: FUNC_LEAVE_API(ret_value) } /* H5Fget_mdc_image_info() */ + +/*------------------------------------------------------------------------- + * Function: H5Fget_eoa + * + * Purpose: Returns the address of the first byte after the last + * allocated memory in the file. + * (See H5FDget_eoa() in H5FD.c) + * + * Return: Success: First byte after allocated memory. + * Failure: HADDR_UNDEF + * + * Return: Non-negative on success/Negative on errors + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_eoa(hid_t file_id, haddr_t *eoa) +{ + H5F_t *file; /* File object for file ID */ + haddr_t rel_eoa; /* Relative address of EOA */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*a", file_id, eoa); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "hid_t identifier is not a file ID") + + /* This public routine will work only for drivers with this feature enabled.*/ + /* We might introduce a new feature flag in the future */ + if(!H5F_HAS_FEATURE(file, H5FD_FEAT_SUPPORTS_SWMR_IO)) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "must use a SWMR-compatible VFD for this public routine") + + /* The real work */ + if(HADDR_UNDEF == (rel_eoa = H5FD_get_eoa(file->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "get_eoa request failed") + + /* (Note compensating for base address subtraction in internal routine) */ + if(eoa) + *eoa = rel_eoa + H5FD_get_base_addr(file->shared->lf); +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fget_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fincrement_filesize + * + * Purpose: Set the EOA for the file to the maximum of (EOA, EOF) + increment + * + * Return: Non-negative on success/Negative on errors + *------------------------------------------------------------------------- + */ +herr_t +H5Fincrement_filesize(hid_t file_id, hsize_t increment) +{ + H5F_t *file; /* File object for file ID */ + haddr_t max_eof_eoa; /* Maximum of the relative EOA & EOF */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ih", file_id, increment); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "hid_t identifier is not a file ID") + + /* This public routine will work only for drivers with this feature enabled.*/ + /* We might introduce a new feature flag in the future */ + if(!H5F_HAS_FEATURE(file, H5FD_FEAT_SUPPORTS_SWMR_IO)) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "must use a SWMR-compatible VFD for this public routine") + + /* Get the maximum of EOA and EOF */ + if(H5F__get_max_eof_eoa(file, &max_eof_eoa) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "file can't get max eof/eoa ") + + /* Set EOA to the maximum value + increment */ + /* H5FD_set_eoa() will add base_addr to max_eof_eoa */ + if(H5FD_set_eoa(file->shared->lf, H5FD_MEM_DEFAULT, max_eof_eoa + increment) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "driver set_eoa request failed") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fincrement_filesize() */ diff --git a/src/H5Fint.c b/src/H5Fint.c index cc5931a..24844a4 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -124,7 +124,6 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) H5FD_driver_prop_t driver_prop; /* Property for driver ID & info */ hbool_t driver_prop_copied = FALSE; /* Whether the driver property has been set up */ unsigned efc_size = 0; - hbool_t latest_format = FALSE; /* Always use the latest format? */ hid_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -820,7 +819,6 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t else { H5P_genplist_t *plist; /* Property list */ unsigned efc_size; /* External file cache size */ - hbool_t latest_format; /* Always use the latest format? */ size_t u; /* Local index variable */ HDassert(lf != NULL); @@ -1602,7 +1600,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, } /* end if */ else if (1 == shared->nrefs) { /* Read the superblock if it hasn't been read before. */ - if(H5F__super_read(file, meta_dxpl_id, raw_dxpl_id, TRUE) < 0) + if(H5F__super_read(file, meta_dxpl_id, raw_dxpl_id, fapl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") /* Create the page buffer before initializing the superblock */ @@ -2921,6 +2919,43 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__set_paged_aggr() */ +/*------------------------------------------------------------------------- + * Function: H5F__get_max_eof_eoa + * + * Purpose: Determine the maximum of (EOA, EOF) for the file + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +herr_t +H5F__get_max_eof_eoa(const H5F_t *f, haddr_t *max_eof_eoa) +{ + haddr_t eof; /* Relative address for EOF */ + haddr_t eoa; /* Relative address for EOA */ + haddr_t tmp_max; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + + /* Get the relative EOA and EOF */ + eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT); + eof = H5FD_get_eof(f->shared->lf, H5FD_MEM_DEFAULT); + + /* Determine the maximum */ + tmp_max = MAX(eof, eoa); + if(HADDR_UNDEF == tmp_max) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "file get eof/eoa requests failed") + + *max_eof_eoa = tmp_max; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__get_max_eof_eoa() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index c9aba56..a4c1a24 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -412,7 +412,7 @@ H5_DLL herr_t H5F_mount_count_ids(H5F_t *f, unsigned *nopen_files, unsigned *nop /* Superblock related routines */ H5_DLL herr_t H5F__super_init(H5F_t *f, hid_t dxpl_id); -H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, +H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hid_t fapl_id, hbool_t initial_read); H5_DLL herr_t H5F__super_size(H5F_t *f, hid_t dxpl_id, hsize_t *super_size, hsize_t *super_ext_size); H5_DLL herr_t H5F__super_free(H5F_super_t *sblock); @@ -457,6 +457,7 @@ H5_DLL htri_t H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5_DLL herr_t H5F__set_eoa(const H5F_t *f, H5F_mem_t type, haddr_t addr); H5_DLL herr_t H5F__set_base_addr(const H5F_t *f, haddr_t addr); H5_DLL herr_t H5F__set_paged_aggr(const H5F_t *f, hbool_t paged); +H5_DLL herr_t H5F__get_max_eof_eoa(const H5F_t *f, haddr_t *max_eof_eoa); /* Functions that flush or evict */ H5_DLL herr_t H5F__evict_cache_entries(H5F_t *f, hid_t dxpl_id); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index dadbbac..1886c5c 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -492,6 +492,10 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */ #define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */ #define H5F_ACS_CLEAR_STATUS_FLAGS_NAME "clear_status_flags" /* Whether to clear superblock status_flags (private property only used by h5clear) */ +#define H5F_ACS_NULL_FSM_ADDR_NAME "null_fsm_addr" /* Nullify addresses of free-space managers */ + /* Private property used only by h5clear */ +#define H5F_ACS_SKIP_EOF_CHECK_NAME "skip_eof_check" /* Skip EOF check */ + /* Private property used only by h5clear */ #define H5F_ACS_USE_MDC_LOGGING_NAME "use_mdc_logging" /* Whether to use metadata cache logging */ #define H5F_ACS_MDC_LOG_LOCATION_NAME "mdc_log_location" /* Name of metadata cache log location */ #define H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME "start_mdc_log_on_access" /* Whether logging starts on file create/open */ diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index d333fa7..73c59f5 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -241,6 +241,8 @@ H5_DLL herr_t H5Fmount(hid_t loc, const char *name, hid_t child, hid_t plist); H5_DLL herr_t H5Funmount(hid_t loc, const char *name); H5_DLL hssize_t H5Fget_freespace(hid_t file_id); H5_DLL herr_t H5Fget_filesize(hid_t file_id, hsize_t *size); +H5_DLL herr_t H5Fget_eoa(hid_t file_id, haddr_t *eoa); +H5_DLL herr_t H5Fincrement_filesize(hid_t file_id, hsize_t increment); H5_DLL ssize_t H5Fget_file_image(hid_t file_id, void * buf_ptr, size_t buf_len); H5_DLL herr_t H5Fget_mdc_config(hid_t file_id, H5AC_cache_config_t * config_ptr); diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 4250ff0..3db9b2f 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -324,7 +324,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial_read) +H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hid_t fapl_id, hbool_t initial_read) { H5P_genplist_t *dxpl = NULL; /* DXPL object */ H5AC_ring_t ring, orig_ring = H5AC_RING_INV; @@ -337,7 +337,8 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial haddr_t eof; /* End of file address */ unsigned rw_flags; /* Read/write permissions for file */ hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */ - herr_t ret_value = SUCCEED; /* Return value */ + H5P_genplist_t *a_plist; /* File access property list */ + herr_t ret_value = SUCCEED; /* Return value */ #ifdef H5_HAVE_PARALLEL int mpi_rank = 0, mpi_size = 1; int mpi_result; @@ -595,6 +596,20 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial * as the file can appear truncated if only part of it has been * been flushed to disk by the SWMR writer process. */ + /* The EOF check is also skipped when the private property + * H5F_ACS_SKIP_EOF_CHECK_NAME exists in the fapl. + * This property is enabled by the tool h5clear with these + * two options: (1) --filesize (2) --increment + */ + + /* Check if this private property exists in fapl */ + if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not file access property list") + if(H5P_exist_plist(a_plist, H5F_ACS_SKIP_EOF_CHECK_NAME) > 0) { + if(H5P_get(a_plist, H5F_ACS_SKIP_EOF_CHECK_NAME, &skip_eof_check) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get clearance for persisting fsm addr") + } + if(H5F_INTENT(f) & H5F_ACC_SWMR_READ) { /* * When the file is opened for SWMR read access, skip the check if: @@ -757,6 +772,7 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial if(status) { H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */ uint8_t flags; /* Message flags */ + hbool_t null_fsm_addr = FALSE; /* Whether to drop free-space to the floor */ /* Get message flags */ if(H5O_msg_get_flags(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id, &flags) < 0) @@ -765,6 +781,13 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial /* If message is NOT marked "unknown"--set up file space info */ if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) { + /* The tool h5clear uses this property to tell the library + to drop free-space to the floor */ + if(H5P_exist_plist(a_plist, H5F_ACS_NULL_FSM_ADDR_NAME) > 0) { + if(H5P_get(a_plist, H5F_ACS_NULL_FSM_ADDR_NAME, &null_fsm_addr) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get clearance for persisting fsm addr") + } + /* Retrieve the 'file space info' structure */ if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") @@ -808,13 +831,30 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc) f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc; - /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF - * in the absence of persistant free space managers. + /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF + * in the absence of persistant free space managers. + */ + /* If the following two conditions are true: + * (1) skipping EOF check (skip_eof_check) + * (2) dropping free-space to the floor (null_fsm_addr) + * skip the asserts as "eoa_pre_fsm_fsalloc" may be undefined + * for a crashed file with persistant free space managers. + * #1 and #2 are enabled when the tool h5clear --increment + * option is used. */ - HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF)); - HDassert(!f->shared->first_alloc_dealloc); + if(!skip_eof_check && !null_fsm_addr) { + HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF)); + HDassert(!f->shared->first_alloc_dealloc); + } - if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF) && + /* As "eoa_pre_fsm_fsalloc" may be undefined for a crashed file + * with persistant free space managers, therefore, set + * "first_alloc_dealloc" when the condition + * "dropping free-space to the floor is true. + * This will ensure that no action is done to settle things on file + * close via H5MF_settle_meta_data_fsm() and H5MF_settle_raw_data_fsm(). + */ + if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF || null_fsm_addr) && (H5F_INTENT(f) & H5F_ACC_RDWR)) f->shared->first_alloc_dealloc = TRUE; @@ -822,7 +862,20 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial for(u = 1; u < NELMTS(f->shared->fs_addr); u++) f->shared->fs_addr[u] = fsinfo.fs_addr[u - 1]; - if(fsinfo.mapped && (rw_flags & H5AC__READ_ONLY_FLAG) == 0) { + /* If the following two conditions are true: + * (1) file is persisting free-space + * (2) dropping free-space to the floor (null_fsm_addr) + * nullify the addresses of the FSMs + */ + if(f->shared->fs_persist && null_fsm_addr) { + for(u = 0; u < NELMTS(fsinfo.fs_addr); u++) + f->shared->fs_addr[u] = fsinfo.fs_addr[u] = HADDR_UNDEF; + } + + /* For fsinfo.mapped: remove the FSINFO message from the superblock extension + and write a new message to the extension */ + /* For null_fsm_addr: just update FSINFO message in the superblock extension */ + if(((fsinfo.mapped || null_fsm_addr) && (rw_flags & H5AC__READ_ONLY_FLAG) == 0)) { /* Do the same kluge until we know for sure. VC */ #if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ @@ -836,11 +889,16 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial f->shared->sblock = sblock; #endif /* JRM */ - if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_FSINFO_ID) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension") + if(null_fsm_addr) { + if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") + } else { + if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_FSINFO_ID) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension") - if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) - HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") + if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") + } #if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ f->shared->sblock = NULL; #endif /* JRM */ @@ -1637,13 +1695,13 @@ H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, void *mesg, /* Open/create the superblock extension object header */ if(H5F_addr_defined(f->shared->sblock->ext_addr)) { - if(H5F_super_ext_open(f, f->shared->sblock->ext_addr, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") + if(H5F_super_ext_open(f, f->shared->sblock->ext_addr, &ext_loc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") } /* end if */ else { HDassert(may_create); - if(H5F_super_ext_create(f, dxpl_id, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTCREATE, FAIL, "unable to create file's superblock extension") + if(H5F_super_ext_create(f, dxpl_id, &ext_loc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTCREATE, FAIL, "unable to create file's superblock extension") ext_created = TRUE; } /* end else */ HDassert(H5F_addr_defined(ext_loc.addr)); @@ -1651,24 +1709,24 @@ H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, unsigned id, void *mesg, /* Check if message with ID does not exist in the object header */ if((status = H5O_msg_exists(&ext_loc, id, dxpl_id)) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to check object header for message or message exists") + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to check object header for message or message exists") /* Check for creating vs. writing */ if(may_create) { - if(status) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should not exist") + if(status) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should not exist") - /* Create the message with ID in the superblock extension */ - if(H5O_msg_create(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg, dxpl_id) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to create the message in object header") + /* Create the message with ID in the superblock extension */ + if(H5O_msg_create(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg, dxpl_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to create the message in object header") } /* end if */ else { - if(!status) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should exist") + if(!status) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should exist") - /* Update the message with ID in the superblock extension */ - if(H5O_msg_write(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg, dxpl_id) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to write the message in object header") + /* Update the message with ID in the superblock extension */ + if(H5O_msg_write(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg, dxpl_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to write the message in object header") } /* end else */ done: diff --git a/src/H5Gcache.c b/src/H5Gcache.c index 65115a5..b447cad 100644 --- a/src/H5Gcache.c +++ b/src/H5Gcache.c @@ -170,6 +170,7 @@ H5G__cache_node_deserialize(const void *_image, size_t len, void *_udata, H5F_t *f = (H5F_t *)_udata; /* User data for callback */ H5G_node_t *sym = NULL; /* Symbol table node created */ const uint8_t *image = (const uint8_t *)_image; /* Pointer to image to deserialize */ + const uint8_t *image_end = image + len - 1; /* Pointer to end of image buffer */ void *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -203,7 +204,7 @@ H5G__cache_node_deserialize(const void *_image, size_t len, void *_udata, UINT16DECODE(image, sym->nsyms); /* entries */ - if(H5G__ent_decode_vec(f, &image, sym->entry, sym->nsyms) < 0) + if(H5G__ent_decode_vec(f, &image, image_end, sym->entry, sym->nsyms) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTLOAD, NULL, "unable to decode symbol table entries") /* Set return value */ diff --git a/src/H5Gent.c b/src/H5Gent.c index 7987850..6e076ae 100644 --- a/src/H5Gent.c +++ b/src/H5Gent.c @@ -91,7 +91,7 @@ H5FL_BLK_EXTERN(str_buf); *------------------------------------------------------------------------- */ herr_t -H5G__ent_decode_vec(const H5F_t *f, const uint8_t **pp, H5G_entry_t *ent, unsigned n) +H5G__ent_decode_vec(const H5F_t *f, const uint8_t **pp, const uint8_t *p_end, H5G_entry_t *ent, unsigned n) { unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -104,9 +104,12 @@ H5G__ent_decode_vec(const H5F_t *f, const uint8_t **pp, H5G_entry_t *ent, unsign HDassert(ent); /* decode entries */ - for(u = 0; u < n; u++) + for(u = 0; u < n; u++) { + if(*pp > p_end) + HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, FAIL, "ran off the end of the image buffer") if(H5G_ent_decode(f, pp, ent + u) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTDECODE, FAIL, "can't decode") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Gpkg.h b/src/H5Gpkg.h index 76bf08b..20e595f 100644 --- a/src/H5Gpkg.h +++ b/src/H5Gpkg.h @@ -395,7 +395,7 @@ H5_DLL void H5G__ent_copy(H5G_entry_t *dst, const H5G_entry_t *src, H5_copy_depth_t depth); H5_DLL void H5G__ent_reset(H5G_entry_t *ent); H5_DLL herr_t H5G__ent_decode_vec(const H5F_t *f, const uint8_t **pp, - H5G_entry_t *ent, unsigned n); + const uint8_t *p_end, H5G_entry_t *ent, unsigned n); H5_DLL herr_t H5G__ent_encode_vec(const H5F_t *f, uint8_t **pp, const H5G_entry_t *ent, unsigned n); H5_DLL herr_t H5G__ent_convert(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 94049ef..2260e12 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -262,12 +262,23 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) uint32_t stored_chksum; /* Stored metadata checksum value */ uint32_t computed_chksum; /* Computed metadata checksum value */ - /* Get stored and computed checksums */ - H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); - - if(stored_chksum != computed_chksum) - ret_value = FALSE; + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) { + /* These fields are not deserialized yet in H5O__prefix_deserialize() */ + HDassert(udata->oh->chunk == NULL); + HDassert(udata->oh->mesg == NULL); + HDassert(udata->oh->proxy == NULL); + + /* Indicate that udata->oh is to be freed later + in H5O__prefix_deserialize() */ + udata->free_oh = TRUE; + ret_value = FALSE; + } /* end if */ } /* end if */ + else + HDassert(!(udata->common.file_intent & H5F_ACC_SWMR_WRITE)); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_verify_chksum() */ @@ -1263,8 +1274,22 @@ H5O__prefix_deserialize(const uint8_t *_image, H5O_cache_ud_t *udata) /* Verify object header prefix length */ HDassert((size_t)(image - _image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); - /* Save the object header for later use in 'deserialize' callback */ - udata->oh = oh; + /* If udata->oh is to be freed (see H5O__cache_verify_chksum), + save the pointer to udata->oh and free it later after setting + udata->oh with the new object header */ + if(udata->free_oh) { + H5O_t *saved_oh = udata->oh; + HDassert(udata->oh); + + /* Save the object header for later use in 'deserialize' callback */ + udata->oh = oh; + if(H5O__free(saved_oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "can't destroy object header") + udata->free_oh = FALSE; + } else + /* Save the object header for later use in 'deserialize' callback */ + udata->oh = oh; + oh = NULL; done: diff --git a/src/H5Oint.c b/src/H5Oint.c index 08eb28d..2351779 100644 --- a/src/H5Oint.c +++ b/src/H5Oint.c @@ -875,6 +875,7 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags, udata.v1_pfx_nmesgs = 0; udata.chunk0_size = 0; udata.oh = NULL; + udata.free_oh = FALSE; udata.common.f = loc->file; udata.common.dxpl_id = dxpl_id; udata.common.file_intent = file_intent; diff --git a/src/H5Opkg.h b/src/H5Opkg.h index e970406..9392fa8 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -379,6 +379,7 @@ typedef struct H5O_cache_ud_t { unsigned v1_pfx_nmesgs; /* Number of messages from v1 prefix header */ size_t chunk0_size; /* Size of serialized first chunk */ H5O_t *oh; /* Partially deserialized object header, for later use */ + hbool_t free_oh; /* Whether to free the object header or not */ H5O_common_cache_ud_t common; /* Common object header cache callback info */ } H5O_cache_ud_t; diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 13409d2..a7c8218 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -193,6 +193,14 @@ /* Definition for status_flags in the superblock */ #define H5F_ACS_CLEAR_STATUS_FLAGS_SIZE sizeof(hbool_t) #define H5F_ACS_CLEAR_STATUS_FLAGS_DEF FALSE + +/* Definition for dropping free-space to the floor when reading in the superblock */ +#define H5F_ACS_NULL_FSM_ADDR_SIZE sizeof(hbool_t) +#define H5F_ACS_NULL_FSM_ADDR_DEF FALSE +/* Definition for skipping EOF check when reading in the superblock */ +#define H5F_ACS_SKIP_EOF_CHECK_SIZE sizeof(hbool_t) +#define H5F_ACS_SKIP_EOF_CHECK_DEF FALSE + /* Definition for 'use metadata cache logging' flag */ #define H5F_ACS_USE_MDC_LOGGING_SIZE sizeof(hbool_t) #define H5F_ACS_USE_MDC_LOGGING_DEF FALSE @@ -374,6 +382,9 @@ static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE static const unsigned H5F_def_metadata_read_attempts_g = H5F_ACS_METADATA_READ_ATTEMPTS_DEF; /* Default setting for the # of metadata read attempts */ static const H5F_object_flush_t H5F_def_object_flush_cb_g = H5F_ACS_OBJECT_FLUSH_CB_DEF; /* Default setting for object flush callback */ static const hbool_t H5F_def_clear_status_flags_g = H5F_ACS_CLEAR_STATUS_FLAGS_DEF; /* Default to clear the superblock status_flags */ +static const hbool_t H5F_def_skip_eof_check_g = H5F_ACS_SKIP_EOF_CHECK_DEF; /* Default setting for skipping EOF check */ +static const hbool_t H5F_def_null_fsm_addr_g = H5F_ACS_NULL_FSM_ADDR_DEF; /* Default setting for dropping free-space to the floor */ + static const hbool_t H5F_def_use_mdc_logging_g = H5F_ACS_USE_MDC_LOGGING_DEF; /* Default metadata cache logging flag */ static const char *H5F_def_mdc_log_location_g = H5F_ACS_MDC_LOG_LOCATION_DEF; /* Default mdc log location */ static const hbool_t H5F_def_start_mdc_log_on_access_g = H5F_ACS_START_MDC_LOG_ON_ACCESS_DEF; /* Default mdc log start on access flag */ @@ -565,6 +576,16 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the private property of whether to skip EOF check. It's used by h5clear only. */ + if(H5P_register_real(pclass, H5F_ACS_SKIP_EOF_CHECK_NAME, H5F_ACS_SKIP_EOF_CHECK_SIZE, &H5F_def_skip_eof_check_g, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the private property of whether to drop free-space to the floor. It's used by h5clear only. */ + if(H5P_register_real(pclass, H5F_ACS_NULL_FSM_ADDR_NAME, H5F_ACS_NULL_FSM_ADDR_SIZE, &H5F_def_null_fsm_addr_g, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the metadata cache logging flag. */ if(H5P_register_real(pclass, H5F_ACS_USE_MDC_LOGGING_NAME, H5F_ACS_USE_MDC_LOGGING_SIZE, &H5F_def_use_mdc_logging_g, NULL, NULL, NULL, H5F_ACS_USE_MDC_LOGGING_ENC, H5F_ACS_USE_MDC_LOGGING_DEC, NULL, NULL, NULL, NULL) < 0) diff --git a/test/tfile.c b/test/tfile.c index de1f362..200f221 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -134,6 +134,9 @@ #define NGROUPS 2 #define NDSETS 4 +/* Declaration for test_incr_filesize() */ +#define FILE8 "tfile8.h5" /* Test file */ + /* Files created under 1.6 branch and 1.8 branch--used in test_filespace_compatible() */ const char *OLD_FILENAME[] = { "filespace_1_6.h5", /* 1.6 HDF5 file */ @@ -6926,6 +6929,130 @@ test_libver_macros2(void) /**************************************************************** ** +** test_filesize(): +** Verify H5Fincrement_filesize() and H5Fget_eoa() works as +** indicated in the "RFC: Enhancement to the tool h5clear". +** +****************************************************************/ +static void +test_incr_filesize(const char *env_h5_drvr) +{ + hid_t fid; /* File opened with read-write permission */ + h5_stat_size_t filesize; /* Size of file when empty */ + hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File access property list */ + hid_t dspace; /* Dataspace ID */ + hid_t dset; /* Dataset ID */ + hid_t dcpl; /* Dataset creation property list */ + unsigned u; /* Local index variable */ + char filename[FILENAME_LEN]; /* Filename to use */ + char name[32]; /* Dataset name */ + haddr_t stored_eoa; /* The stored EOA value */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("Testing H5Fincrement_filesize() and H5Fget_eoa())\n")); + + fapl = h5_fileaccess(); + h5_fixname(FILE8, fapl, filename, sizeof filename); + + /* Get the VFD feature flags */ + driver_id = H5Pget_driver(fapl); + CHECK(driver_id, FAIL, "H5Pget_driver"); + + ret = H5FDdriver_query(driver_id, &driver_flags); + CHECK(ret, FAIL, "H5PDdriver_query"); + + /* Check whether the VFD feature flag supports these two public routines */ + if(driver_flags & H5FD_FEAT_SUPPORTS_SWMR_IO) { + + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + + /* Set file space strategy */ + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5P_set_file_space_strategy"); + + /* Create the test file */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + CHECK(fid, FAIL, "H5Fcreate"); + + /* Create dataspace for datasets */ + dspace = H5Screate(H5S_SCALAR); + CHECK(dspace, FAIL, "H5Screate"); + + /* Create a dataset creation property list */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + CHECK(dcpl, FAIL, "H5Pcreate"); + + /* Set the space allocation time to early */ + ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); + CHECK(ret, FAIL, "H5Pset_alloc_time"); + + /* Create datasets in file */ + for(u = 0; u < 10; u++) { + sprintf(name, "Dataset %u", u); + dset = H5Dcreate2(fid, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset, FAIL, "H5Dcreate2"); + + ret = H5Dclose(dset); + CHECK(ret, FAIL, "H5Dclose"); + } /* end for */ + + /* Close dataspace */ + ret = H5Sclose(dspace); + CHECK(ret, FAIL, "H5Sclose"); + + /* Close dataset creation property list */ + ret = H5Pclose(dcpl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Close file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Get the file size */ + filesize = h5_get_file_size(filename, fapl); + + /* Open the file */ + fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + CHECK(fid, FAIL, "H5Fopen"); + + /* Get the stored EOA */ + ret = H5Fget_eoa(fid, &stored_eoa); + CHECK(ret, FAIL, "H5Fget_eoa"); + + /* Verify the stored EOA is the same as filesize */ + VERIFY(filesize, stored_eoa, "file size"); + + /* Set the EOA to the MAX(EOA, EOF) + 512 */ + ret = H5Fincrement_filesize(fid, 512); + CHECK(ret, FAIL, "H5Fincrement_filesize"); + + /* Close file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Get the file size */ + filesize = h5_get_file_size(filename, fapl); + + /* Verify the filesize is the previous stored_eoa + 512 */ + VERIFY(filesize, stored_eoa+512, "file size"); + + /* Close the file access property list */ + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Close the file creation property list */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + } +} /* end test_incr_filesize() */ + +/**************************************************************** +** ** test_deprec(): ** Test deprecated functionality. ** @@ -7210,6 +7337,7 @@ test_file(void) test_libver_bounds_low_high(); test_libver_macros(); /* Test the macros for library version comparison */ test_libver_macros2(); /* Test the macros for library version comparison */ + test_incr_filesize(env_h5_drvr); /* Test H5Fincrement_filesize() and H5Fget_eoa() */ #ifndef H5_NO_DEPRECATED_SYMBOLS test_deprec(); /* Test deprecated routines */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c index e3b989d..5724e1b 100644 --- a/tools/src/misc/h5clear.c +++ b/tools/src/misc/h5clear.c @@ -12,9 +12,11 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Purpose: A tool to clear the status_flags field from the file's superblock via -s option. - * A tool to remove cache image from the file via -m option. - * + * Purpose: A tool used to do the following: + * (1) -s, --status: clear the status_flags field from the file's superblock + * (2) -m, --image: remove the metadata cache image from the file + * (3) --increment=C: set the file's EOA to the maximum of (EOA, EOF) + C + * (4) --filesize: print the file's EOA and EOF */ #include "hdf5.h" #include "H5private.h" @@ -24,18 +26,25 @@ /* Name of tool */ #define PROGRAMNAME "h5clear" -/* Make this private property (defined in H5Fprivate.h) available to h5clear. */ -#define H5F_ACS_CLEAR_STATUS_FLAGS_NAME "clear_status_flags" +/* Make these private properties (defined in H5Fprivate.h) available to h5clear. */ +#define H5F_ACS_CLEAR_STATUS_FLAGS_NAME "clear_status_flags" +#define H5F_ACS_NULL_FSM_ADDR_NAME "null_fsm_addr" +#define H5F_ACS_SKIP_EOF_CHECK_NAME "skip_eof_check" + +/* Default increment is 1 megabytes for the --increment option */ +#define DEFAULT_INCREMENT 1024*1024 static char *fname_g = NULL; static hbool_t clear_status_flags = FALSE; static hbool_t remove_cache_image = FALSE; +static hbool_t print_filesize = FALSE; +static hbool_t increment_eoa_eof = FALSE; +static hsize_t increment = DEFAULT_INCREMENT; /* - * Command-line options: The user can specify short or long-named - * parameters. + * Command-line options: only publicize long options */ -static const char *s_opts = "hVsm"; +static const char *s_opts = "hVsmzi*"; static struct long_options l_opts[] = { { "help", no_arg, 'h' }, { "hel", no_arg, 'h'}, @@ -54,6 +63,21 @@ static struct long_options l_opts[] = { { "imag", no_arg, 'm' }, { "ima", no_arg, 'm' }, { "im", no_arg, 'm' }, + { "filesize", no_arg, 'z' }, + { "filesiz", no_arg, 'z' }, + { "filesi", no_arg, 'z' }, + { "files", no_arg, 'z' }, + { "file", no_arg, 'z' }, + { "fil", no_arg, 'z' }, + { "fi", no_arg, 'z' }, + { "increment", optional_arg, 'i' }, + { "incremen", optional_arg, 'i' }, + { "increme", optional_arg, 'i' }, + { "increm", optional_arg, 'i' }, + { "incre", optional_arg, 'i' }, + { "incr", optional_arg, 'i' }, + { "inc", optional_arg, 'i' }, + { "in", optional_arg, 'i' }, { NULL, 0, '\0' } }; @@ -76,6 +100,9 @@ static void usage(const char *prog) HDfprintf(stdout, " -V, --version Print version number and exit\n"); HDfprintf(stdout, " -s, --status Clear the status_flags field in the file's superblock\n"); HDfprintf(stdout, " -m, --image Remove the metadata cache image from the file\n"); + HDfprintf(stdout, " --filesize Print the file's EOA and EOF\n"); + HDfprintf(stdout, " --increment=C Set the file's EOA to the maximum of (EOA, EOF) + C for the file <file_name>\n"); + HDfprintf(stdout, " C is >= 0; C is optional and will default to 1M when not set"); HDfprintf(stdout, "\n"); HDfprintf(stdout, "Examples of use:\n"); HDfprintf(stdout, "\n"); @@ -84,6 +111,12 @@ static void usage(const char *prog) HDfprintf(stdout, "\n"); HDfprintf(stdout, "h5clear -m file_name\n"); HDfprintf(stdout, " Remove the metadata cache image from the HDF5 file <file_name>.\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "h5clear --increment file_name\n"); + HDfprintf(stdout, " Set the EOA to the maximum of (EOA, EOF) + 1M for the file <file_name>.\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "h5clear --increment=512 file_name\n"); + HDfprintf(stdout, " Set the EOA to the maximum of (EOA, EOF) + 512 for the file <file_name>.\n"); } /* usage() */ @@ -131,6 +164,18 @@ parse_command_line(int argc, const char **argv) remove_cache_image = TRUE; break; + case 'z': + print_filesize = TRUE; + break; + + case 'i': + increment_eoa_eof = TRUE; + if(opt_arg != NULL && (increment = HDatoi(opt_arg)) < 0) { + usage(h5tools_getprogname()); + goto done; + } + break; + default: usage(h5tools_getprogname()); h5tools_setstatus(EXIT_FAILURE); @@ -176,8 +221,24 @@ leave(int ret) /*------------------------------------------------------------------------- * Function: main * - * Purpose: To clear the status_flags field in the file's superblock (-s option). - * To remove the cache image from the file (-m option). + * Purpose: The options are: + * (1) -s, --status: clear the status_flags field from the file's superblock + * (2) -m, --image: remove the metadata cache image from the file + * (3) --increment=C: set the file's EOA to the maximum of (EOA, EOF) + C + * (4) --filesize: print the file's EOA and EOF + * + * The three options: -s, -m, and --increment will modify the file + * so the file is opened with write access. + * The --filesize option just prints the EOA and EOF, so the file + * is opened with read access. + * + * The -s option will activate the private property: + * --H5F_ACS_CLEAR_STATUS_FLAGS_NAME + * The --increment option will active these two private properties: + * --H5F_ACS_NULL_FSM_ADDR_NAME + * --H5F_ACS_SKIP_EOF_CHECK_NAME + * The --filesize will activate the private property: + * --H5F_ACS_SKIP_EOF_CHECK_NAME * * Return: Success: 0 * Failure: 1 @@ -187,11 +248,12 @@ leave(int ret) int main (int argc, const char *argv[]) { - char *fname = NULL; /* File name */ - hid_t fapl = -1; /* File access property list */ - hid_t fid = -1; /* File ID */ + char *fname = NULL; /* File name */ + hid_t fapl = -1; /* File access property list */ + hid_t fid = -1; /* File ID */ haddr_t image_addr; hsize_t image_len; + unsigned flags = H5F_ACC_RDWR; /* file access flags */ h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); @@ -209,12 +271,22 @@ main (int argc, const char *argv[]) if(fname_g == NULL) goto done; - if(!clear_status_flags && !remove_cache_image) { + /* Print usage/exit if not using at least one of the options */ + if(!clear_status_flags && !remove_cache_image && + !increment_eoa_eof && !print_filesize) { usage(h5tools_getprogname()); h5tools_setstatus(EXIT_FAILURE); goto done; } + /* Cannot combine the --filesize option with other options */ + if(print_filesize && + (clear_status_flags || remove_cache_image || increment_eoa_eof)) { + error_msg("Cannot combine --filesize with other options\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + /* Duplicate the file name */ fname = HDstrdup(fname_g); @@ -228,7 +300,7 @@ main (int argc, const char *argv[]) /* -s option */ if(clear_status_flags) { /* Set to clear the status_flags in the file's superblock */ - /* This is a private property used by h5clear only */ + /* Activate this private property */ if(H5Pset(fapl, H5F_ACS_CLEAR_STATUS_FLAGS_NAME, &clear_status_flags) < 0) { error_msg("H5Pset\n"); h5tools_setstatus(EXIT_FAILURE); @@ -236,12 +308,64 @@ main (int argc, const char *argv[]) } } - if((fid = h5tools_fopen(fname, H5F_ACC_RDWR, fapl, NULL, NULL, (size_t)0)) < 0) { + /* --increment option */ + if(increment_eoa_eof) { + /* Activate this private property */ + if(H5Pset(fapl, H5F_ACS_SKIP_EOF_CHECK_NAME, &increment_eoa_eof) < 0) { + error_msg("H5Pset\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + /* Activate this private property */ + if(H5Pset(fapl, H5F_ACS_NULL_FSM_ADDR_NAME, &increment_eoa_eof) < 0) { + error_msg("H5Pset\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + + /* --filesize option; open the file read-only */ + if(print_filesize) { + /* Activate this private property */ + if(H5Pset(fapl, H5F_ACS_SKIP_EOF_CHECK_NAME, &print_filesize) < 0) { + error_msg("H5Pset\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + flags = H5F_ACC_RDONLY; + } + + /* Open the file */ + if((fid = h5tools_fopen(fname, flags, fapl, NULL, NULL, (size_t)0)) < 0) { error_msg("h5tools_fopen\n"); h5tools_setstatus(EXIT_FAILURE); goto done; } + /* --filesize option */ + if(print_filesize) { + h5_stat_t st; /* Stat info call */ + haddr_t eoa; /* The EOA value */ + + /* Get the file's EOA and EOF */ + if(H5Fget_eoa(fid, &eoa) < 0 || HDstat(fname, &st) < 0) { + error_msg("H5Fget_eoa or HDstat\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + HDfprintf(stdout, "EOA is %a; EOF is %a \n", eoa, st.st_size); + } + + /* --increment option */ + if(increment_eoa_eof) { + /* Set the file's EOA to the maximum of (EOA, EOF) + increment */ + if(H5Fincrement_filesize(fid, increment) < 0) { + error_msg("H5Fset_eoa\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + /* -m option */ if(remove_cache_image) { if(H5Fget_mdc_image_info(fid, &image_addr, &image_len) < 0) { @@ -253,7 +377,9 @@ main (int argc, const char *argv[]) warn_msg("No cache image in the file\n"); } + h5tools_setstatus(EXIT_SUCCESS); + done: if(fname) HDfree(fname); diff --git a/tools/test/misc/CMakeTestsClear.cmake b/tools/test/misc/CMakeTestsClear.cmake index 942ae7a..a51d0e9 100644 --- a/tools/test/misc/CMakeTestsClear.cmake +++ b/tools/test/misc/CMakeTestsClear.cmake @@ -20,11 +20,19 @@ # Copy all the HDF5 files from the source directory into the test directory # -------------------------------------------------------------------- set (HDF5_TEST_FILES + h5clear_fsm_persist_equal.h5 + h5clear_fsm_persist_greater.h5 + h5clear_fsm_persist_less.h5 + h5clear_fsm_persist_noclose.h5 + h5clear_fsm_persist_user_equal.h5 + h5clear_fsm_persist_user_greater.h5 + h5clear_fsm_persist_user_less.h5 h5clear_log_v3.h5 h5clear_mdc_image.h5 - mod_h5clear_mdc_image.h5 + h5clear_status_noclose.h5 latest_h5clear_log_v3.h5 latest_h5clear_sec2_v3.h5 + mod_h5clear_mdc_image.h5 ) set (HDF5_SEC2_TEST_FILES h5clear_sec2_v0.h5 @@ -32,10 +40,25 @@ h5clear_sec2_v3.h5 ) set (HDF5_REFERENCE_TEST_FILES - h5clear_usage.ddl - h5clear_open_fail.ddl + h5clear_equal_after_size.ddl + h5clear_equal_before_size.ddl + h5clear_greater_after_size.ddl + h5clear_greater_before_size.ddl + h5clear_less_after_size.ddl + h5clear_less_before_size.ddl h5clear_missing_file.ddl + h5clear_noclose_after_size.ddl + h5clear_noclose_before_size.ddl h5clear_no_mdc_image.ddl + h5clear_open_fail.ddl + h5clear_status_noclose_after_size.ddl + h5clear_usage.ddl + h5clear_user_equal_after_size.ddl + h5clear_user_equal_before_size.ddl + h5clear_user_greater_after_size.ddl + h5clear_user_greater_before_size.ddl + h5clear_user_less_after_size.ddl + h5clear_user_less_before_size.ddl ) foreach (h5_file ${HDF5_TEST_FILES} ${HDF5_SEC2_TEST_FILES} ${HDF5_REFERENCE_TEST_FILES}) @@ -85,6 +108,42 @@ endif () endmacro () + macro (ADD_H5_CMP_WITH_COPY testname resultcode resultfile testfile) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5CLEAR_CMP-${testname}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + testfiles/${testname}.out + testfiles/${testname}.out.err + testfiles/${testfile} + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR_CMP-${testname}-clear-objects PROPERTIES DEPENDS ${last_test}) + endif () + add_test ( + NAME H5CLEAR_CMP-copy_${testname} + COMMAND ${CMAKE_COMMAND} + -E copy_if_different + "${PROJECT_SOURCE_DIR}/testfiles/${testfile}" "${PROJECT_BINARY_DIR}/testfiles/${testfile}" + ) + set_tests_properties (H5CLEAR_CMP-copy_${testname} PROPERTIES DEPENDS H5CLEAR_CMP-${testname}-clear-objects) + add_test ( + NAME H5CLEAR_CMP-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5clear>" + -D "TEST_ARGS:STRING=${ARGN};${testfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5CLEAR_CMP-${testname} PROPERTIES DEPENDS H5CLEAR_CMP-copy_${testname}) + set (last_test "H5CLEAR_CMP-${testname}") + endif () + endmacro () + macro (ADD_H5_RETTEST testname resultcode) if (NOT HDF5_ENABLE_USING_MEMCHECKER) add_test ( @@ -100,6 +159,70 @@ endif () endmacro () + macro (ADD_H5_FILESIZE_TEST testname resultcode resultfile incr_size) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5CLEAR_FILESIZE_TEST-${testname}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + testfiles/${testname}_before_size.out + testfiles/${testname}_before_size.out.err + testfiles/${testname}_after_size.out + testfiles/${testname}_after_size.out.err + testfiles/${testname}.h5 + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR_FILESIZE_TEST-${testname}-clear-objects PROPERTIES DEPENDS ${last_test}) + endif () + add_test ( + NAME H5CLEAR_FILESIZE_TEST-copy_${testname} + COMMAND ${CMAKE_COMMAND} + -E copy_if_different + "${PROJECT_SOURCE_DIR}/testfiles/${testname}.h5" "${PROJECT_BINARY_DIR}/testfiles/${testname}.h5" + ) + set_tests_properties (H5CLEAR_FILESIZE_TEST-copy_${testname} PROPERTIES DEPENDS H5CLEAR_FILESIZE_TEST-${testname}-clear-objects) + add_test ( + NAME H5CLEAR_FILESIZE_CMP-${testname}_before_size + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5clear>" + -D "TEST_ARGS:STRING=--filesize;${testname}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${testname}_before_size.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}_before_size.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5CLEAR_FILESIZE_CMP-${testname}_before_size PROPERTIES DEPENDS H5CLEAR_FILESIZE_TEST-copy_${testname}) + if (NOT ${incr_size} MATCHES "NONE") + add_test ( + NAME H5CLEAR_FILESIZE_INCR-${testname} + COMMAND $<TARGET_FILE:h5clear> --increment=${incr_size} ${testname}.h5 + ) + else () + add_test ( + NAME H5CLEAR_FILESIZE_INCR-${testname} + COMMAND $<TARGET_FILE:h5clear> --increment ${testname}.h5 + ) + endif () + set_tests_properties (H5CLEAR_FILESIZE_INCR-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set_tests_properties (H5CLEAR_FILESIZE_INCR-${testname} PROPERTIES WILL_FAIL "${resultcode}") + set_tests_properties (H5CLEAR_FILESIZE_INCR-${testname} PROPERTIES DEPENDS H5CLEAR_FILESIZE_CMP-${testname}_before_size) + add_test ( + NAME H5CLEAR_FILESIZE_CMP-${testname}_after_size + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5clear>" + -D "TEST_ARGS:STRING=--filesize;${testname}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${testname}_after_size.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}_after_size.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5CLEAR_FILESIZE_CMP-${testname}_after_size PROPERTIES DEPENDS H5CLEAR_FILESIZE_INCR-${testname}) + set (last_test "H5CLEAR_FILESIZE_CMP-${testname}_after_size") + endif () + endmacro () + macro (ADD_H5_TEST testname testfile resultcode) if (NOT HDF5_ENABLE_USING_MEMCHECKER) add_test ( @@ -160,6 +283,7 @@ if (HDF5_ENABLE_USING_MEMCHECKER) latest_h5clear_sec2_v3.h5 mod_h5clear_mdc_image.h5 mod_h5clear_mdc_image2.h5 + ${HDF5_TEST_FILES} ) if (NOT "${last_test}" STREQUAL "") set_tests_properties (H5CLEAR-clearall-objects PROPERTIES DEPENDS ${last_test}) @@ -270,3 +394,60 @@ endif() ADD_H5_TEST (latest_h5clr_log_v3 latest_h5clear_log_v3 "true") ADD_H5_TEST (h5clr_sec2_v0 h5clear_sec2_v0 "false") ADD_H5_TEST (h5clr_sec2_v2 h5clear_sec2_v2 "false") +# +# +# +# The following tests verify the filesize, increment the filesize, then verify the filesize again. +# +# (1) h5clear_status_noclose.h5 +# "h5clear --filesize h5clear_status_noclose.h5" (unable to open the file because status_flags is enabled) +# "h5clear -s --increment=0 h5clear_status_noclose.h5" (clear status_flag, EOA = MAX(EOA, EOF) + 0) +# (no output, check exit code) +# "h5clear --filesize h5clear_status_noclose.h5" (print EOA/EOF after the last action) + ADD_H5_CMP_WITH_COPY (h5clr_open_fail_nc_s 1 h5clear_open_fail h5clear_status_noclose.h5 "--filesize") + ADD_H5_RETTEST (h5clr_mdc_image_nc "false" "-s" "--increment=0" h5clear_status_noclose.h5) + ADD_H5_CMP (h5clr_no_mdc_image_nc_m h5clear_status_noclose_after_size 0 "--filesize" h5clear_status_noclose.h5) +# +# (2) h5clear_fsm_persist_noclose.h5 +# "h5clear --filesize h5clear_fsm_persist_noclose.h5" (print EOA/EOF before the next action) +# "h5clear --increment=0 h5clear_fsm_persist_noclose.h5" (EOA = MAX(EOA, EOF)) (no output, just check exit code) +# "h5clear --filesize h5clear_fsm_persist_noclose.h5" (print EOA/EOF after the last action) + ADD_H5_FILESIZE_TEST (h5clear_fsm_persist_noclose 0 h5clear_noclose 0) +# +# (3) h5clear_fsm_persist_equal.h5 +# "h5clear --filesize h5clear_fsm_persist_equal.h5" (print EOA/EOF before the next action) +# "h5clear --increment h5clear_fsm_persist_equal.h5" (EOA = MAX(EOA, EOF) + 1M) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_equal.h5" (print EOA/EOF after the last action) + ADD_H5_FILESIZE_TEST (h5clear_fsm_persist_equal 0 h5clear_equal NONE) +# +# (4) h5clear_fsm_persist_greater.h5 +# "h5clear --filesize h5clear_fsm_persist_greater.h5" (print EOA/EOF before the next action) +# "h5clear --increment=0 h5clear_fsm_persist_greater.h5" (EOA = MAX(EOA, EOF) + 0) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_greater.h5" (print EOA/EOF after the last action) + ADD_H5_FILESIZE_TEST (h5clear_fsm_persist_greater 0 h5clear_greater 0) +# +# (5) h5clear_fsm_persist_less.h5 +# "h5clear --filesize h5clear_fsm_persist_less.h5" (print EOA/EOF before the next action) +# "h5clear --increment=200 h5clear_fsm_persist_less.h5" (EOA = MAX(EOA, EOF) + 200) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_less.h5" (print EOA/EOF after the last action) + ADD_H5_FILESIZE_TEST (h5clear_fsm_persist_less 0 h5clear_less 200) +# +# (6) h5clear_fsm_persist_user_equal.h5 +# "h5clear --filesize h5clear_fsm_persist_user_equal.h5" (print EOA/EOF before the next action) +# "h5clear --increment h5clear_fsm_persist_user_equal.h5" (EOA = MAX(EOA, EOF) + 1M) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_user_equal.h5" (print EOA/EOF after the last action) + ADD_H5_FILESIZE_TEST (h5clear_fsm_persist_user_equal 0 h5clear_user_equal NONE) +# +# (7) h5clear_fsm_persist_user_greater.h5 +# "h5clear --filesize h5clear_fsm_persist_user_greater.h5" (print EOA/EOF before the next action) +# "h5clear --increment=0 h5clear_fsm_persist_user_greater.h5" (EOA = MAX(EOA, EOF) + 0) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_user_greater.h5" (print EOA/EOF after the last action) + ADD_H5_FILESIZE_TEST (h5clear_fsm_persist_user_greater 0 h5clear_user_greater 0) +# +# (8) h5clear_fsm_persist_user_less.h5 +# "h5clear --filesize h5clear_fsm_persist_user_less.h5" (print EOA/EOF before the next action) +# "h5clear --increment=200 h5clear_fsm_persist_user_less.h5" (EOA = MAX(EOA, EOF) + 200) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_user_less.h5" (print EOA/EOF after the last action) + ADD_H5_FILESIZE_TEST (h5clear_fsm_persist_user_less 0 h5clear_user_less 200) +# +# diff --git a/tools/test/misc/h5clear_gentest.c b/tools/test/misc/h5clear_gentest.c index 326109c..ccb510f 100644 --- a/tools/test/misc/h5clear_gentest.c +++ b/tools/test/misc/h5clear_gentest.c @@ -21,10 +21,25 @@ const char *FILENAME[] = { "h5clear_sec2_v2.h5" /* 3 -- sec2 file with superblock version 2 */ }; +const char *FILENAME_ENHANCE[] = { + "h5clear_fsm_persist_equal.h5", /* 0: persisting free-space, stored EOA = actual EOF */ + "h5clear_fsm_persist_greater.h5", /* 1: persisting free-space, stored EOA > actual EOF */ + "h5clear_fsm_persist_less.h5", /* 2: persisting free-space, stored EOA < actual EOF */ + "h5clear_fsm_persist_user_equal.h5", /* 3: user block, persisting free-space, stored EOA = actual EOF */ + "h5clear_fsm_persist_user_greater.h5", /* 4: user block, persisting free-space, stored EOA > actual EOF */ + "h5clear_fsm_persist_user_less.h5", /* 5: user block, persisting free-space, stored EOA < actual EOF */ + "h5clear_status_noclose.h5", /* 6 -- v3 superblock, nonzero status_flags, no flush, exit, + stored EOA < actual EOF */ + "h5clear_fsm_persist_noclose.h5" /* 7 -- persisting free-space, no flush, exit, stored EOA < actual EOF */ +}; + #define KB 1024U #define CACHE_IMAGE_FILE "h5clear_mdc_image.h5" #define DSET "DSET" +#define DATASET "dset" +#define NUM_ELMTS 100 +#define USERBLOCK 512 /*------------------------------------------------------------------------- * Function: gen_cache_image_file @@ -118,26 +133,224 @@ error: H5Pclose(dcpl); } H5E_END_TRY; return 1; -} +} /* gen_cache_image_file() */ + +/*------------------------------------------------------------------------- + * Function: gen_enhance_files + * + * Purpose: To create the first 6 files in FILENAME_ENHANCE[]: + * (0) FILENAME_ENHANCE[0]: "h5clear_fsm_persist_equal.h5" + * (1) FILENAME_ENHANCE[1]: "h5clear_fsm_persist_greater.h5" + * (2) FILENAME_ENHANCE[2]: "h5clear_fsm_persist_less.h5" + * (3) FILENAME_ENHANCE[3]: "h5clear_user_fsm_persist_equal.h5" + * (4) FILENAME_ENHANCE[4]: "h5clear_user_fsm_persist_greater.h5" + * (5) FILENAME_ENHANCE[5]: "h5clear_user_fsm_persist_less.h5" + * After creating the files for #1, #2, #4 #5, write invalid EOA + * value to the location where the EOA is stored in the superblock. + * Also modify the chksum in the superblock due to this change. + * + * The first call to this routine (without user block) will generate + * the first 3 files. + * The second call to this routine (with user block) will generate + * the last 3 files. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; March 2017 + * + *------------------------------------------------------------------------- + */ +static int +gen_enhance_files(hbool_t user) +{ + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t fapl = -1; /* File access property list */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1; /* Dataset ID */ + hsize_t dim[1]; /* Dimension sizes */ + int data[NUM_ELMTS]; /* Buffer for data */ + int fd = -1; /* The file descriptor ID */ + int64_t eoa; /* The EOA value */ + int32_t chksum; /* The chksum value */ + int i = 0 , j = 0, u = 0; /* Local index variable */ + + /* Get a copy of the default file creation property */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + + /* Check to see if user block will be added */ + if(user) { + if(H5Pset_userblock(fcpl, (hsize_t)USERBLOCK) < 0) + goto error; + u = 3; + } + + /* Set file space strategy and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + goto error; + + /* + * Create the file, then write invalid EOA to the file. + */ + for(i = 0+u; i < 3+u; i++) { + + /* Create the file with the file space strategy and persisting free-space */ + if((fid = H5Fcreate(FILENAME_ENHANCE[i], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Create the dataset */ + dim[0] = NUM_ELMTS; + if((sid = H5Screate_simple(1, dim, NULL)) < 0) + goto error; + if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + for(j = 0; j < NUM_ELMTS; j++) + data[j] = j; + + /* Write the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + goto error; + + /* Closing */ + if(H5Dclose(did) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Fclose(fid) < 0) + goto error; + + /* + * No further action for: + * --FILENAME_ENHANCE[0]: "h5clear_fsm_persist_equal.h5" + * --FILENAME_ENHANCE[3]: "h5clear_fsm_persist_user_equal.h5", + */ + if(!(i % 3)) + continue; + /* + * For the following files: + * --FILENAME_ENHANCE[1]: "h5clear_fsm_persist_greater.h5" + * --FILENAME_ENHANCE[2]: "h5clear_fsm_persist_less.h5" + * --FILENAME_ENHANCE[4]: "h5clear_fsm_persist_greater.h5" + * --FILENAME_ENHANCE[5]: "h5clear_fsm_persist_less.h5" + * + * Write invalid value to the location for stored eoa and + * update the chksum value. + */ + /* Open the file */ + if((fd = open(FILENAME_ENHANCE[i], O_RDWR, 0663)) < 0) + goto error; + + switch(i) { + case 1: /* stored EOA is > EOF */ + eoa = 3048; + chksum = 268376587; + break; + + case 2: /* stored EOA is < EOF */ + eoa = 512; + chksum = 372920305; + break; + + case 4: /* with userblock, stored EOA > EOF */ + eoa = 4000; + chksum = 4168810027; + break; + + case 5: /* with userblock, stored EOA < EOF */ + eoa = 3000; + chksum = 3716054346; + break; + + default: + break; + } + + /* location of "end of file address" */ + if(lseek(fd, (off_t)(28+(user?USERBLOCK:0)), SEEK_SET) < 0) + goto error; + + /* Write the bad eoa value to the file */ + if(write(fd, &eoa, sizeof(eoa)) < 0) + goto error; + + /* location of "superblock checksum" */ + if(lseek(fd, (off_t)(44+(user?USERBLOCK:0)), SEEK_SET) < 0) + goto error; + + /* Write the chksum value to the file */ + if(write(fd, &chksum, sizeof(chksum)) < 0) + goto error; + + /* Close the file */ + if(close(fd) < 0) + goto error; + + } /* end for */ + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + goto error; + + return 0; + +error: + H5E_BEGIN_TRY { + H5Sclose(sid); + H5Dclose(did); + H5Fclose(fid); + H5Pclose(fcpl); + } H5E_END_TRY; + return 1; +} /* gen_enhance_files() */ /*------------------------------------------------------------------------- * Function: main * - * Purpose: To create HDF5 files with non-zero status_flags in the superblock - * via flushing and exiting without closing the library. + * Purpose: Generate test files used by h5clear. * - * Due to file locking, status_flags in the superblock will be - * nonzero after H5Fcreate. The library will clear status_flags - * on file closing. This program, after "H5Fcreate" the files, - * exits without going through library closing. Thus, status_flags - * for these files are not cleared. - * The library will check consistency of status_flags when opening - * a file with superblock >= v3 and will return error accordingly. - * The library will not check status_flags when opening a file - * with < v3 superblock. + * (A) gen_cache_image_file(): + * --generate a file with cache image feature + * --"h5clear_mdc_image.h5" + * (B) gen_enhance_files(): + * --generate the first 6 files in FILENAME_ENHANCE[]: + * (0) "h5clear_fsm_persist_equal.h5" + * (1) "h5clear_fsm_persist_greater.h5" + * (2) "h5clear_fsm_persist_less.h5" + * (3) "h5clear_fsm_persist_user_equal.h5" + * (4) "h5clear_fsm_persist_user_greater.h5" + * (5) "h5clear_fsm_persist_user_less.h5" * - * These files are used by "h5clear" to see if the tool clears - * status_flags properly so users can open the files afterwards. + * (C) Generate the following FILENAME[] files in main(): + * (0a) "h5clear_sec2_v3.h5" + * (0b) "latest_h5clear_sec2_v3.h5" + * (1a) "h5clear_log_v3.h5", + * (1b) "latest_h5clear_log_v3.h5" + * (2) "h5clear_sec2_v0.h5" + * (3) "h5clear_sec2_v2.h5" + * + * These HDF5 files are created with non-zero status_flags in + * the superblock via flushing and exiting without closing the + * library. + * Due to file locking, status_flags in the superblock will be + * nonzero after H5Fcreate. The library will clear status_flags + * on file closing. + * This program, after "H5Fcreate" the files, exits without + * going through library closing. Thus, status_flags for these + * files are not cleared. + * The library will check consistency of status_flags when + * opening a file with superblock >= v3 and will return error + * accordingly. + * The library will not check status_flags when opening a file + * with < v3 superblock. + * These files are used by "h5clear" to see if the tool clears + * status_flags properly so users can open the files afterwards. + * + * (D) Generate the last two files in FILENAME_ENHANCE[] in main(): + * (6) "h5clear_status_noclose.h5", + * (7) "h5clear_fsm_persist_noclose.h5" * * Return: Success: 0 * Failure: 1 @@ -149,20 +362,30 @@ error: int main(void) { - hid_t fid; /* File ID */ - hid_t fcpl; /* File creation property list */ - hid_t fapl, new_fapl; /* File access property lists */ + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t fapl = -1, new_fapl = -1; /* File access property lists */ char fname[512]; /* File name */ - unsigned new_format; /* To use latest library format or not */ + unsigned new_format; /* To use latest library format or not */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1; /* Dataset ID */ + hsize_t dim[1]; /* Dimension sizes */ + int data[NUM_ELMTS]; /* Buffer for data */ + int i; /* Local index variables */ /* Generate a file with cache image feature enabled */ if(gen_cache_image_file(CACHE_IMAGE_FILE) < 0) goto error; + /* Generate the first 6 files in FILENAME_ENHANCE[] */ + if(gen_enhance_files(FALSE) < 0) + goto error; + if(gen_enhance_files(TRUE) < 0) + goto error; + /* - * Generate files with invalid status_flags + * Generate files in FILENAME[] */ - /* Create a copy of the file access property list */ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; @@ -174,7 +397,11 @@ main(void) if(H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) goto error; - /* Files created within this for loop will have v3 superblock and nonzero status_flags */ + /* + * Files created within this for loop will have v3 superblock and nonzero status_flags + * --FILENAME[0]: "h5clear_sec2_v3.h5", "latest_h5clear_sec2_v3.h5" + * --FILENAME[1]: "h5clear_log_v3.h5", "latest_h5clear_log_v3.h5" + */ for(new_format = FALSE; new_format <= TRUE; new_format++) { hid_t fapl2, my_fapl; /* File access property lists */ @@ -228,7 +455,8 @@ main(void) } /* end for */ /* - * Create a sec2 file with v0 superblock but nonzero status_flags + * Create a sec2 file with v0 superblock but nonzero status_flags: + * FILENAME[2]: "h5clear_sec2_v0.h5" */ if((fid = H5Fcreate(FILENAME[2], H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) goto error; @@ -239,7 +467,8 @@ main(void) /* - * Create a sec2 file with v2 superblock but nonzero status_flags + * Create a sec2 file with v2 superblock but nonzero status_flags: + * FILENAME[3]: "h5clear_sec2_v2.h5" */ if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) goto error; @@ -264,6 +493,100 @@ main(void) if(H5Pclose(fcpl) < 0) goto error; + /* + * Create the last two files in FILENAME_ENHANCE[]: + * --FILENAME_ENHANCE[6]: h5clear_status_noclose.h5 + * --FILENAME_ENHANCE[7]: h5clear_fsm_persist_noclose.h5 + */ + /* + * FILENAME_ENHANCE[6]: h5clear_status_noclose.h5 + * --stored EOA < actual EOF + * --version 3 superblock + * --nonzero status_flags + * --does not persist free-space + * --does not flush the file, just exit without closing file: + * --this file is similar to the user-suppplied test file attached with HDFFV-10347 + */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + + /* Set to latest format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + goto error; + + /* Create file with SWMR-write access */ + if((fid = H5Fcreate(FILENAME_ENHANCE[6], H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE, H5P_DEFAULT, fapl)) < 0) + goto error; + + /* Create the dataset */ + dim[0] = NUM_ELMTS; + if((sid = H5Screate_simple(1, dim, NULL)) < 0) + goto error; + if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + for(i = 0; i < NUM_ELMTS; i++) + data[i] = i; + + /* Write the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + goto error; + + /* Closing */ + if(H5Dclose(did) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Pclose(fapl) < 0) + goto error; + + /* Does not flush and does not close the file */ + + + /* + * FILENAME_ENHANCE[7]: h5clear_fsm_persist_noclose.h5 + * --stored EOA < actual EOF + * --persisting free-space + * --undefined fsinfo.eoa_pre_fsm_fsalloc + * --undefined fsinfo.fs_addr + * --does not flush the file, just exit without closing + */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + + /* Set file space strategy and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + goto error; + + /* Create the file with the set file space info */ + if((fid = H5Fcreate(FILENAME_ENHANCE[7], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Create the dataset */ + dim[0] = NUM_ELMTS; + if((sid = H5Screate_simple(1, dim, NULL)) < 0) + goto error; + if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + for(i = 0; i < NUM_ELMTS; i++) + data[i] = i; + + /* Write the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + goto error; + + /* Closing */ + if(H5Dclose(did) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Pclose(fcpl) < 0) + goto error; + + /* Does not flush and does not close the file */ + + fflush(stdout); fflush(stderr); diff --git a/tools/test/misc/testfiles/h5clear_equal_after_size.ddl b/tools/test/misc/testfiles/h5clear_equal_after_size.ddl new file mode 100644 index 0000000..1b9a4e4 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_equal_after_size.ddl @@ -0,0 +1 @@ +EOA is 1051141; EOF is 1051141 diff --git a/tools/test/misc/testfiles/h5clear_equal_before_size.ddl b/tools/test/misc/testfiles/h5clear_equal_before_size.ddl new file mode 100644 index 0000000..9beed42 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_equal_before_size.ddl @@ -0,0 +1 @@ +EOA is 2565; EOF is 2565 diff --git a/tools/test/misc/testfiles/h5clear_fsm_persist_equal.h5 b/tools/test/misc/testfiles/h5clear_fsm_persist_equal.h5 Binary files differnew file mode 100644 index 0000000..0577690 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_fsm_persist_equal.h5 diff --git a/tools/test/misc/testfiles/h5clear_fsm_persist_greater.h5 b/tools/test/misc/testfiles/h5clear_fsm_persist_greater.h5 Binary files differnew file mode 100644 index 0000000..6358878 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_fsm_persist_greater.h5 diff --git a/tools/test/misc/testfiles/h5clear_fsm_persist_less.h5 b/tools/test/misc/testfiles/h5clear_fsm_persist_less.h5 Binary files differnew file mode 100644 index 0000000..c1f3221 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_fsm_persist_less.h5 diff --git a/tools/test/misc/testfiles/h5clear_fsm_persist_noclose.h5 b/tools/test/misc/testfiles/h5clear_fsm_persist_noclose.h5 Binary files differnew file mode 100644 index 0000000..57462db --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_fsm_persist_noclose.h5 diff --git a/tools/test/misc/testfiles/h5clear_fsm_persist_user_equal.h5 b/tools/test/misc/testfiles/h5clear_fsm_persist_user_equal.h5 Binary files differnew file mode 100644 index 0000000..5389c41 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_fsm_persist_user_equal.h5 diff --git a/tools/test/misc/testfiles/h5clear_fsm_persist_user_greater.h5 b/tools/test/misc/testfiles/h5clear_fsm_persist_user_greater.h5 Binary files differnew file mode 100644 index 0000000..f40e760 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_fsm_persist_user_greater.h5 diff --git a/tools/test/misc/testfiles/h5clear_fsm_persist_user_less.h5 b/tools/test/misc/testfiles/h5clear_fsm_persist_user_less.h5 Binary files differnew file mode 100644 index 0000000..55d0cc5 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_fsm_persist_user_less.h5 diff --git a/tools/test/misc/testfiles/h5clear_greater_after_size.ddl b/tools/test/misc/testfiles/h5clear_greater_after_size.ddl new file mode 100644 index 0000000..74c8f19 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_greater_after_size.ddl @@ -0,0 +1 @@ +EOA is 3048; EOF is 3048 diff --git a/tools/test/misc/testfiles/h5clear_greater_before_size.ddl b/tools/test/misc/testfiles/h5clear_greater_before_size.ddl new file mode 100644 index 0000000..03b22fb --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_greater_before_size.ddl @@ -0,0 +1 @@ +EOA is 3048; EOF is 2565 diff --git a/tools/test/misc/testfiles/h5clear_less_after_size.ddl b/tools/test/misc/testfiles/h5clear_less_after_size.ddl new file mode 100644 index 0000000..bedf0d2 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_less_after_size.ddl @@ -0,0 +1 @@ +EOA is 2765; EOF is 2765 diff --git a/tools/test/misc/testfiles/h5clear_less_before_size.ddl b/tools/test/misc/testfiles/h5clear_less_before_size.ddl new file mode 100644 index 0000000..50ba4c4 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_less_before_size.ddl @@ -0,0 +1 @@ +EOA is 512; EOF is 2565 diff --git a/tools/test/misc/testfiles/h5clear_missing_file.ddl b/tools/test/misc/testfiles/h5clear_missing_file.ddl index 1e5150c..13eb2c9 100644 --- a/tools/test/misc/testfiles/h5clear_missing_file.ddl +++ b/tools/test/misc/testfiles/h5clear_missing_file.ddl @@ -4,7 +4,9 @@ usage: h5clear [OPTIONS] file_name -V, --version Print version number and exit -s, --status Clear the status_flags field in the file's superblock -m, --image Remove the metadata cache image from the file - + --filesize Print the file's EOA and EOF + --increment=C Set the file's EOA to the maximum of (EOA, EOF) + C for the file <file_name> + C is >= 0; C is optional and will default to 1M when not set Examples of use: h5clear -s file_name @@ -12,4 +14,10 @@ h5clear -s file_name h5clear -m file_name Remove the metadata cache image from the HDF5 file <file_name>. + +h5clear --increment file_name + Set the EOA to the maximum of (EOA, EOF) + 1M for the file <file_name>. + +h5clear --increment=512 file_name + Set the EOA to the maximum of (EOA, EOF) + 512 for the file <file_name>. h5clear error: missing file name diff --git a/tools/test/misc/testfiles/h5clear_noclose_after_size.ddl b/tools/test/misc/testfiles/h5clear_noclose_after_size.ddl new file mode 100644 index 0000000..7846b47 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_noclose_after_size.ddl @@ -0,0 +1 @@ +EOA is 2448; EOF is 2448 diff --git a/tools/test/misc/testfiles/h5clear_noclose_before_size.ddl b/tools/test/misc/testfiles/h5clear_noclose_before_size.ddl new file mode 100644 index 0000000..f294a6d --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_noclose_before_size.ddl @@ -0,0 +1 @@ +EOA is 2048; EOF is 2448 diff --git a/tools/test/misc/testfiles/h5clear_status_noclose.h5 b/tools/test/misc/testfiles/h5clear_status_noclose.h5 Binary files differnew file mode 100644 index 0000000..94a950d --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_status_noclose.h5 diff --git a/tools/test/misc/testfiles/h5clear_status_noclose_after_size.ddl b/tools/test/misc/testfiles/h5clear_status_noclose_after_size.ddl new file mode 100644 index 0000000..7846b47 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_status_noclose_after_size.ddl @@ -0,0 +1 @@ +EOA is 2448; EOF is 2448 diff --git a/tools/test/misc/testfiles/h5clear_usage.ddl b/tools/test/misc/testfiles/h5clear_usage.ddl index a399ae7..32dd549 100644 --- a/tools/test/misc/testfiles/h5clear_usage.ddl +++ b/tools/test/misc/testfiles/h5clear_usage.ddl @@ -4,7 +4,9 @@ usage: h5clear [OPTIONS] file_name -V, --version Print version number and exit -s, --status Clear the status_flags field in the file's superblock -m, --image Remove the metadata cache image from the file - + --filesize Print the file's EOA and EOF + --increment=C Set the file's EOA to the maximum of (EOA, EOF) + C for the file <file_name> + C is >= 0; C is optional and will default to 1M when not set Examples of use: h5clear -s file_name @@ -12,3 +14,9 @@ h5clear -s file_name h5clear -m file_name Remove the metadata cache image from the HDF5 file <file_name>. + +h5clear --increment file_name + Set the EOA to the maximum of (EOA, EOF) + 1M for the file <file_name>. + +h5clear --increment=512 file_name + Set the EOA to the maximum of (EOA, EOF) + 512 for the file <file_name>. diff --git a/tools/test/misc/testfiles/h5clear_user_equal_after_size.ddl b/tools/test/misc/testfiles/h5clear_user_equal_after_size.ddl new file mode 100644 index 0000000..028e134 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_user_equal_after_size.ddl @@ -0,0 +1 @@ +EOA is 1051653; EOF is 1051653 diff --git a/tools/test/misc/testfiles/h5clear_user_equal_before_size.ddl b/tools/test/misc/testfiles/h5clear_user_equal_before_size.ddl new file mode 100644 index 0000000..ef7c391 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_user_equal_before_size.ddl @@ -0,0 +1 @@ +EOA is 3077; EOF is 3077 diff --git a/tools/test/misc/testfiles/h5clear_user_greater_after_size.ddl b/tools/test/misc/testfiles/h5clear_user_greater_after_size.ddl new file mode 100644 index 0000000..9d7de6f --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_user_greater_after_size.ddl @@ -0,0 +1 @@ +EOA is 4000; EOF is 4000 diff --git a/tools/test/misc/testfiles/h5clear_user_greater_before_size.ddl b/tools/test/misc/testfiles/h5clear_user_greater_before_size.ddl new file mode 100644 index 0000000..c3fe625 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_user_greater_before_size.ddl @@ -0,0 +1 @@ +EOA is 4000; EOF is 3077 diff --git a/tools/test/misc/testfiles/h5clear_user_less_after_size.ddl b/tools/test/misc/testfiles/h5clear_user_less_after_size.ddl new file mode 100644 index 0000000..02c0d2a --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_user_less_after_size.ddl @@ -0,0 +1 @@ +EOA is 3277; EOF is 3277 diff --git a/tools/test/misc/testfiles/h5clear_user_less_before_size.ddl b/tools/test/misc/testfiles/h5clear_user_less_before_size.ddl new file mode 100644 index 0000000..0651c2b --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_user_less_before_size.ddl @@ -0,0 +1 @@ +EOA is 3000; EOF is 3077 diff --git a/tools/test/misc/testh5clear.sh.in b/tools/test/misc/testh5clear.sh.in index 2966b2c..11c2ff9 100644 --- a/tools/test/misc/testh5clear.sh.in +++ b/tools/test/misc/testh5clear.sh.in @@ -66,6 +66,21 @@ $SRC_H5CLEAR_TESTFILES/h5clear_usage.ddl $SRC_H5CLEAR_TESTFILES/h5clear_open_fail.ddl $SRC_H5CLEAR_TESTFILES/h5clear_missing_file.ddl $SRC_H5CLEAR_TESTFILES/h5clear_no_mdc_image.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_status_noclose_after_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_noclose_before_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_noclose_after_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_equal_before_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_equal_after_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_greater_before_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_greater_after_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_less_before_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_less_after_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_user_equal_before_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_user_equal_after_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_user_greater_before_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_user_greater_after_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_user_less_before_size.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_user_less_after_size.ddl $SRC_H5CLEAR_TESTFILES/h5clear_sec2_v0.h5 $SRC_H5CLEAR_TESTFILES/h5clear_sec2_v2.h5 $SRC_H5CLEAR_TESTFILES/h5clear_sec2_v3.h5 @@ -74,6 +89,14 @@ $SRC_H5CLEAR_TESTFILES/latest_h5clear_log_v3.h5 $SRC_H5CLEAR_TESTFILES/latest_h5clear_sec2_v3.h5 $SRC_H5CLEAR_TESTFILES/h5clear_mdc_image.h5 $SRC_H5CLEAR_TESTFILES/mod_h5clear_mdc_image.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_status_noclose.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_fsm_persist_noclose.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_fsm_persist_equal.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_fsm_persist_greater.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_fsm_persist_less.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_fsm_persist_user_equal.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_fsm_persist_user_greater.h5 +$SRC_H5CLEAR_TESTFILES/h5clear_fsm_persist_user_less.h5 " COPY_TESTFILES_TO_TESTDIR() @@ -152,7 +175,8 @@ TOOLTEST_OUT() { fname=$1 option1=$2 option2=$3 - expected=$4 + option3=$4 + expected=$5 # Prepare expected and actual output expect="$TESTDIR/$expected" actual="$TESTDIR/`basename $expected .ddl`.out" @@ -161,10 +185,10 @@ TOOLTEST_OUT() { actual_err_sav=${actual_err}-sav # Run test. - TESTING $H5CLEAR $option1 $option2 $fname + TESTING $H5CLEAR $option1 $option2 $option3 $option4 $fname ( cd $TESTDIR - $RUNSERIAL $H5CLEAR_BIN $option1 $option2 $fname + $RUNSERIAL $H5CLEAR_BIN $option1 $option2 $option3 $option4 $fname ) >$actual 2>$actual_err cp $actual $actual_sav cp $actual_err $actual_err_sav @@ -245,17 +269,16 @@ $CP -f $TESTDIR/h5clear_sec2_v3.h5 $TESTDIR/orig_h5clear_sec2_v3.h5 # "h5clear -m -s junk.h5" (valid 2 options, nonexisting file) # "h5clear -m orig_h5clear_sec2_v2.h5" (valid 1 option, existing file, no cache image) # "h5clear -s -m orig_h5clear_sec2_v0.h5" (valid 2 options, existing file, no cache image) -TOOLTEST_OUT "" -h "" h5clear_usage.ddl -TOOLTEST_OUT "" "" "" h5clear_usage.ddl -TOOLTEST_OUT junk.h5 "" "" h5clear_usage.ddl -TOOLTEST_OUT orig_h5clear_sec2_v3.h5 "" "" h5clear_usage.ddl -TOOLTEST_OUT "" -m "" h5clear_missing_file.ddl -TOOLTEST_OUT junk.h5 -s "" h5clear_open_fail.ddl -TOOLTEST_OUT "" -m -s h5clear_missing_file.ddl -TOOLTEST_OUT junk.h5 -m -s h5clear_open_fail.ddl -TOOLTEST_OUT orig_h5clear_sec2_v2.h5 -m "" h5clear_no_mdc_image.ddl -TOOLTEST_OUT orig_h5clear_sec2_v0.h5 -s -m h5clear_no_mdc_image.ddl -# +TOOLTEST_OUT "" -h "" "" h5clear_usage.ddl +TOOLTEST_OUT "" "" "" "" h5clear_usage.ddl +TOOLTEST_OUT junk.h5 "" "" "" h5clear_usage.ddl +TOOLTEST_OUT orig_h5clear_sec2_v3.h5 "" "" "" h5clear_usage.ddl +TOOLTEST_OUT "" -m "" "" h5clear_missing_file.ddl +TOOLTEST_OUT junk.h5 -s "" "" h5clear_open_fail.ddl +TOOLTEST_OUT "" -m -s "" h5clear_missing_file.ddl +TOOLTEST_OUT junk.h5 -m -s "" h5clear_open_fail.ddl +TOOLTEST_OUT orig_h5clear_sec2_v2.h5 -m "" "" h5clear_no_mdc_image.ddl +TOOLTEST_OUT orig_h5clear_sec2_v0.h5 -s -m "" h5clear_no_mdc_image.ddl # # # The following are tests to verify the expected exit code from h5clear: @@ -285,8 +308,8 @@ TOOLTEST h5clear_sec2_v0.h5 -l -m $FAIL # # # h5clear_mdc_image.h5 already has cache image removed earlier, verify the expected warning from h5clear: -TOOLTEST_OUT mod_h5clear_mdc_image.h5 -m "" h5clear_no_mdc_image.ddl -TOOLTEST_OUT mod_h5clear_mdc_image.h5 -s -m h5clear_no_mdc_image.ddl +TOOLTEST_OUT mod_h5clear_mdc_image.h5 -m "" "" h5clear_no_mdc_image.ddl +TOOLTEST_OUT mod_h5clear_mdc_image.h5 -s -m "" h5clear_no_mdc_image.ddl # # # @@ -319,6 +342,73 @@ OPEN_CHK h5clear_sec2_v2.h5 $SUCCEED # # # +# (1) h5clear_status_noclose.h5 +# "h5clear --filesize h5clear_status_noclose.h5" (unable to open the file because status_flag is on) +# "h5clear -s --increment=0 h5clear_status_noclose.h5" (clear status_flag, EOA = MAX(EOA, EOF) + 0) +# (no output, check exit code) +# "h5clear --filesize h5clear_status_noclose_user.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_status_noclose.h5 --filesize "" "" h5clear_open_fail.ddl +TOOLTEST h5clear_status_noclose.h5 -s --increment=0 $SUCCEED +TOOLTEST_OUT h5clear_status_noclose.h5 --filesize "" "" h5clear_status_noclose_after_size.ddl +# +# (2) h5clear_fsm_persist_noclose.h5 +# "h5clear --filesize h5clear_fsm_persist_noclose.h5" (print EOA/EOF before the next action) +# "h5clear --increment=0 h5clear_fsm_persist_noclose.h5" (EOA = MAX(EOA, EOF)) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_noclose.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_fsm_persist_noclose.h5 --filesize "" "" h5clear_noclose_before_size.ddl +TOOLTEST h5clear_fsm_persist_noclose.h5 --increment=0 "" $SUCCEED +TOOLTEST_OUT h5clear_fsm_persist_noclose.h5 --filesize "" "" h5clear_noclose_after_size.ddl +# +# (3) h5clear_fsm_persist_equal.h5 +# "h5clear --filesize h5clear_fsm_persist_equal.h5" (print EOA/EOF before the next action) +# "h5clear --increment h5clear_fsm_persist_equal.h5" (EOA = MAX(EOA, EOF) + 1M) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_equal.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_fsm_persist_equal.h5 --filesize "" "" h5clear_equal_before_size.ddl +TOOLTEST h5clear_fsm_persist_equal.h5 --increment "" $SUCCEED +TOOLTEST_OUT h5clear_fsm_persist_equal.h5 --filesize "" "" h5clear_equal_after_size.ddl +# +# (4) h5clear_fsm_persist_greater.h5 +# "h5clear --filesize h5clear_fsm_persist_greater.h5" (print EOA/EOF before the next action) +# "h5clear --increment=0 h5clear_fsm_persist_greater.h5" (EOA = MAX(EOA, EOF) + 0) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_greater.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_fsm_persist_greater.h5 --filesize "" "" h5clear_greater_before_size.ddl +TOOLTEST h5clear_fsm_persist_greater.h5 --increment=0 "" $SUCCEED +TOOLTEST_OUT h5clear_fsm_persist_greater.h5 --filesize "" "" h5clear_greater_after_size.ddl +# +# (5) h5clear_fsm_persist_less.h5 +# "h5clear --filesize h5clear_fsm_persist_less.h5" (print EOA/EOF before the next action) +# "h5clear --increment=200 h5clear_fsm_persist_less.h5" (EOA = MAX(EOA, EOF) + 200) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_less.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_fsm_persist_less.h5 --filesize "" "" h5clear_less_before_size.ddl +TOOLTEST h5clear_fsm_persist_less.h5 --increment=200 "" $SUCCEED +TOOLTEST_OUT h5clear_fsm_persist_less.h5 --filesize "" "" h5clear_less_after_size.ddl +# +# (6) h5clear_fsm_persist_user_equal.h5 +# "h5clear --filesize h5clear_fsm_persist_user_equal.h5" (print EOA/EOF before the next action) +# "h5clear --increment h5clear_fsm_persist_user_equal.h5" (EOA = MAX(EOA, EOF) + 1M) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_user_equal.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_fsm_persist_user_equal.h5 --filesize "" "" h5clear_user_equal_before_size.ddl +TOOLTEST h5clear_fsm_persist_user_equal.h5 --increment "" $SUCCEED +TOOLTEST_OUT h5clear_fsm_persist_user_equal.h5 --filesize "" "" h5clear_user_equal_after_size.ddl +# +# (7) h5clear_fsm_persist_user_greater.h5 +# "h5clear --filesize h5clear_fsm_persist_user_greater.h5" (print EOA/EOF before the next action) +# "h5clear --increment=0 h5clear_fsm_persist_user_greater.h5" (EOA = MAX(EOA, EOF) + 0) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_user_greater.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_fsm_persist_user_greater.h5 --filesize "" "" h5clear_user_greater_before_size.ddl +TOOLTEST h5clear_fsm_persist_user_greater.h5 --increment=0 "" $SUCCEED +TOOLTEST_OUT h5clear_fsm_persist_user_greater.h5 --filesize "" "" h5clear_user_greater_after_size.ddl +# +# (8) h5clear_fsm_persist_user_less.h5 +# "h5clear --filesize h5clear_fsm_persist_user_less.h5" (print EOA/EOF before the next action) +# "h5clear --increment=200 h5clear_fsm_persist_user_less.h5" (EOA = MAX(EOA, EOF) + 200) (no output, check exit code) +# "h5clear --filesize h5clear_fsm_persist_user_less.h5" (print EOA/EOF after the last action) +TOOLTEST_OUT h5clear_fsm_persist_user_less.h5 --filesize "" "" h5clear_user_less_before_size.ddl +TOOLTEST h5clear_fsm_persist_user_less.h5 --increment=200 "" $SUCCEED +TOOLTEST_OUT h5clear_fsm_persist_user_less.h5 --filesize "" "" h5clear_user_less_after_size.ddl +# +# +# # Clean up test files if test -z "$HDF5_NOCLEANUP"; then rm -f h5clear_*.h5 latest_h5clear*.h5 |