diff options
202 files changed, 20987 insertions, 5762 deletions
@@ -579,6 +579,7 @@ ./src/H5Fmpi.c ./src/H5Fquery.c ./src/H5Fsfile.c +./src/H5Fspace.c ./src/H5Fsuper.c ./src/H5Fsuper_cache.c ./src/H5Fpkg.h @@ -793,6 +794,10 @@ ./src/H5Ppublic.h ./src/H5Pstrcpl.c ./src/H5Ptest.c +./src/H5PB.c +./src/H5PBmodule.h +./src/H5PBpkg.h +./src/H5PBprivate.h ./src/H5PL.c ./src/H5PLmodule.h ./src/H5PLprivate.h @@ -895,6 +900,7 @@ ./test/SWMR_UseCase_UG.txt ./test/accum.c ./test/accum_swmr_reader.c +./test/aggr.h5 ./test/app_ref.c ./test/atomic_reader.c ./test/atomic_writer.c @@ -957,6 +963,8 @@ ./test/flush1.c ./test/flush2.c ./test/flushrefresh.c +./test/fsm_aggr_nopersist.h5 +./test/fsm_aggr_persist.h5 ./test/genall5.c ./test/genall5.h ./test/gen_bad_ohdr.c @@ -966,7 +974,6 @@ ./test/gen_deflate.c ./test/gen_file_image.c ./test/gen_filespace.c -./test/gen_idx.c ./test/gen_mergemsg.c ./test/gen_new_array.c ./test/gen_new_fill.c @@ -986,6 +993,12 @@ ./test/getname.c ./test/gheap.c ./test/group_old.h5 +./test/h5fc_ext1_f.h5 +./test/h5fc_ext1_i.h5 +./test/h5fc_ext2_if.h5 +./test/h5fc_ext2_sf.h5 +./test/h5fc_ext3_isf.h5 +./test/h5fc_ext_none.h5 ./test/h5test.c ./test/h5test.h ./test/hyperslab.c @@ -1002,9 +1015,13 @@ ./test/multi_file_v16-r.h5 ./test/multi_file_v16-s.h5 ./test/noencoder.h5 +./test/none.h5 ./test/ntypes.c ./test/ohdr.c ./test/objcopy.c +./test/page_buffer.c +./test/paged_nopersist.h5 +./test/paged_persist.h5 ./test/plugin.c ./test/reserved.c ./test/pool.c @@ -1203,6 +1220,7 @@ ./testpar/COPYING ./testpar/Makefile.am ./testpar/t_cache.c +./testpar/t_cache_image.c ./testpar/t_chunk_alloc.c ./testpar/t_coll_chunk.c ./testpar/t_dset.c @@ -1448,6 +1466,14 @@ ./tools/test/misc/testh5mkgrp.sh.in ./tools/test/misc/testh5repart.sh.in ./tools/test/misc/talign.c +./tools/test/misc/testfiles/h5clear_missing_file.ddl +./tools/test/misc/testfiles/h5clear_no_mdc_image.ddl +./tools/test/misc/testfiles/h5clear_open_fail.ddl +./tools/test/misc/testfiles/h5clear_usage.ddl +./tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 +./tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 +./tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 +./tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 ./tools/test/misc/testfiles/h5mkgrp_help.txt ./tools/test/misc/testfiles/h5mkgrp_version.txt.in ./tools/test/misc/h5perf_gentest.c @@ -1637,7 +1663,6 @@ ./tools/testfiles/tdset.h5 ./tools/testfiles/tdset2.h5 ./tools/testfiles/tdset_idx.ls -./tools/testfiles/tdset_idx.ddl ./tools/testfiles/tdset_idx.h5 ./tools/testfiles/tempty.ddl ./tools/testfiles/tempty.h5 @@ -2270,7 +2295,6 @@ ./tools/test/h5diff/testfiles/h5diff_457.txt ./tools/test/h5diff/testfiles/h5diff_458.txt ./tools/test/h5diff/testfiles/h5diff_459.txt -./tools/test/h5diff/testfiles/h5diff_idx.txt ./tools/test/h5diff/testfiles/h5diff_465.txt ./tools/test/h5diff/testfiles/h5diff_466.txt ./tools/test/h5diff/testfiles/h5diff_467.txt @@ -2319,8 +2343,6 @@ ./tools/test/h5diff/testfiles/h5diff_basic2.h5 ./tools/test/h5diff/testfiles/h5diff_dset1.h5 ./tools/test/h5diff/testfiles/h5diff_dset2.h5 -./tools/test/h5diff/testfiles/h5diff_dset_idx1.h5 -./tools/test/h5diff/testfiles/h5diff_dset_idx2.h5 ./tools/test/h5diff/testfiles/h5diff_dtypes.h5 ./tools/test/h5diff/testfiles/h5diff_empty.h5 ./tools/test/h5diff/testfiles/h5diff_hyper1.h5 @@ -3123,7 +3145,9 @@ ./tools/test/h5stat/CMakeTests.cmake ./tools/src/misc/CMakeLists.txt ./tools/test/misc/CMakeLists.txt -./tools/test/misc/CMakeTests.cmake +./tools/test/misc/CMakeTestsClear.cmake +./tools/test/misc/CMakeTestsMkgrp.cmake +./tools/test/misc/CMakeTestsRepart.cmake ./tools/test/misc/vds/CMakeLists.txt ./tools/test/perform/CMakeLists.txt ./tools/test/perform/CMakeTests.cmake @@ -49,9 +49,11 @@ $Source = ""; "H5E_error_t" => "Ee", "H5E_type_t" => "Et", "H5F_close_degree_t" => "Fd", + "H5F_fspace_strategy_t" => "Ff", "H5F_file_space_type_t" => "Ff", "H5F_mem_t" => "Fm", "H5F_scope_t" => "Fs", + "H5F_fspace_type_t" => "Ft", "H5F_libver_t" => "Fv", "H5G_obj_t" => "Go", "H5G_stat_t" => "Gs", diff --git a/c++/src/H5Attribute.cpp b/c++/src/H5Attribute.cpp index 6701f6e..c506906 100644 --- a/c++/src/H5Attribute.cpp +++ b/c++/src/H5Attribute.cpp @@ -1,7 +1,7 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * * Copyright by the Board of Trustees of the University of Illinois. * -/* All rights reserved. * + * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * * terms governing use, modification, and redistribution, is contained in * diff --git a/c++/src/H5FcreatProp.cpp b/c++/src/H5FcreatProp.cpp index c490f26..ea69049 100644 --- a/c++/src/H5FcreatProp.cpp +++ b/c++/src/H5FcreatProp.cpp @@ -294,6 +294,7 @@ unsigned FileCreatPropList::getIstorek() const return(ik); } +#ifndef H5_NO_DEPRECATED_SYMBOLS //-------------------------------------------------------------------------- // Function: FileCreatPropList::setFileSpace ///\brief Sets the strategy and the threshold value that the library @@ -364,6 +365,7 @@ hsize_t FileCreatPropList::getFileSpaceThreshold() const } return(threshold); } +#endif /* H5_NO_DEPRECATED_SYMBOLS */ //-------------------------------------------------------------------------- // Function: FileCreatPropList destructor diff --git a/c++/src/H5FcreatProp.h b/c++/src/H5FcreatProp.h index 1bbaabf..1d999cb 100644 --- a/c++/src/H5FcreatProp.h +++ b/c++/src/H5FcreatProp.h @@ -65,6 +65,7 @@ class H5_DLLCPP FileCreatPropList : public PropList { // indexing chunked datasets. void setIstorek(unsigned ik) const; +#ifndef H5_NO_DEPRECATED_SYMBOLS // Sets the strategy and the threshold value that the library will // will employ in managing file space. void setFileSpace(H5F_file_space_type_t strategy, hsize_t threshold) const; @@ -75,6 +76,7 @@ class H5_DLLCPP FileCreatPropList : public PropList { // Returns the threshold value that the library uses in tracking free // space sections. hsize_t getFileSpaceThreshold() const; +#endif /* H5_NO_DEPRECATED_SYMBOLS */ ///\brief Returns this class name. virtual H5std_string fromClass() const { return("FileCreatPropList"); } diff --git a/c++/test/tfile.cpp b/c++/test/tfile.cpp index 261beab..eb07657 100644 --- a/c++/test/tfile.cpp +++ b/c++/test/tfile.cpp @@ -817,12 +817,14 @@ static void test_file_info() // Output message about test being performed SUBTEST("File general information"); +#ifndef H5_NO_DEPRECATED_SYMBOLS hsize_t in_threshold = 2; // Free space section threshold to set */ hsize_t out_threshold = 0; // Free space section threshold to get */ // File space handling strategy H5F_file_space_type_t in_strategy = H5F_FILE_SPACE_ALL; // File space handling strategy H5F_file_space_type_t out_strategy = H5F_FILE_SPACE_DEFAULT; +#endif /* H5_NO_DEPRECATED_SYMBOLS */ try { // Create a file using default properties. @@ -846,7 +848,9 @@ static void test_file_info() fcpl.setSizes(F2_OFFSET_SIZE, F2_LENGTH_SIZE); fcpl.setSymk(F2_SYM_INTERN_K, F2_SYM_LEAF_K); fcpl.setIstorek(F2_ISTORE); +#ifndef H5_NO_DEPRECATED_SYMBOLS fcpl.setFileSpace(in_strategy, in_threshold); +#endif /* H5_NO_DEPRECATED_SYMBOLS */ // Creating a file with the non-default file creation property list // should create a version 1 superblock @@ -859,7 +863,11 @@ static void test_file_info() // Get the file's version information. file7.getFileInfo(finfo); +#ifndef H5_NO_DEPRECATED_SYMBOLS verify_val(finfo.super.version, 2, "H5File::getFileInfo", __LINE__, __FILE__); +#else /* H5_NO_DEPRECATED_SYMBOLS */ + verify_val(finfo.super.version, 1, "H5File::getFileInfo", __LINE__, __FILE__); +#endif /* H5_NO_DEPRECATED_SYMBOLS */ verify_val(finfo.free.version, 0, "H5File::getFileInfo", __LINE__, __FILE__); verify_val(finfo.sohm.version, 0, "H5File::getFileInfo", __LINE__, __FILE__); @@ -874,7 +882,11 @@ static void test_file_info() // Get the file's version information. file7.getFileInfo(finfo); +#ifndef H5_NO_DEPRECATED_SYMBOLS verify_val(finfo.super.version, 2, "H5File::getFileInfo", __LINE__, __FILE__); +#else /* H5_NO_DEPRECATED_SYMBOLS */ + verify_val(finfo.super.version, 1, "H5File::getFileInfo", __LINE__, __FILE__); +#endif /* H5_NO_DEPRECATED_SYMBOLS */ verify_val(finfo.free.version, 0, "H5File::getFileInfo", __LINE__, __FILE__); verify_val(finfo.sohm.version, 0, "H5File::getFileInfo", __LINE__, __FILE__); @@ -900,12 +912,14 @@ static void test_file_info() VERIFY(nindexes, MISC11_NINDEXES, "H5Pget_shared_mesg_nindexes"); */ +#ifndef H5_NO_DEPRECATED_SYMBOLS // Get and verify the file space info from the creation property list */ out_strategy = fcpl2.getFileSpaceStrategy(); verify_val(static_cast<unsigned>(out_strategy), static_cast<unsigned>(in_strategy), "FileCreatPropList::getFileSpaceStrategy", __LINE__, __FILE__); out_threshold = fcpl2.getFileSpaceThreshold(); verify_val(static_cast<unsigned>(out_threshold), static_cast<unsigned>(in_threshold), "FileCreatPropList::getFileSpaceThreshold", __LINE__, __FILE__); +#endif /* H5_NO_DEPRECATED_SYMBOLS */ PASSED(); } // end of try block diff --git a/fortran/test/tH5F.F90 b/fortran/test/tH5F.F90 index 8334b30..ea8c53f 100644 --- a/fortran/test/tH5F.F90 +++ b/fortran/test/tH5F.F90 @@ -729,7 +729,7 @@ CONTAINS CALL h5fget_freespace_f(fid, free_space, error) CALL check("h5fget_freespace_f",error,total_error) - if(error .eq.0 .and. free_space .ne. 0) then + if(error .eq.0 .and. free_space .ne. 1248) then total_error = total_error + 1 write(*,*) "1: Wrong amount of free space reported, ", free_space endif @@ -745,7 +745,7 @@ CONTAINS ! Check the free space now CALL h5fget_freespace_f(fid, free_space, error) CALL check("h5fget_freespace_f",error,total_error) - if(error .eq.0 .and. free_space .ne. 0) then + if(error .eq.0 .and. free_space .ne. 216) then total_error = total_error + 1 write(*,*) "2: Wrong amount of free space reported, ", free_space endif @@ -757,7 +757,7 @@ CONTAINS ! Check the free space now CALL h5fget_freespace_f(fid, free_space, error) CALL check("h5fget_freespace_f",error,total_error) - if(error .eq.0 .and. free_space .ne. 0) then + if(error .eq.0 .and. free_space .ne. 1248) then total_error = total_error + 1 write(*,*) "3: Wrong amount of free space reported, ", free_space endif diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 02d35ba..5b35d17 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -5356,39 +5356,24 @@ public class H5 implements java.io.Serializable { throws HDF5LibraryException, IllegalArgumentException; /** - * H5Pset_file_space sets the file space management strategy for the file associated with fcpl_id to strategy. + * H5Pset_file_space_strategy sets the file space management strategy for the file associated with fcpl_id to strategy. * There are four strategies that applications can select and they are described in the Parameters section. * * @param fcpl_id * IN: File creation property list identifier * @param strategy * IN: The strategy for file space management. - * Passing a value of zero (0) indicates that the value of strategy is not to be modified. - * H5F_FILE_SPACE_ALL_PERSIST - * With this strategy, the free-space managers track the free space that results from the - * manipulation of HDF5 objects in the HDF5 file. The free space information is saved when the - * file is closed, and reloaded when the file is reopened. When space is needed for file metadata - * or raw data, the HDF5 library first requests space from the library's free-space managers. - * If the request is not satisfied, the library requests space from the aggregators. If the request - * is still not satisfied, the library requests space from the virtual file driver. That is, the - * library will use all of the mechanisms for allocating space. - * H5F_FILE_SPACE_ALL (Default file space management strategy) - * With this strategy, the free-space managers track the free space that results from the manipulation - * of HDF5 objects in the HDF5 file. The free space information is NOT saved when the file is closed - * and the free space that exists upon file closing becomes unaccounted space in the file. - * Like the previous strategy, the library will try all of the mechanisms for allocating space. When - * space is needed for file metadata or raw data, the library first requests space from the free-space - * managers. If the request is not satisfied, the library requests space from the aggregators. If the - * request is still not satisfied, the library requests space from the virtual file driver. - * H5F_FILE_SPACE_AGGR_VFD - * With this strategy, the library does not track free space that results from the manipulation of HDF5 - * obejcts in the HDF5 file and the free space becomes unaccounted space in the file. - * When space is needed for file metadata or raw data, the library first requests space from the - * aggregators. If the request is not satisfied, the library requests space from the virtual file driver. - * H5F_FILE_SPACE_VFD - * With this strategy, the library does not track free space that results from the manipulation of HDF5 - * obejcts in the HDF5 file and the free space becomes unaccounted space in the file. - * When space is needed for file metadata or raw data, the library requests space from the virtual file driver. + * H5F_FSPACE_STRATEGY_FSM_AGGR + * Mechanisms: free-space managers, aggregators, and virtual file drivers + * This is the library default when not set. + * H5F_FSPACE_STRATEGY_PAGE + * Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers + * H5F_FSPACE_STRATEGY_AGGR + * Mechanisms: aggregators and virtual file drivers + * H5F_FSPACE_STRATEGY_NONE + * Mechanisms: virtual file drivers + * @param persist + * IN: True to persist free-space. * @param threshold * IN: The free-space section threshold. The library default is 1, which is to track all free-space sections. * Passing a value of zero (0) indicates that the value of threshold is not to be modified. @@ -5399,28 +5384,100 @@ public class H5 implements java.io.Serializable { * - Invalid values of max_list and min_btree. * **/ - public synchronized static native void H5Pset_file_space(long fcpl_id, int strategy, long threshold) + public synchronized static native void H5Pset_file_space_strategy(long fcpl_id, int strategy, boolean persist, long threshold) throws HDF5LibraryException, IllegalArgumentException; /** - * H5Pget_file_space provides the means for applications to manage the HDF5 file's file space for their specific needs. + * H5Pget_file_space_strategy provides the means for applications to manage the HDF5 file's file space strategy for their specific needs. * * @param fcpl_id * IN: File creation property list identifier - * @param strategy - * IN/OUT: The current file space management strategy in use for the file. NULL, strategy not queried. + * @param persist + * IN/OUT: The current free-space persistence. NULL, persist not queried. * @param threshold * IN/OUT: The current free-space section threshold. NULL, threshold not queried. * + * @return the current free-space strategy. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native int H5Pget_file_space_strategy(long fcpl_id, boolean[] persist, long[] threshold) + throws HDF5LibraryException, IllegalArgumentException; + + /** + * H5Pget_file_space_strategy_persist provides the means for applications to manage the HDF5 file's file space strategy for their specific needs. + * + * @param fcpl_id + * IN: File creation property list identifier + * + * @return the current free-space persistence. + * * @exception HDF5LibraryException * - Error from the HDF-5 Library. * @exception IllegalArgumentException * - Invalid values of max_list and min_btree. * **/ - public synchronized static native void H5Pget_file_space(long fcpl_id, int[] strategy, long[] threshold) + public synchronized static native boolean H5Pget_file_space_strategy_persist(long fcpl_id) throws HDF5LibraryException, IllegalArgumentException; + /** + * H5Pget_file_space_strategy_threshold provides the means for applications to manage the HDF5 file's file space strategy for their specific needs. + * + * @param fcpl_id + * IN: File creation property list identifier + * + * @return the current free-space section threshold. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native long H5Pget_file_space_strategy_threshold(long fcpl_id) + throws HDF5LibraryException, IllegalArgumentException; + + /** + * H5Pset_file_space_page_size retrieves the file space page size for aggregating small metadata or raw data. + * + * @param fcpl_id + * IN: File creation property list identifier + * @param page_size + * IN: the file space page size. + * + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native void H5Pset_file_space_page_size(long fcpl_id, long page_size) + throws HDF5LibraryException, IllegalArgumentException; + + /** + * H5Pget_file_space_page_size Sets the file space page size for paged aggregation. + * + * @param fcpl_id + * IN: File creation property list identifier + * + * @return the current file space page size. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * @exception IllegalArgumentException + * - Invalid values of max_list and min_btree. + * + **/ + public synchronized static native long H5Pget_file_space_page_size(long fcpl_id) + throws HDF5LibraryException, IllegalArgumentException; + + // File access property list (FAPL) routines /** diff --git a/java/src/hdf/hdf5lib/HDF5Constants.java b/java/src/hdf/hdf5lib/HDF5Constants.java index acd3eb5..9773100 100644 --- a/java/src/hdf/hdf5lib/HDF5Constants.java +++ b/java/src/hdf/hdf5lib/HDF5Constants.java @@ -219,12 +219,12 @@ public class HDF5Constants { public static final int H5F_SCOPE_GLOBAL = H5F_SCOPE_GLOBAL(); public static final int H5F_SCOPE_LOCAL = H5F_SCOPE_LOCAL(); public static final int H5F_UNLIMITED = H5F_UNLIMITED(); - public static final int H5F_FILE_SPACE_DEFAULT = H5F_FILE_SPACE_DEFAULT(); - public static final int H5F_FILE_SPACE_ALL_PERSIST = H5F_FILE_SPACE_ALL_PERSIST(); - public static final int H5F_FILE_SPACE_ALL = H5F_FILE_SPACE_ALL(); - public static final int H5F_FILE_SPACE_AGGR_VFD = H5F_FILE_SPACE_AGGR_VFD(); - public static final int H5F_FILE_SPACE_VFD = H5F_FILE_SPACE_VFD(); - public static final int H5F_FILE_SPACE_NTYPES = H5F_FILE_SPACE_NTYPES(); + + public static final int H5F_FSPACE_STRATEGY_FSM_AGGR = H5F_FSPACE_STRATEGY_FSM_AGGR(); + public static final int H5F_FSPACE_STRATEGY_AGGR = H5F_FSPACE_STRATEGY_AGGR(); + public static final int H5F_FSPACE_STRATEGY_PAGE = H5F_FSPACE_STRATEGY_PAGE(); + public static final int H5F_FSPACE_STRATEGY_NONE = H5F_FSPACE_STRATEGY_NONE(); + public static final int H5F_FSPACE_STRATEGY_NTYPES = H5F_FSPACE_STRATEGY_NTYPES(); public static final long H5FD_CORE = H5FD_CORE(); public static final long H5FD_DIRECT = H5FD_DIRECT(); @@ -1024,17 +1024,15 @@ public class HDF5Constants { private static native final int H5F_UNLIMITED(); - private static native final int H5F_FILE_SPACE_DEFAULT(); - - private static native final int H5F_FILE_SPACE_ALL_PERSIST(); + private static native final int H5F_FSPACE_STRATEGY_FSM_AGGR(); - private static native final int H5F_FILE_SPACE_ALL(); + private static native final int H5F_FSPACE_STRATEGY_AGGR(); - private static native final int H5F_FILE_SPACE_AGGR_VFD(); + private static native final int H5F_FSPACE_STRATEGY_PAGE(); - private static native final int H5F_FILE_SPACE_VFD(); + private static native final int H5F_FSPACE_STRATEGY_NONE(); - private static native final int H5F_FILE_SPACE_NTYPES(); + private static native final int H5F_FSPACE_STRATEGY_NTYPES(); private static native final long H5FD_CORE(); diff --git a/java/src/jni/h5Constants.c b/java/src/jni/h5Constants.c index ccd0b91..c6aa16c 100644 --- a/java/src/jni/h5Constants.c +++ b/java/src/jni/h5Constants.c @@ -403,17 +403,15 @@ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_HDF5Constants_H5F_1UNLIMITED(JNIEnv *env, jclass cls) { return (jint)H5F_UNLIMITED; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1DEFAULT(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_DEFAULT; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1FSM_1AGGR(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_FSM_AGGR; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1ALL_1PERSIST(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_ALL_PERSIST; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1AGGR(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_AGGR; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1ALL(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_ALL; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1PAGE(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_PAGE; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1AGGR_1VFD(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_AGGR_VFD; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1NONE(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_NONE; } JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1VFD(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_VFD; } -JNIEXPORT jint JNICALL -Java_hdf_hdf5lib_HDF5Constants_H5F_1FILE_1SPACE_1NTYPES(JNIEnv *env, jclass cls) { return H5F_FILE_SPACE_NTYPES; } +Java_hdf_hdf5lib_HDF5Constants_H5F_1FSPACE_1STRATEGY_1NTYPES(JNIEnv *env, jclass cls) { return H5F_FSPACE_STRATEGY_NTYPES; } JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5FD_1CORE(JNIEnv *env, jclass cls) { return H5FD_CORE; } diff --git a/java/src/jni/h5pImp.c b/java/src/jni/h5pImp.c index 1368139..f39f0e6 100644 --- a/java/src/jni/h5pImp.c +++ b/java/src/jni/h5pImp.c @@ -4883,61 +4883,130 @@ Java_hdf_hdf5lib_H5_H5Pset_1virtual_1printf_1gap /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_file_space - * Signature: (J[I[J)V + * Method: H5Pget_file_space_strategy + * Signature: (J[Z[J)I */ -JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pget_1file_1space - (JNIEnv *env, jclass clss, jlong fcpl_id, jintArray strategy, jlongArray threshold) +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy + (JNIEnv *env, jclass clss, jlong fcpl_id, jbooleanArray persist, jlongArray threshold) { herr_t status = -1; - jint *thestrategyArray = NULL; + H5F_fspace_strategy_t thestrategy = H5F_FSPACE_STRATEGY_FSM_AGGR; /* Library default */ jlong *thethresholdArray = NULL; + jboolean *thepersistArray = NULL; jboolean isCopy; - if (strategy) { - thestrategyArray = (jint*)ENVPTR->GetIntArrayElements(ENVPAR strategy, &isCopy); - if (thestrategyArray == NULL) { - h5JNIFatalError(env, "H5Pget_file_space: strategy not pinned"); - return; + if (persist) { + thepersistArray = (jboolean*)ENVPTR->GetBooleanArrayElements(ENVPAR persist, &isCopy); + if (thepersistArray == NULL) { + h5JNIFatalError(env, "H5Pget_file_space: persist not pinned"); + return -1; } } if (threshold) { thethresholdArray = (jlong*)ENVPTR->GetLongArrayElements(ENVPAR threshold, &isCopy); if (thethresholdArray == NULL) { - if (strategy) ENVPTR->ReleaseIntArrayElements(ENVPAR strategy, thestrategyArray, JNI_ABORT); + if (persist) ENVPTR->ReleaseBooleanArrayElements(ENVPAR persist, thepersistArray, JNI_ABORT); h5JNIFatalError(env, "H5Pget_file_space: threshold not pinned"); - return; + return -1; } /* end if */ } /* end if */ - status = H5Pget_file_space((hid_t)fcpl_id, (H5F_file_space_type_t*)thestrategyArray, (hsize_t*)thethresholdArray); + status = H5Pget_file_space_strategy((hid_t)fcpl_id, &thestrategy, (hbool_t*)thepersistArray, (hsize_t*)thethresholdArray); if (status < 0) { - if (strategy) ENVPTR->ReleaseIntArrayElements(ENVPAR strategy, thestrategyArray, JNI_ABORT); + if (persist) ENVPTR->ReleaseBooleanArrayElements(ENVPAR persist, thepersistArray, JNI_ABORT); if (threshold) ENVPTR->ReleaseLongArrayElements(ENVPAR threshold, thethresholdArray, JNI_ABORT); h5libraryError(env); } /* end if */ else { - if (strategy) ENVPTR->ReleaseIntArrayElements(ENVPAR strategy, thestrategyArray, 0); + if (persist) ENVPTR->ReleaseBooleanArrayElements(ENVPAR persist, thepersistArray, 0); if (threshold) ENVPTR->ReleaseLongArrayElements(ENVPAR threshold, thethresholdArray, 0); } /* end else */ -} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space */ + return (jint)thestrategy; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_persist + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1persist + (JNIEnv *env, jclass clss, jlong fcpl_id) +{ + herr_t status = -1; + hbool_t thepersist = FALSE; + + status = H5Pget_file_space_strategy((hid_t)fcpl_id, NULL, &thepersist, NULL); + + if (status < 0) + h5libraryError(env); + + return (jboolean)thepersist; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1persist */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_threshold + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1threshold + (JNIEnv *env, jclass clss, jlong fcpl_id) +{ + herr_t status = -1; + hsize_t thethreshold; + + status = H5Pget_file_space_strategy((hid_t)fcpl_id, NULL, NULL, &thethreshold); + + if (status < 0) + h5libraryError(env); + + return (jlong)thethreshold; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1threshold */ /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_file_space - * Signature: (JIJ)V + * Method: H5Pset_file_space_strategy + * Signature: (JIZJ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1file_1space - (JNIEnv *env, jclass clss, jlong fcpl_id, jint strategy, jlong threshold) +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1strategy + (JNIEnv *env, jclass clss, jlong fcpl_id, jint strategy, jboolean persist, jlong threshold) { - if (H5Pset_file_space((hid_t)fcpl_id, (H5F_file_space_type_t)strategy, (hsize_t)threshold) < 0) + if (H5Pset_file_space_strategy((hid_t)fcpl_id, (H5F_fspace_strategy_t)strategy, (hbool_t)persist, (hsize_t)threshold) < 0) h5libraryError(env); -} /* end Java_hdf_hdf5lib_H5_H5Pset_1file_1space */ +} /* end Java_hdf_hdf5lib_H5_H5Pset_file_space_strategy */ +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_space_page_size + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1page_1size + (JNIEnv *env, jclass clss, jlong fcpl_id, jlong fsp_size) +{ + if (H5Pset_file_space_page_size((hid_t)fcpl_id, (hsize_t)fsp_size) < 0) + h5libraryError(env); +} + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_page_size + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1page_1size + (JNIEnv *env, jclass clss, jlong fcpl_id) +{ + hsize_t fsp_size = 0; + if (H5Pget_file_space_page_size((hid_t)fcpl_id, &fsp_size) < 0) + h5libraryError(env); + return (jlong)fsp_size; +} static herr_t H5P_cls_create_cb diff --git a/java/src/jni/h5pImp.h b/java/src/jni/h5pImp.h index 927501e..03b0e03 100644 --- a/java/src/jni/h5pImp.h +++ b/java/src/jni/h5pImp.h @@ -1385,22 +1385,57 @@ Java_hdf_hdf5lib_H5_H5Pset_1virtual_1printf_1gap /* * Class: hdf_hdf5lib_H5 - * Method: H5Pget_file_space - * Signature: (J[I[J)V + * Method: H5Pget_file_space_strategy + * Signature: (J[Z[J)I + */ +JNIEXPORT jint JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy +(JNIEnv *, jclass, jlong, jbooleanArray, jlongArray); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_persist + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1persist +(JNIEnv *, jclass, jlong); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_strategy_threshold + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1strategy_1threshold +(JNIEnv *, jclass, jlong); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_space_strategy + * Signature: (JIZJ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pget_1file_1space -(JNIEnv *, jclass, jlong, jintArray, jlongArray); +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1strategy +(JNIEnv *, jclass, jlong, jint, jboolean, jlong); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_file_space_page_size + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1file_1space_1page_1size +(JNIEnv *, jclass, jlong); /* * Class: hdf_hdf5lib_H5 - * Method: H5Pset_file_space - * Signature: (JIJ)V + * Method: H5Pset_file_space_page_size + * Signature: (JJ)V */ JNIEXPORT void JNICALL -Java_hdf_hdf5lib_H5_H5Pset_1file_1space -(JNIEnv *, jclass, jlong, jint, jlong); +Java_hdf_hdf5lib_H5_H5Pset_1file_1space_1page_1size +(JNIEnv *, jclass, jlong, jlong); /* * Class: hdf_hdf5lib_H5 diff --git a/java/test/JUnit-interface.txt b/java/test/JUnit-interface.txt index 34ec02a..cae8cef 100644 --- a/java/test/JUnit-interface.txt +++ b/java/test/JUnit-interface.txt @@ -416,6 +416,7 @@ JUnit version 4.11 .testH5P_layout .testH5Pget_link_creation_order .testH5Pset_shared_mesg_nindexes_InvalidHIGHnindexes +.testH5P_file_space_page_size .testH5Pget_shared_mesg_index_Invalid_indexnum .testH5Pset_data_transform_NullExpression .testH5Pset_elink_prefix_null @@ -438,9 +439,9 @@ JUnit version 4.11 .testH5Pset_shared_mesg_index .testH5Pset_copy_object .testH5Pset_link_creation_order_trackedPLUSindexed +.testH5P_file_space_strategy .testH5Pset_copy_object_invalidobject .testH5Pset_est_link_info_InvalidValues -.testH5P_file_space .testH5Pset_local_heap_size_hint .testH5Pget_est_link_info .testH5Pset_scaleoffset @@ -638,7 +639,7 @@ JUnit version 4.11 Time: XXXX -OK (636 tests) +OK (637 tests) HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): #000: (file name) line (number) in H5Fopen(): can't set access and transfer property lists diff --git a/java/test/TestH5P.java b/java/test/TestH5P.java index b8a4376..fa8b525 100644 --- a/java/test/TestH5P.java +++ b/java/test/TestH5P.java @@ -1198,25 +1198,40 @@ public class TestH5P { } @Test - public void testH5P_file_space() { + public void testH5P_file_space_strategy() { long[] threshold = {0}; - int[] strategy = {0}; + boolean[] persist = {false}; + int strategy = 0; try { - H5.H5Pget_file_space(fcpl_id, strategy, threshold); - assertTrue("strategy: "+strategy[0], strategy[0] == HDF5Constants.H5F_FILE_SPACE_ALL); + strategy = H5.H5Pget_file_space_strategy(fcpl_id, persist, threshold); + assertTrue("strategy(default): "+strategy, strategy == HDF5Constants.H5F_FSPACE_STRATEGY_FSM_AGGR); + assertTrue("persist(default): "+persist[0], persist[0] == false); + assertTrue("theshold(default): "+threshold[0], threshold[0] == 1); + H5.H5Pset_file_space_strategy(fcpl_id, HDF5Constants.H5F_FSPACE_STRATEGY_PAGE, true, 1); + strategy = H5.H5Pget_file_space_strategy(fcpl_id, persist, threshold); + assertTrue("strategy: "+strategy, strategy == HDF5Constants.H5F_FSPACE_STRATEGY_PAGE); + assertTrue("persist: "+persist[0], persist[0] == true); assertTrue("theshold: "+threshold[0], threshold[0] == 1); - H5.H5Pset_file_space(fcpl_id, HDF5Constants.H5F_FILE_SPACE_ALL_PERSIST, 10); - H5.H5Pget_file_space(fcpl_id, strategy, threshold); - assertTrue("strategy: "+strategy[0], strategy[0] == HDF5Constants.H5F_FILE_SPACE_ALL_PERSIST); - assertTrue("theshold: "+threshold[0], threshold[0] == 10); - H5.H5Pset_file_space(fcpl_id, HDF5Constants.H5F_FILE_SPACE_VFD, 0); - H5.H5Pget_file_space(fcpl_id, strategy, threshold); - assertTrue("strategy: "+strategy[0], strategy[0] == HDF5Constants.H5F_FILE_SPACE_VFD); - assertTrue("theshold: "+threshold[0], threshold[0] == 10); } catch (Throwable err) { err.printStackTrace(); - fail("testH5P_file_space: " + err); + fail("testH5P_file_space_strategy: " + err); + } + } + + @Test + public void testH5P_file_space_page_size() { + long page_size = 0; + try { + page_size = H5.H5Pget_file_space_page_size(fcpl_id); + assertTrue("page_size(default): "+page_size, page_size == 4096); + H5.H5Pset_file_space_page_size(fcpl_id, 512); + page_size = H5.H5Pget_file_space_page_size(fcpl_id); + assertTrue("page_size: "+page_size, page_size == 512); + } + catch (Throwable err) { + err.printStackTrace(); + fail("testH5P_file_space_page_size: " + err); } } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef361b2..9321bbd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -195,6 +195,7 @@ set (H5F_SOURCES ${HDF5_SRC_DIR}/H5Fmpi.c ${HDF5_SRC_DIR}/H5Fquery.c ${HDF5_SRC_DIR}/H5Fsfile.c + ${HDF5_SRC_DIR}/H5Fspace.c ${HDF5_SRC_DIR}/H5Fsuper.c ${HDF5_SRC_DIR}/H5Fsuper_cache.c ${HDF5_SRC_DIR}/H5Ftest.c @@ -504,6 +505,15 @@ set (H5P_HDRS ) IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SOURCES}" ) +set (H5PB_SOURCES + ${HDF5_SRC_DIR}/H5PB.c +) + +set (H5PB_HDRS + ${HDF5_SRC_DIR}/H5PBpkg.h +) +IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" ) + set (H5PL_SOURCES ${HDF5_SRC_DIR}/H5PL.c ) @@ -699,6 +709,7 @@ set (common_SRCS ${H5MP_SOURCES} ${H5O_SOURCES} ${H5P_SOURCES} + ${H5PB_SOURCES} ${H5PL_SOURCES} ${H5R_SOURCES} ${H5UC_SOURCES} @@ -739,6 +750,7 @@ set (H5_PUBLIC_HEADERS ${H5MP_HDRS} ${H5O_HDRS} ${H5P_HDRS} + ${H5PB_HDRS} ${H5PL_HDRS} ${H5R_HDRS} ${H5S_HDRS} @@ -777,6 +789,7 @@ set (H5_PRIVATE_HEADERS ${HDF5_SRC_DIR}/H5MPprivate.h ${HDF5_SRC_DIR}/H5Oprivate.h ${HDF5_SRC_DIR}/H5Pprivate.h + ${HDF5_SRC_DIR}/H5PBprivate.h ${HDF5_SRC_DIR}/H5PLprivate.h ${HDF5_SRC_DIR}/H5UCprivate.h ${HDF5_SRC_DIR}/H5Rprivate.h @@ -353,6 +353,44 @@ H5AC_term_package(void) /*------------------------------------------------------------------------- + * + * Function: H5AC_cache_image_pending() + * + * Purpose: Debugging function that tests to see if the load of a + * metadata cache image load is pending (i.e. will be executed + * on the next protect or insert) + * + * Returns TRUE if a cache image load is pending, and FALSE + * if not. Throws an assertion failure on error. + * + * Return: TRUE if a cache image load is pending, and FALSE otherwise. + * + * Programmer: John Mainzer, 1/10/17 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +hbool_t +H5AC_cache_image_pending(const H5F_t *f) +{ + H5C_t *cache_ptr; + hbool_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + + ret_value = H5C_cache_image_pending(cache_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_cache_image_pending() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_create * * Purpose: Initialize the cache just after a file is opened. The @@ -835,6 +873,46 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_force_cache_image_load() + * + * Purpose: On rare occasions, it is necessary to run + * H5MF_tidy_self_referential_fsm_hack() prior to the first + * metadata cache access. This is a problem as if there is a + * cache image at the end of the file, that routine will + * discard it. + * + * We solve this issue by calling this function, which will + * load the cache image and then call + * H5MF_tidy_self_referential_fsm_hack() to discard it. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 1/11/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_force_cache_image_load(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if(H5C_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't load cache image") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_force_cache_image_load() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_get_entry_status * * Purpose: Given a file address, determine whether the metadata @@ -3119,6 +3197,48 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_unsettle_ring() + * + * Purpose: Advise the metadata cache that the specified free space + * manager ring is no longer settled (if it was on entry). + * + * If the target free space manager ring is already + * unsettled, do nothing, and return SUCCEED. + * + * If the target free space manager ring is settled, and + * we are not in the process of a file shutdown, mark + * the ring as unsettled, and return SUCCEED. + * + * If the target free space manager is settled, and we + * are in the process of a file shutdown, post an error + * message, and return FAIL. + * + * Note that this function simply passes the call on to + * the metadata cache proper, and returns the result. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/15/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_unsettle_ring(H5F_t * f, H5C_ring_t ring) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if(FAIL == (ret_value = H5C_unsettle_ring(f, ring))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_unsettle_ring() failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_unsettle_ring() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_remove_entry() * * Purpose: Remove an entry from the cache. Must be not protected, pinned, diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 1dc8270..1f8299b 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -451,6 +451,8 @@ H5_DLL herr_t H5AC_validate_config(H5AC_cache_config_t *config_ptr); H5_DLL herr_t H5AC_load_cache_image_on_next_protect(H5F_t *f, haddr_t addr, hsize_t len, hbool_t rw); H5_DLL herr_t H5AC_validate_cache_image_config(H5AC_cache_image_config_t *config_ptr); +H5_DLL hbool_t H5AC_cache_image_pending(const H5F_t *f); +H5_DLL herr_t H5AC_force_cache_image_load(H5F_t * f, hid_t dxpl_id); /* Tag & Ring routines */ H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag); @@ -464,6 +466,7 @@ H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dx H5AC_ring_t *orig_ring); H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring); H5_DLL herr_t H5AC_unsettle_entry_ring(void *entry); +H5_DLL herr_t H5AC_unsettle_ring(H5F_t * f, H5AC_ring_t ring); H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag); @@ -732,7 +732,8 @@ herr_t H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id) { H5C_t * cache_ptr; - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t image_generated = FALSE; /* Whether a cache image was generated */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -756,9 +757,48 @@ H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id) HDassert(cache_ptr->pl_len == 0); /* Prepare cache image */ - if(H5C__prep_image_for_file_close(f, dxpl_id) < 0) + if(H5C__prep_image_for_file_close(f, dxpl_id, &image_generated) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create cache image") +#ifdef H5_HAVE_PARALLEL + if(!image_generated && cache_ptr->aux_ptr != NULL && f->shared->fs_persist) { + /* If persistent free space managers are enabled, flushing the + * metadata cache may result in the deletion, insertion, and/or + * dirtying of entries. + * + * This is a problem in PHDF5, as it breaks two invariants of + * our management of the metadata cache across all processes: + * + * 1) Entries will not be dirtied, deleted, inserted, or moved + * during flush in the parallel case. + * + * 2) All processes contain the same set of dirty metadata + * entries on entry to a sync point. + * + * To solve this problem for the persistent free space managers, + * serialize the metadata cache on all processes prior to the + * first sync point on file shutdown. The shutdown warning is + * a convenient location for this call. + * + * This is sufficient since: + * + * 1) FSM settle routines are only invoked on file close. Since + * serialization make the same settle calls as flush on file + * close, and since the close warning is issued after all + * non FSM related space allocations and just before the + * first sync point on close, this call will leave the caches + * in a consistant state across the processes if they were + * consistant before. + * + * 2) Since the FSM settle routines are only invoked once during + * file close, invoking them now will prevent their invocation + * during a flush, and thus avoid any resulting entrie dirties, + * deletions, insertion, or moves during the flush. + */ + if(H5C__serialize_cache(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTSERIALIZE, FAIL, "serialization of the cache failed") + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -3537,6 +3577,73 @@ done: /*------------------------------------------------------------------------- + * Function: H5C_unsettle_ring() + * + * Purpose: Advise the metadata cache that the specified free space + * manager ring is no longer settled (if it was on entry). + * + * If the target free space manager ring is already + * unsettled, do nothing, and return SUCCEED. + * + * If the target free space manager ring is settled, and + * we are not in the process of a file shutdown, mark + * the ring as unsettled, and return SUCCEED. + * + * If the target free space manager is settled, and we + * are in the process of a file shutdown, post an error + * message, and return FAIL. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/15/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring) +{ + H5C_t * cache_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + HDassert((H5C_RING_RDFSM == ring) || (H5C_RING_MDFSM == ring)); + cache_ptr = f->shared->cache; + HDassert(H5C__H5C_T_MAGIC == cache_ptr->magic); + + switch(ring) { + case H5C_RING_RDFSM: + if(cache_ptr->rdfsm_settled) { + if(cache_ptr->close_warning_received) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected rdfsm ring unsettle") + cache_ptr->rdfsm_settled = FALSE; + } /* end if */ + break; + + case H5C_RING_MDFSM: + if(cache_ptr->mdfsm_settled) { + if(cache_ptr->close_warning_received) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected mdfsm ring unsettle") + cache_ptr->mdfsm_settled = FALSE; + } /* end if */ + break; + + default: + HDassert(FALSE); /* this should be un-reachable */ + break; + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_unsettle_ring() */ + + +/*------------------------------------------------------------------------- * Function: H5C_validate_resize_config() * * Purpose: Run a sanity check on the specified sections of the @@ -6007,6 +6114,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, hbool_t write_entry; /* internal flag */ hbool_t destroy_entry; /* internal flag */ hbool_t generate_image; /* internal flag */ + hbool_t update_page_buffer; /* internal flag */ hbool_t was_dirty; hbool_t suppress_image_entry_writes = FALSE; hbool_t suppress_image_entry_frees = FALSE; @@ -6032,6 +6140,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, del_from_slist_on_destroy = ((flags & H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) != 0); during_flush = ((flags & H5C__DURING_FLUSH_FLAG) != 0); generate_image = ((flags & H5C__GENERATE_IMAGE_FLAG) != 0); + update_page_buffer = ((flags & H5C__UPDATE_PAGE_BUFFER_FLAG) != 0); /* Set the flag for destroying the entry, based on the 'take ownership' * and 'destroy' flags @@ -6447,6 +6556,19 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, } /* end else */ } /* if (destroy) */ + /* Check if we have to update the page buffer with cleared entries + * so it doesn't go out of date + */ + if(update_page_buffer) { + /* Sanity check */ + HDassert(!destroy); + HDassert(entry_ptr->image_ptr); + + if(f->shared->page_buf && f->shared->page_buf->page_size >= entry_ptr->size) + if(H5PB_update_entry(f->shared->page_buf, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) > 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Failed to update PB with metadata cache") + } /* end if */ + if(cache_ptr->log_flush) if((cache_ptr->log_flush)(cache_ptr, entry_addr, was_dirty, flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "log_flush callback failed") diff --git a/src/H5Cimage.c b/src/H5Cimage.c index 3a21137..fc58dac 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -147,6 +147,40 @@ H5FL_DEFINE(H5C_cache_entry_t); /*------------------------------------------------------------------------- + * + * Function: H5C_cache_image_pending() + * + * Purpose: Tests to see if the load of a metadata cache image + * load is pending (i.e. will be executed on the next + * protect or insert) + * + * Returns TRUE if a cache image load is pending, and FALSE + * if not. Throws an assertion failure on error. + * + * Return: TRUE if a cache image load is pending, and FALSE otherwise. + * + * Programmer: John Mainzer, 6/18/16 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5C_cache_image_pending(const H5C_t *cache_ptr) +{ + hbool_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + ret_value = (cache_ptr->load_image && !cache_ptr->image_loaded); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_cache_image_pending() */ + + +/*------------------------------------------------------------------------- * Function: H5C_cache_image_status() * * Purpose: Examine the metadata cache associated with the supplied @@ -832,6 +866,54 @@ H5C__free_image_entries_array(H5C_t * cache_ptr) /*------------------------------------------------------------------------- + * Function: H5C_force_cache_image_load() + * + * Purpose: On rare occasions, it is necessary to run + * H5MF_tidy_self_referential_fsm_hack() prior to the first + * metadata cache access. This is a problem as if there is a + * cache image at the end of the file, that routine will + * discard it. + * + * We solve this issue by calling this function, which will + * load the cache image and then call + * H5MF_tidy_self_referential_fsm_hack() to discard it. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 1/11/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_force_cache_image_load(H5F_t *f, hid_t dxpl_id) +{ + H5C_t *cache_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->load_image); + + /* Load the cache image, if requested */ + if(cache_ptr->load_image) { + cache_ptr->load_image = FALSE; + if(H5C__load_cache_image(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "can't load cache image") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_force_cache_image_load() */ + + +/*------------------------------------------------------------------------- * Function: H5C_get_cache_image_config * * Purpose: Copy the current configuration for cache image generation @@ -1239,7 +1321,7 @@ H5C__image_entry_cmp(const void *_entry1, const void *_entry2) *------------------------------------------------------------------------- */ herr_t -H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) +H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id, hbool_t *image_generated) { H5C_t * cache_ptr = NULL; haddr_t eoa_frag_addr = HADDR_UNDEF; @@ -1255,6 +1337,7 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) cache_ptr = f->shared->cache; HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(image_generated); /* If the file is opened and closed without any access to * any group or data set, it is possible that the cache image (if @@ -1379,6 +1462,15 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) (hsize_t)(cache_ptr->image_data_len), &eoa_frag_addr, &eoa_frag_size))) HGOTO_ERROR(H5E_CACHE, H5E_NOSPACE, FAIL, "can't allocate file space for metadata cache image") + /* Make note of the eoa after allocation of the cache image + * block. This value is used for sanity checking when we + * shutdown the self referential free space managers after + * we destroy the metadata cache. + */ + HDassert(HADDR_UNDEF == f->shared->eoa_post_mdci_fsalloc); + if(HADDR_UNDEF == (f->shared->eoa_post_mdci_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + /* For now, drop any fragment left over from the allocation of the * image block on the ground. A fragment should only be returned * if the underlying file alignment is greater than 1. @@ -1468,6 +1560,9 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) cache_ptr->image_ctl.generate_image = FALSE; } /* end else */ + + /* Indicate that a cache image was generated */ + *image_generated = TRUE; } /* end if */ done: diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index 1f43866..06ce714 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -791,7 +791,7 @@ H5C_mark_entries_as_clean(H5F_t * f, entries_cleared++; if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, - (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG)) < 0) + (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry") } /* end if */ else @@ -821,7 +821,7 @@ H5C_mark_entries_as_clean(H5F_t * f, progress = TRUE; if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, - (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG)) < 0) + (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry") } /* end if */ else @@ -1242,7 +1242,8 @@ H5C__flush_candidates_in_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned init_index_len; #endif /* H5C_DO_SANITY_CHECKS */ unsigned clear_flags = H5C__FLUSH_CLEAR_ONLY_FLAG | - H5C__GENERATE_IMAGE_FLAG; + H5C__GENERATE_IMAGE_FLAG | + H5C__UPDATE_PAGE_BUFFER_FLAG; unsigned flush_flags = H5C__NO_FLAGS_SET; unsigned op_flags; H5C_cache_entry_t *op_ptr; diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 321f1fb..69e8145 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -4901,7 +4901,8 @@ typedef int (*H5C_tag_iter_cb_t)(H5C_cache_entry_t *entry, void *ctx); /******************************/ /* Package Private Prototypes */ /******************************/ -H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id, + hbool_t *image_generated); H5_DLL herr_t H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t * cache_ptr, H5C_cache_entry_t** entry_ptr_ptr, const H5C_class_t * type, haddr_t addr, void * udata); diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 3408839..539dece 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -185,6 +185,7 @@ * H5C__TAKE_OWNERSHIP_FLAG * H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG * H5C__GENERATE_IMAGE_FLAG + * H5C__UPDATE_PAGE_BUFFER_FLAG */ #define H5C__NO_FLAGS_SET 0x00000 #define H5C__SET_FLUSH_MARKER_FLAG 0x00001 @@ -205,6 +206,7 @@ #define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x08000 #define H5C__DURING_FLUSH_FLAG 0x10000 /* Set when the entire cache is being flushed */ #define H5C__GENERATE_IMAGE_FLAG 0x20000 /* Set during parallel I/O */ +#define H5C__UPDATE_PAGE_BUFFER_FLAG 0x40000 /* Set during parallel I/O */ /* Debugging/sanity checking/statistics settings */ #ifndef NDEBUG @@ -2249,6 +2251,7 @@ H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, unsigned flags); H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); +H5_DLL herr_t H5C_force_cache_image_load(H5F_t * f, hid_t dxpl_id); H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global); H5_DLL herr_t H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); H5_DLL herr_t H5C_get_tag(const void *thing, /*OUT*/ haddr_t *tag); @@ -2313,9 +2316,11 @@ H5_DLL herr_t H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked); H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring); H5_DLL herr_t H5C_unsettle_entry_ring(void *thing); +H5_DLL herr_t H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring); H5_DLL herr_t H5C_remove_entry(void *thing); H5_DLL herr_t H5C_cache_image_status(H5F_t * f, hbool_t *load_ci_ptr, hbool_t *write_ci_ptr); +H5_DLL hbool_t H5C_cache_image_pending(const H5C_t *cache_ptr); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id, @@ -1975,7 +1975,9 @@ H5Fformat_convert(hid_t fid) /* Check for persistent freespace manager, which needs to be downgraded */ if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF && - f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF)) { + f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF && + f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF && + f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF)) { /* Check to remove free-space manager info message from superblock extension */ if(H5F_addr_defined(f->shared->sblock->ext_addr)) if(H5F_super_ext_remove_msg(f, H5AC_ind_read_dxpl_id, H5O_FSINFO_ID) < 0) @@ -1987,7 +1989,9 @@ H5Fformat_convert(hid_t fid) /* Set non-persistent freespace manager */ f->shared->fs_strategy = H5F_FILE_SPACE_STRATEGY_DEF; + f->shared->fs_persist = H5F_FREE_SPACE_PERSIST_DEF; f->shared->fs_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; + f->shared->fs_page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF; /* Indicate that the superblock should be marked dirty */ mark_dirty = TRUE; @@ -2006,3 +2010,79 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Fformat_convert() */ + +/*------------------------------------------------------------------------- + * Function: H5Freset_page_buffering_stats + * + * Purpose: Resets statistics for the page buffer layer. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5Freset_page_buffering_stats(hid_t file_id) +{ + H5F_t *file; /* File to reset stats on */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", file_id); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object(file_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier") + if(NULL == file->shared->page_buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file") + + /* Reset the statistics */ + if(H5PB_reset_stats(file->shared->page_buf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't reset stats for page buffering") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Freset_page_buffering_stats() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fget_page_buffering_stats + * + * Purpose: Retrieves statistics for the page buffer layer. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], unsigned hits[2], + unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]) +{ + H5F_t *file; /* File object for file ID */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "i*Iu*Iu*Iu*Iu*Iu", file_id, accesses, hits, misses, evictions, + bypasses); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") + if(NULL == file->shared->page_buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file") + if(NULL == accesses || NULL == hits || NULL == misses || NULL == evictions || NULL == bypasses) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL input parameters for stats") + + /* Get the statistics */ + if(H5PB_get_stats(file->shared->page_buf, accesses, hits, misses, evictions, bypasses) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve stats for page buffering") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fget_page_buffering_stats() */ + @@ -2054,3 +2054,27 @@ H5FD_get_base_addr(const H5FD_t *file) FUNC_LEAVE_NOAPI(file->base_addr) } /* end H5FD_get_base_addr() */ + +/*-------------------------------------------------------------------------- + * Function: H5FD_set_paged_aggr + * + * Purpose: Set "paged_aggr" for the file. + * + * Return: Non-negative if succeed; negative if fails. + * + * Programmer: Vailin Choi; April 2013 + * + *-------------------------------------------------------------------------- + */ +herr_t +H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(file); + + /* Indicate whether paged aggregation for handling file space is enabled or not */ + file->paged_aggr = paged; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_set_paged_aggr() */ diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 03228d2..f320946 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -934,13 +934,7 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsi /* Compute the address for the block to allocate */ addr = file->eoa; - /* Check if we need to align this block */ - if(size >= file->pub.threshold) { - /* Check for an already aligned block */ - if(addr % file->pub.alignment != 0) - addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment; - } /* end if */ - + /* Extend the end-of-allocated space address */ file->eoa = addr + size; /* Retain the (first) flavor of the information written to the file */ diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 7e12869..0ab7fc5 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1194,6 +1194,8 @@ H5FD_multi_query(const H5FD_t *_f, unsigned long *flags /* out */) *flags = 0; *flags |= H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */ *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ + *flags |= H5FD_FEAT_USE_ALLOC_SIZE; /* OK just pass the allocation size to the alloc callback */ + *flags |= H5FD_FEAT_PAGED_AGGR; /* OK special file space mapping for paged aggregation */ } /* end if */ return(0); @@ -1538,6 +1540,14 @@ H5FD_multi_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) mmt = file->fa.memb_map[type]; if (H5FD_MEM_DEFAULT==mmt) mmt = type; + /* XXX: NEED to work on this again */ + if(file->pub.paged_aggr) { + ALL_MEMBERS(mt) { + if(file->memb[mt]) + file->memb[mt]->paged_aggr = file->pub.paged_aggr; + } END_MEMBERS; + } + if (HADDR_UNDEF==(addr=H5FDalloc(file->memb[mmt], mmt, dxpl_id, size))) H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "member file can't alloc", HADDR_UNDEF) addr += file->fa.memb_addr[mmt]; diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 0e2928d..c64ec30 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -154,8 +154,8 @@ H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); H5_DLL herr_t H5FD_close(H5FD_t *file); H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2); -H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f, - hsize_t size, haddr_t *align_addr, hsize_t *align_size); +H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, + struct H5F_t *f, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size); H5_DLL herr_t H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f, haddr_t addr, hsize_t size); H5_DLL htri_t H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f, @@ -179,6 +179,7 @@ H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum); H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle); H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); +H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); /* Function prototypes for MPI based VFDs*/ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 436be10..883b28d 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -239,6 +239,19 @@ typedef enum H5F_mem_t H5FD_mem_t; * driver supports the single-writer/multiple-readers I/O pattern. */ #define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000 + /* + * Defining H5FD_FEAT_USE_ALLOC_SIZE for a VFL driver + * means that the library will just pass the allocation size to the + * the driver's allocation callback which will eventually handle alignment. + * This is specifically used for the multi/split driver. + */ +#define H5FD_FEAT_USE_ALLOC_SIZE 0x00002000 + /* + * Defining H5FD_FEAT_PAGED_AGGR for a VFL driver + * means that the driver needs special file space mapping for paged aggregation. + * This is specifically used for the multi/split driver. + */ +#define H5FD_FEAT_PAGED_AGGR 0x00004000 /* Forward declaration */ typedef struct H5FD_t H5FD_t; @@ -307,6 +320,7 @@ struct H5FD_t { /* Space allocation management fields */ hsize_t threshold; /* Threshold for alignment */ hsize_t alignment; /* Allocation alignment */ + hbool_t paged_aggr; /* Paged aggregation for file space is enabled or not */ }; /* Define enum for the source of file image callbacks */ diff --git a/src/H5FDspace.c b/src/H5FDspace.c index fcddecf..0ad3cf0 100644 --- a/src/H5FDspace.c +++ b/src/H5FDspace.c @@ -99,7 +99,7 @@ H5FL_DEFINE(H5FD_free_t); *------------------------------------------------------------------------- */ static haddr_t -H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +H5FD_extend(H5FD_t *file, H5FD_mem_t type, hsize_t size) { hsize_t orig_size = size; /* Original allocation size */ haddr_t eoa; /* Address of end-of-allocated space */ @@ -117,40 +117,18 @@ H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, hadd /* Get current end-of-allocated space address */ eoa = file->cls->get_eoa(file, type); - /* Compute extra space to allocate, if this is a new block and should be aligned */ - extra = 0; - if(new_block && file->alignment > 1 && orig_size >= file->threshold) { - hsize_t mis_align; /* Amount EOA is misaligned */ - - /* Check for EOA already aligned */ - if((mis_align = (eoa % file->alignment)) > 0) { - extra = file->alignment - mis_align; - if(frag_addr) - *frag_addr = eoa - file->base_addr; /* adjust for file's base address */ - if(frag_size) - *frag_size = extra; - } /* end if */ - } /* end if */ - - /* Add in extra allocation amount */ - size += extra; - /* Check for overflow when extending */ if(H5F_addr_overflow(eoa, size) || (eoa + size) > file->maxaddr) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed") - /* Set the [possibly aligned] address to return */ - ret_value = eoa + extra; + /* Set the [NOT aligned] address to return */ + ret_value = eoa; /* Extend the end-of-allocated space address */ eoa += size; if(file->cls->set_eoa(file, type, eoa) < 0) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed") - /* Post-condition sanity check */ - if(new_block && file->alignment && orig_size >= file->threshold) - HDassert(!(ret_value % file->alignment)); - done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_extend() */ @@ -160,6 +138,8 @@ done: * Function: H5FD_alloc_real * * Purpose: Allocate space in the file with the VFD + * Note: the handling of alignment is moved up from each driver to + * this routine. * * Return: Success: The format address of the new file memory. * Failure: The undefined address HADDR_UNDEF @@ -170,8 +150,14 @@ done: *------------------------------------------------------------------------- */ haddr_t -H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size, + haddr_t *frag_addr, hsize_t *frag_size) { + hsize_t orig_size = size; /* Original allocation size */ + haddr_t eoa; /* Address of end-of-allocated space */ + hsize_t extra; /* Extra space to allocate, to align request */ + unsigned long flags = 0; /* Driver feature flags */ + hbool_t use_alloc_size; /* Just pass alloc size to the driver */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ FUNC_ENTER_NOAPI(HADDR_UNDEF) @@ -185,16 +171,53 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); HDassert(size > 0); + /* Check for query driver and call it */ + if(file->cls->query) + (file->cls->query)(file, &flags); + + /* Check for the driver feature flag */ + use_alloc_size = flags & H5FD_FEAT_USE_ALLOC_SIZE; + + /* Get current end-of-allocated space address */ + eoa = file->cls->get_eoa(file, type); + + /* Compute extra space to allocate, if this should be aligned */ + extra = 0; + if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold) { + hsize_t mis_align; /* Amount EOA is misaligned */ + + /* Check for EOA already aligned */ + if((mis_align = (eoa % file->alignment)) > 0) { + extra = file->alignment - mis_align; + if(frag_addr) + *frag_addr = eoa - file->base_addr; /* adjust for file's base address */ + if(frag_size) + *frag_size = extra; + } /* end if */ + } /* end if */ + /* Dispatch to driver `alloc' callback or extend the end-of-address marker */ + /* For the multi/split driver: the size passed down to the alloc callback is the original size from H5FD_alloc() */ + /* For all other drivers: the size passed down to the alloc callback is the size + [possibly] alignment size */ if(file->cls->alloc) { - if((ret_value = (file->cls->alloc)(file, type, dxpl_id, size)) == HADDR_UNDEF) + ret_value = (file->cls->alloc)(file, type, dxpl_id, use_alloc_size ? size : size + extra); + if(!H5F_addr_defined(ret_value)) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver allocation request failed") } /* end if */ else { - if((ret_value = H5FD_extend(file, type, TRUE, size, frag_addr, frag_size)) == HADDR_UNDEF) + ret_value = H5FD_extend(file, type, size + extra); + if(!H5F_addr_defined(ret_value)) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver eoa update request failed") } /* end else */ + /* Set the [possibly aligned] address to return */ + if(!use_alloc_size) + ret_value += extra; + + /* Post-condition sanity check */ + if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold) + HDassert(!(ret_value % file->alignment)); + /* Convert absolute file offset to relative address */ ret_value -= file->base_addr; @@ -420,7 +443,7 @@ H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, hid_t dxpl_id, /* Check if the block is exactly at the end of the file */ if(H5F_addr_eq(blk_end, eoa)) { /* Extend the object by extending the underlying file */ - if(HADDR_UNDEF == H5FD_extend(file, type, FALSE, extra_requested, NULL, NULL)) + if(HADDR_UNDEF == H5FD_extend(file, type, extra_requested)) HGOTO_ERROR(H5E_VFL, H5E_CANTEXTEND, FAIL, "driver extend request failed") /* Mark EOA info dirty in cache, so change will get encoded */ diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 4c62bcc..13f728e 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -601,13 +601,6 @@ H5FD_stdio_alloc(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxp /* Compute the address for the block to allocate */ addr = file->eoa; - /* Check if we need to align this block */ - if(size >= file->pub.threshold) { - /* Check for an already aligned block */ - if((addr % file->pub.alignment) != 0) - addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment; - } /* end if */ - file->eoa = addr + size; return addr; @@ -135,7 +135,7 @@ HDfprintf(stderr, "%s: Creating free space manager, nclasses = %Zu\n", FUNC, ncl fspace->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0; fspace->alignment = alignment; - fspace->threshold = threshold; + fspace->align_thres = threshold; /* Check if the free space tracker is supposed to be persistant */ if(fs_addr) { @@ -227,7 +227,7 @@ HDfprintf(stderr, "%s: fspace->rc = %u\n", FUNC, fspace->rc); HGOTO_ERROR(H5E_FSPACE, H5E_CANTINC, NULL, "unable to increment ref. count on free space header") fspace->alignment = alignment; - fspace->threshold = threshold; + fspace->align_thres = threshold; /* Unlock free space header */ if(H5AC_unprotect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, fspace, H5AC__NO_FLAGS_SET) < 0) @@ -873,10 +873,7 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) HDassert(fspace); if(!H5F_addr_defined(fspace->sect_addr) && fspace->sinfo && fspace->serial_sect_count > 0) { - /* Allocate space for section info from aggregator/vfd (or temp. address space) */ - /* (The original version called H5MF_alloc(), but that may cause sect_size to change again) */ - /* (This routine is only called during file close operations, so don't allocate from temp. address space) */ - if(HADDR_UNDEF == (fspace->sect_addr = H5MF_aggr_vfd_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size))) + if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size))) HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info") fspace->alloc_sect_size = fspace->sect_size; @@ -901,169 +898,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5FS_alloc_vfd_alloc_hdr_and_section_info - * - * Purpose: This function is part of a hack to patch over a design - * flaw in the free space managers for file space allocation. - * Specifically, if a free space manager allocates space for - * its own section info, it is possible for it to - * go into an infinite loop as it: - * - * 1) computes the size of the section info - * - * 2) allocates file space for the section info - * - * 3) notices that the size of the section info - * has changed - * - * 4) deallocates the section info file space and - * returns to 1) above. - * - * Similarly, if it allocates space for its own header, it - * can go into an infinte loop as it: - * - * 1) allocates space for the header - * - * 2) notices that the free space manager is empty - * and thus should not have file space allocated - * to its header - * - * 3) frees the space allocated to the header - * - * 4) notices that the free space manager is not - * empty and thus must have file space allocated - * to it, and thus returns to 1) above. - * - * In a nutshell, the solution applied in this hack is to - * deallocate file space for the free space manager(s) that - * handle FSM header and/or section info file space allocations, - * wait until all other allocation/deallocation requests have - * been handled, and then test to see if the free space manager(s) - * in question are empty. If they are, do nothing. If they - * are not, allocate space for them at end of file bypassing the - * usual file space allocation calls, and thus avoiding the - * potential infinite loops. - * - * The purpose of this function is to allocate file space for - * the header and section info of the target free space manager - * directly from the VFD if needed. In this case the function - * also re-inserts the header and section info in the metadata - * cache with this allocation. - * - * Note that if f->shared->alignment > 1, and EOA is not a - * multiple of the alignment, it is possible that performing - * this allocation may generate a fragment of file space in - * addition to the space allocated for the section info. - * - * At present we deal with this by screaming and dying. - * Obviously, this is not acceptatable, but it should do - * for now. - * - * - * Return: Success: non-negative - * Failure: negative - * - * Programmer: John Mainzer - * 6/6/16 - * - *------------------------------------------------------------------------- - */ -herr_t -H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, - H5FS_t *fspace, haddr_t *fs_addr_ptr) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) - - /* Check arguments */ - HDassert(f); - HDassert(fspace); - HDassert(fs_addr_ptr); - - /* The section info should be unlocked */ - HDassert(fspace->sinfo_lock_count == 0); - - /* No space should be allocated */ - HDassert(*fs_addr_ptr == HADDR_UNDEF); - HDassert(fspace->addr == HADDR_UNDEF); - HDassert(fspace->sect_addr == HADDR_UNDEF); - HDassert(fspace->alloc_sect_size == 0); - - /* Check if any space will be needed */ - if(fspace->serial_sect_count > 0) { - haddr_t sect_addr; /* Address of sinfo */ - - /* The section info is floating, so fspace->sinfo should be defined */ - HDassert(fspace->sinfo); - - /* Start by allocating file space for the header */ - if(HADDR_UNDEF == (fspace->addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_HDR, - (hsize_t)H5FS_HEADER_SIZE(f), FALSE))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr") - - /* Cache the new free space header (pinned) */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space header to cache") - - - /* Now allocate file space for the section info */ - if(HADDR_UNDEF == (sect_addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_SINFO, - fspace->sect_size, FALSE))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space") - - /* Update fspace->alloc_sect_size and fspace->sect_addr to reflect - * the allocation - */ - fspace->alloc_sect_size = fspace->sect_size; - fspace->sect_addr = sect_addr; - - /* We have changed the sinfo address -- Mark free space header dirty */ - if(H5AC_mark_entry_dirty(fspace) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") - - /* Insert the new section info into the metadata cache. */ - - /* Question: Do we need to worry about this insertion causing an - * eviction from the metadata cache? Think on this. If so, add a - * flag to H5AC_insert() to force it to skip the call to make space in - * cache. - * - * On reflection, no. - * - * On a regular file close, any eviction will not change the - * the contents of the free space manger(s), as all entries - * should have correct file space allocated by the time this - * function is called. - * - * In the cache image case, the selection of entries for inclusion - * in the cache image will not take place until after this call. - * (Recall that this call is made during the metadata fsm settle - * routine, which is called during the serialization routine in - * the cache image case. Entries are not selected for inclusion - * in the image until after the cache is serialized.) - * - * JRM -- 11/4/16 - */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space sinfo to cache") - - /* Since space has been allocated for the section info and the sinfo - * has been inserted into the cache, relinquish owership (i.e. float) - * the section info. - */ - fspace->sinfo = NULL; - - /* Set the address of the free space header, on success */ - *fs_addr_ptr = fspace->addr; - } /* end if */ - -done: - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5FS_alloc_vfd_alloc_hdr_and_section_info() */ - - -/*------------------------------------------------------------------------- * Function: H5FS_free() * * Purpose: Free space for free-space manager header and section info header @@ -1076,7 +910,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) +H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id, hbool_t free_file_space) { haddr_t saved_addr; /* Previous address of item */ unsigned cache_flags; /* Flags for unprotecting cache entries */ @@ -1123,7 +957,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) /* Free space for the free-space manager section info */ if(!H5F_IS_TMP_ADDR(f, saved_addr)) { - if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0) + if(free_file_space && + H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections") } /* end if */ @@ -1165,7 +1000,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) fspace->addr = HADDR_UNDEF; /* Free space for the free-space manager header */ - if(H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0) + if(free_file_space && + H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space header") } /* end if */ @@ -1339,6 +1175,23 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FS_sinfo_dest() */ +herr_t +H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(frsp); + HDassert(tot_sect_count); + + /* Report statistics for free space */ + *tot_sect_count = frsp->serial_sect_count; + + FUNC_LEAVE_NOAPI(ret_value) +} + #ifdef H5FS_DEBUG_ASSERT /*------------------------------------------------------------------------- diff --git a/src/H5FScache.c b/src/H5FScache.c index dcc8122..ddc66fd 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -37,6 +37,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5ACprivate.h" /* Metadata cache */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -296,7 +297,7 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* # of section classes */ /* (only check if we actually have some classes) */ UINT16DECODE(image, nclasses); - if(fspace->nclasses > 0 && fspace->nclasses != nclasses) + if(fspace->nclasses > 0 && nclasses > fspace->nclasses) HGOTO_ERROR(H5E_FSPACE, H5E_CANTLOAD, NULL, "section class count mismatch") /* Shrink percent */ @@ -596,9 +597,11 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, * real file space lest the header be written to file with * a nonsense section info address. */ - HDassert(fspace->serial_sect_count > 0); - HDassert(fspace->sect_size > 0); - HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size); + if(!H5F_POINT_OF_NO_RETURN(f)) { + HDassert(fspace->serial_sect_count > 0); + HDassert(fspace->sect_size > 0); + HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size); + } /* end if */ if(H5F_IS_TMP_ADDR(f, fspace->sect_addr)) { unsigned sect_status = 0; @@ -702,10 +705,12 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len, * The following asserts are a cursory check on this. */ HDassert((! H5F_addr_defined(fspace->sect_addr)) || (! H5F_IS_TMP_ADDR(f, fspace->sect_addr))); - HDassert((! H5F_addr_defined(fspace->sect_addr)) || - ((fspace->serial_sect_count > 0) && - (fspace->sect_size > 0) && - (fspace->alloc_sect_size == (size_t)fspace->sect_size))); + + if(!H5F_POINT_OF_NO_RETURN(f)) + HDassert((! H5F_addr_defined(fspace->sect_addr)) || + ((fspace->serial_sect_count > 0) && + (fspace->sect_size > 0) && + (fspace->alloc_sect_size == (size_t)fspace->sect_size))); /* Magic number */ HDmemcpy(image, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC); @@ -1061,7 +1066,7 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, /* Insert section in free space manager, unless requested not to */ if(!(des_flags & H5FS_DESERIALIZE_NO_ADD)) - if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, NULL) < 0) + if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, udata) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, NULL, "can't add section to free space manager") } /* end for */ } while(image < (((const uint8_t *)_image + old_sect_size) - H5FS_SIZEOF_CHKSUM)); @@ -1177,8 +1182,9 @@ H5FS__cache_sinfo_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, HDassert(new_len); HDassert(flags); - /* we shouldn't be called if the section info is empty */ - HDassert(fspace->serial_sect_count > 0); + /* we shouldn't be called if the section info is empty, unless we hit the point of no return. */ + if(!H5F_POINT_OF_NO_RETURN(f)) + HDassert(fspace->serial_sect_count > 0); sinfo_addr = addr; /* this will change if we relocate the section data */ diff --git a/src/H5FSpkg.h b/src/H5FSpkg.h index 4411236..ab97955 100644 --- a/src/H5FSpkg.h +++ b/src/H5FSpkg.h @@ -187,8 +187,9 @@ struct H5FS_t { /* must be either H5C__NO_FLAGS_SET (i.e r/w) */ /* or H5AC__READ_ONLY_FLAG (i.e. r/o). */ size_t max_cls_serial_size; /* Max. additional size of serialized form of section */ - hsize_t threshold; /* Threshold for alignment */ hsize_t alignment; /* Alignment */ + hsize_t align_thres; /* Threshold for alignment */ + /* Memory data structures (not stored directly) */ H5FS_section_class_t *sect_cls; /* Array of section classes for this free list */ diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h index 1816a6e..74f9dcb 100644 --- a/src/H5FSprivate.h +++ b/src/H5FSprivate.h @@ -67,6 +67,12 @@ * managed sections is in flux) */ +#define H5FS_PAGE_END_NO_ADD 0x08 /* For "small" page fs: + * Don't add section to free space: + * when the section is at page end and + * when the section size is <= "small" + */ + /* Flags for deserialize callback */ #define H5FS_DESERIALIZE_NO_ADD 0x01 /* Don't add section to free space * manager after it's deserialized @@ -98,11 +104,11 @@ typedef struct H5FS_section_class_t { herr_t (*term_cls)(struct H5FS_section_class_t *); /* Routine to terminate class-specific settings */ /* Object methods */ - herr_t (*add)(H5FS_section_info_t *, unsigned *, void *); /* Routine called when section is about to be added to manager */ + herr_t (*add)(H5FS_section_info_t **, unsigned *, void *); /* Routine called when section is about to be added to manager */ herr_t (*serialize)(const struct H5FS_section_class_t *, const H5FS_section_info_t *, uint8_t *); /* Routine to serialize a "live" section into a buffer */ H5FS_section_info_t *(*deserialize)(const struct H5FS_section_class_t *, hid_t dxpl_id, const uint8_t *, haddr_t, hsize_t, unsigned *); /* Routine to deserialize a buffer into a "live" section */ htri_t (*can_merge)(const H5FS_section_info_t *, const H5FS_section_info_t *, void *); /* Routine to determine if two nodes are mergable */ - herr_t (*merge)(H5FS_section_info_t *, H5FS_section_info_t *, void *); /* Routine to merge two nodes */ + herr_t (*merge)(H5FS_section_info_t **, H5FS_section_info_t *, void *); /* Routine to merge two nodes */ htri_t (*can_shrink)(const H5FS_section_info_t *, void *); /* Routine to determine if node can shrink container */ herr_t (*shrink)(H5FS_section_info_t **, void *); /* Routine to shrink container */ herr_t (*free)(H5FS_section_info_t *); /* Routine to free node */ @@ -182,9 +188,8 @@ H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr); H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace); H5_DLL herr_t H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id); H5_DLL herr_t H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id); -H5_DLL herr_t H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, - H5FS_t *fspace, haddr_t *fs_addr_ptr); -H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id); +H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id, + hbool_t free_file_space); /* Free space section routines */ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, @@ -192,7 +197,7 @@ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5_DLL htri_t H5FS_sect_try_merge(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *sect, unsigned flags, void *op_data); H5_DLL htri_t H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, - haddr_t addr, hsize_t size, hsize_t extra_requested); + haddr_t addr, hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data); H5_DLL herr_t H5FS_sect_remove(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *node); H5_DLL htri_t H5FS_sect_find(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, @@ -202,11 +207,18 @@ H5_DLL herr_t H5FS_sect_stats(const H5FS_t *fspace, hsize_t *tot_space, hsize_t *nsects); H5_DLL herr_t H5FS_sect_change_class(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *sect, uint16_t new_class); -H5_DLL htri_t H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data); -H5_DLL herr_t H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size); +H5_DLL htri_t H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, + void *op_data); /* Statistics routine */ H5_DLL herr_t H5FS_stat_info(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *stats); +H5_DLL herr_t H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count); + +/* free space manager settling routines */ +H5_DLL herr_t H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, + hid_t dxpl_id, + H5FS_t *fspace, + haddr_t *fs_addr_ptr); /* Debugging routines for dumping file structures */ H5_DLL herr_t H5FS_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, diff --git a/src/H5FSsection.c b/src/H5FSsection.c index efe0031..49ff003 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -25,6 +25,8 @@ /* Module Setup */ /****************/ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ + #include "H5FSmodule.h" /* This source code file is part of the H5FS module */ @@ -33,6 +35,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -1210,11 +1213,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data) HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures") /* Merge the two sections together */ - if((*tmp_sect_cls->merge)(tmp_sect, *sect, op_data) < 0) + if((*tmp_sect_cls->merge)(&tmp_sect, *sect, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections") /* Retarget section pointer to 'less than' node that was merged into */ *sect = tmp_sect; + if(*sect == NULL) + HGOTO_DONE(ret_value); /* Indicate successful merge occurred */ modified = TRUE; @@ -1254,9 +1259,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data) HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures") /* Merge the two sections together */ - if((*sect_cls->merge)(*sect, tmp_sect, op_data) < 0) + if((*sect_cls->merge)(sect, tmp_sect, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections") + /* It's possible that the merge caused the section to be deleted (particularly in the paged allocation case) */ + if(*sect == NULL) + HGOTO_DONE(ret_value); + /* Indicate successful merge occurred */ modified = TRUE; } /* end if */ @@ -1383,7 +1392,7 @@ HDfprintf(stderr, "%s: *sect = {%a, %Hu, %u, %s}\n", FUNC, sect->addr, sect->siz /* Call "add" section class callback, if there is one */ cls = &fspace->sect_cls[sect->type]; if(cls->add) { - if((*cls->add)(sect, &flags, op_data) < 0) + if((*cls->add)(§, &flags, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed") } /* end if */ @@ -1411,7 +1420,7 @@ HDfprintf(stderr, "%s: fspace->tot_space = %Hu\n", FUNC, fspace->tot_space); #endif /* H5FS_SINFO_DEBUG */ /* Mark free space sections as changed */ /* (if adding sections while deserializing sections, don't set the flag) */ - if(!(flags & H5FS_ADD_DESERIALIZING)) + if(!(flags & (H5FS_ADD_DESERIALIZING | H5FS_PAGE_END_NO_ADD))) sinfo_modified = TRUE; done: @@ -1444,7 +1453,7 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); */ htri_t H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, haddr_t addr, - hsize_t size, hsize_t extra_requested) + hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data) { hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */ hbool_t sinfo_modified = FALSE; /* Whether the section info was modified */ @@ -1528,10 +1537,16 @@ if(_section_) /* Adjust section by amount requested */ sect->addr += extra_requested; sect->size -= extra_requested; - - /* Re-add adjusted section to free sections data structures */ - if(H5FS_sect_link(fspace, sect, 0) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list") + if(cls->add) + if((*cls->add)(§, &flags, op_data) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed") + + /* Re-adding the section could cause it to disappear (particularly when paging) */ + if(sect) { + /* Re-add adjusted section to free sections data structures */ + if(H5FS_sect_link(fspace, sect, 0) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list") + } /* end if */ } /* end if */ else { /* Sanity check */ @@ -1667,7 +1682,7 @@ HDfprintf(stderr, "%s: fspace->sinfo->nbins = %u\n", FUNC, fspace->sinfo->nbins) HDfprintf(stderr, "%s: bin = %u\n", FUNC, bin); #endif /* QAK */ alignment = fspace->alignment; - if(!((alignment > 1) && (request >= fspace->threshold))) + if(!((alignment > 1) && (request >= fspace->align_thres))) alignment = 0; /* no alignment */ do { @@ -2349,7 +2364,7 @@ HDfprintf(stderr, "%s: sect->size = %Hu, sect->addr = %a, sect->type = %u\n", "H *------------------------------------------------------------------------- */ htri_t -H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data) +H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, void *op_data) { hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */ hbool_t section_removed = FALSE; /* Whether a section was removed */ @@ -2406,42 +2421,264 @@ done: /*------------------------------------------------------------------------- - * Function: H5FS_sect_query_last_sect - * - * Purpose: Retrieve info about the last section on the merge list - * - * Return: Success: non-negative - * Failure: negative + * Function: H5FS_vfd_alloc_hdr_and_section_info_if_needed + * + * Purpose: This function is part of a hack to patch over a design + * flaw in the free space managers for file space allocation. + * Specifically, if a free space manager allocates space for + * its own section info, it is possible for it to + * go into an infinite loop as it: + * + * 1) computes the size of the section info + * + * 2) allocates file space for the section info + * + * 3) notices that the size of the section info + * has changed + * + * 4) deallocates the section info file space and + * returns to 1) above. + * + * Similarly, if it allocates space for its own header, it + * can go into an infinte loop as it: + * + * 1) allocates space for the header + * + * 2) notices that the free space manager is empty + * and thus should not have file space allocated + * to its header + * + * 3) frees the space allocated to the header + * + * 4) notices that the free space manager is not + * empty and thus must have file space allocated + * to it, and thus returns to 1) above. + * + * In a nutshell, the solution applied in this hack is to + * deallocate file space for the free space manager(s) that + * handle FSM header and/or section info file space allocations, + * wait until all other allocation/deallocation requests have + * been handled, and then test to see if the free space manager(s) + * in question are empty. If they are, do nothing. If they + * are not, allocate space for them at end of file bypassing the + * usual file space allocation calls, and thus avoiding the + * potential infinite loops. + * + * The purpose of this function is to allocate file space for + * the header and section info of the target free space manager + * directly from the VFD if needed. In this case the function + * also re-inserts the header and section info in the metadata + * cache with this allocation. + * + * When paged allocation is not enabled, allocation of space + * for the free space manager header and section info is + * straight forward -- we simply allocate the space directly + * from file driver. + * + * Note that if f->shared->alignment > 1, and EOA is not a + * multiple of the alignment, it is possible that performing + * these allocations may generate a fragment of file space in + * addition to the space allocated for the section info. This + * excess space is dropped on the floor. As shall be seen, + * it will usually be reclaimed later. + * + * When page allocation is enabled, things are more difficult, + * as there is the possibility that page buffering will be + * enabled when the free space managers are read. To allow + * for this, we must ensure that space allocated for the + * free space manager header and section info is either larger + * than a page, or resides completely withing a page. + * + * Do this by allocating space for the free space header and + * section info starting at page boundaries, and extending + * allocation to the next page boundary. This of course wastes + * space, but see below. + * + * On the first free space allocation / deallocation after the + * next file open, we will read the self referential free space + * managers, float them and reduce the EOA to its value prior + * to allocation of file space for the self referential free + * space managers on the preceeding file close. This EOA value + * is stored in the free space manager superblock extension + * message. + * + * Return: Success: non-negative + * Failure: negative * - * Programmer: Vailin Choi + * Programmer: John Mainzer + * 6/6/16 * *------------------------------------------------------------------------- */ herr_t -H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size) +H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, hid_t dxpl_id, + H5FS_t *fspace, haddr_t *fs_addr_ptr) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + hsize_t hdr_alloc_size; + hsize_t sinfo_alloc_size; + haddr_t sect_addr = HADDR_UNDEF; /* address of sinfo */ + haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ + hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */ + haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT /* Check arguments. */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); HDassert(fspace); + HDassert(fs_addr_ptr); - if(fspace->sinfo && fspace->sinfo->merge_list) { - H5SL_node_t *last_node; /* Last node in merge list */ + /* the section info should be unlocked */ + HDassert(fspace->sinfo_lock_count == 0); - /* Check for last node in the merge list */ - if(NULL != (last_node = H5SL_last(fspace->sinfo->merge_list))) { - H5FS_section_info_t *tmp_sect; /* Temporary free space section */ + /* no space should be allocated */ + HDassert(*fs_addr_ptr == HADDR_UNDEF); + HDassert(fspace->addr == HADDR_UNDEF); + HDassert(fspace->sect_addr == HADDR_UNDEF); + HDassert(fspace->alloc_sect_size == 0); - /* Get the pointer to the last section, from the last node */ - tmp_sect = (H5FS_section_info_t *)H5SL_item(last_node); - HDassert(tmp_sect); - if(sect_addr) - *sect_addr = tmp_sect->addr; - if(sect_size) - *sect_size = tmp_sect->size; - } /* end if */ + /* persistant free space managers must be enabled */ + HDassert(f->shared->fs_persist); + + /* At present, all free space strategies enable the free space managers. + * This will probably change -- at which point this assertion should + * be revisited. + */ + /* Updated: Only the following two strategies enable the free-space managers */ + HDassert((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) || + (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE)); + + if(fspace->serial_sect_count > 0) { + /* the section info is floating, so space->sinfo should be defined */ + HDassert(fspace->sinfo); + + /* start by allocating file space for the header */ + + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") + + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space") + + hdr_alloc_size = H5FS_HEADER_SIZE(f); + + /* if page allocation is enabled, extend the hdr_alloc_size to the + * next page boundary. + */ + if(H5F_PAGED_AGGR(f)) { + HDassert(0 == (eoa % f->shared->fs_page_size)); + + hdr_alloc_size = ((hdr_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + + HDassert(hdr_alloc_size >= H5FS_HEADER_SIZE(f)); + HDassert((hdr_alloc_size % f->shared->fs_page_size) == 0); + } /* end if */ + + /* allocate space for the hdr */ + if(HADDR_UNDEF == (fspace->addr = H5FD_alloc(f->shared->lf, dxpl_id, + H5FD_MEM_FSPACE_HDR, f, + hdr_alloc_size, + &eoa_frag_addr, + &eoa_frag_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr") + + /* if the file alignement is 1, there should be no + * eoa fragment. Otherwise, drop any fragment on the floor. + */ + HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + + /* Cache the new free space header (pinned) */ + if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache") + + *fs_addr_ptr = fspace->addr; + + /* now allocate file space for the section info */ + + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa") + + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space") + + sinfo_alloc_size = fspace->sect_size; + + /* if paged allocation is enabled, extend the sinfo_alloc_size to the + * next page boundary. + */ + if(H5F_PAGED_AGGR(f)) { + HDassert(0 == (eoa % f->shared->fs_page_size)); + + sinfo_alloc_size = ((sinfo_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + + HDassert(sinfo_alloc_size >= fspace->sect_size); + HDassert((sinfo_alloc_size % f->shared->fs_page_size) == 0); + } /* end if */ + + /* allocate space for the section info */ + if(HADDR_UNDEF == (sect_addr = H5FD_alloc(f->shared->lf, dxpl_id, + H5FD_MEM_FSPACE_SINFO, f, + sinfo_alloc_size, + &eoa_frag_addr, + &eoa_frag_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space") + + /* if the file alignement is 1, there should be no + * eoa fragment. Otherwise, drop the fragment on the floor. + */ + HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + + /* update fspace->alloc_sect_size and fspace->sect_addr to reflect + * the allocation + */ + fspace->alloc_sect_size = fspace->sect_size; + fspace->sect_addr = sect_addr; + + /* insert the new section info into the metadata cache. */ + + /* Question: Do we need to worry about this insertion causing an + * eviction from the metadata cache? Think on this. If so, add a + * flag to H5AC_insert() to force it to skip the call to make space in + * cache. + * + * On reflection, no. + * + * On a regular file close, any eviction will not change the + * the contents of the free space manger(s), as all entries + * should have correct file space allocated by the time this + * function is called. + * + * In the cache image case, the selection of entries for inclusion + * in the cache image will not take place until after this call. + * (Recall that this call is made during the metadata fsm settle + * routine, which is called during the serialization routine in + * the cache image case. Entries are not selected for inclusion + * in the image until after the cache is serialized.) + * + * JRM -- 11/4/16 + */ + if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache") + + /* We have changed the sinfo address -- Mark free space header dirty */ + if(H5AC_mark_entry_dirty(fspace) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") + + /* since space has been allocated for the section info and the sinfo + * has been inserted into the cache, relinquish owership (i.e. float) + * the section info. + */ + fspace->sinfo = NULL; } /* end if */ - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FS_sect_query_last_sect() */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FS_vfd_alloc_hdr_and_section_info_if_needed() */ diff --git a/src/H5Fint.c b/src/H5Fint.c index 3c98c8f..794be50 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -182,6 +182,14 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) efc_size = H5F_efc_max_nfiles(f->shared->efc); if(H5P_set(new_plist, H5F_ACS_EFC_SIZE_NAME, &efc_size) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set elink file cache size") + if(f->shared->page_buf != NULL) { + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &(f->shared->page_buf->max_size)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set page buffer size") + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &(f->shared->page_buf->min_meta_perc)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum metadata fraction of page buffer") + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &(f->shared->page_buf->min_raw_perc)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum raw data fraction of page buffer") + } /* end if */ #ifdef H5_HAVE_PARALLEL if(H5P_set(new_plist, H5_COLL_MD_READ_FLAG_NAME, &(f->coll_md_read)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set collective metadata read flag") @@ -597,11 +605,26 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t f->shared->flags = flags; f->shared->sohm_addr = HADDR_UNDEF; f->shared->sohm_vers = HDF5_SHAREDHEADER_VERSION; - for(u = 0; u < NELMTS(f->shared->fs_addr); u++) - f->shared->fs_addr[u] = HADDR_UNDEF; f->shared->accum.loc = HADDR_UNDEF; f->shared->lf = lf; + /* Initialization for handling file space */ + for(u = 0; u < NELMTS(f->shared->fs_addr); u++) { + f->shared->fs_state[u] = H5F_FS_STATE_CLOSED; + f->shared->fs_addr[u] = HADDR_UNDEF; + f->shared->fs_man[u] = NULL; + } /* end for */ + f->shared->first_alloc_dealloc = FALSE; + f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + f->shared->eoa_post_fsm_fsalloc = HADDR_UNDEF; + f->shared->eoa_post_mdci_fsalloc = HADDR_UNDEF; + + /* Initialization for handling file space (for paged aggregation) */ + f->shared->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; + + /* intialize point of no return */ + f->shared->point_of_no_return = FALSE; + /* * Copy the file creation and file access property lists into the * new file handle. We do this early because some values might need @@ -621,8 +644,19 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t HDassert(f->shared->sohm_nindexes < 255); if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &f->shared->fs_strategy) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space strategy") + if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &f->shared->fs_persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space persisting status") if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &f->shared->fs_threshold) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get free-space section threshold") + if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &f->shared->fs_page_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space page size") + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + + /* Temporary for multi/split drivers: fail file creation + when persisting free-space or using paged aggregation strategy */ + if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE || f->shared->fs_persist) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't open with this strategy or persistent fs") /* Get the FAPL values to cache */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) @@ -972,6 +1006,11 @@ H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") + /* Shutdown the page buffer cache */ + if(H5PB_dest(f) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing page buffer cache") + /* Clean up the metadata cache log location string */ if(f->shared->mdc_log_location) f->shared->mdc_log_location = (char *)H5MM_xfree(f->shared->mdc_log_location); @@ -1142,6 +1181,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5P_genplist_t *a_plist; /*file access property list */ H5F_close_degree_t fc_degree; /*file close degree */ hid_t raw_dxpl_id = H5AC_rawdata_dxpl_id; /* Raw data dxpl used by library */ + size_t page_buf_size; + unsigned page_buf_min_meta_perc; + unsigned page_buf_min_raw_perc; hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ @@ -1292,6 +1334,25 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + /* Check if page buffering is enabled */ + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size") + if(page_buf_size) { +#ifdef H5_HAVE_PARALLEL + /* Collective metadata writes are not supported with page buffering */ + if(file->coll_md_write) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "collective metadata writes are not supported with page buffering") + + /* Temporary: fail file create when page buffering feature is enabled for parallel */ + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "page buffering is disabled for parallel") +#endif /* H5_HAVE_PARALLEL */ + /* Query for other page buffer cache properties */ + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &page_buf_min_meta_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum metadata fraction of page buffer") + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum raw data fraction of page buffer") + } /* end if */ + /* * Read or write the file superblock, depending on whether the file is * empty or not. @@ -1302,6 +1363,11 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, * to create & write the superblock. */ + /* Create the page buffer before initializing the superblock */ + if(page_buf_size) + if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") + /* Initialize information about the superblock and allocate space for it */ /* (Writes superblock extension messages, if there are any) */ if(H5F__super_init(file, meta_dxpl_id) < 0) @@ -1319,6 +1385,11 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, if(H5F__super_read(file, meta_dxpl_id, raw_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") + /* Create the page buffer before initializing the superblock */ + if(page_buf_size) + if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") + /* Open the root group */ if(H5G_mkroot(file, meta_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group") @@ -1527,6 +1598,11 @@ H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closi /* Push error, but keep going*/ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush metadata accumulator") + /* Flush the page buffer */ + if(H5PB_flush(&fio_info) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "page buffer flush failed") + /* Flush file buffers to disk. */ if(H5FD_flush(f->shared->lf, meta_dxpl_id, closing) < 0) /* Push error, but keep going*/ @@ -2652,6 +2728,38 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__set_eoa() */ + +/*------------------------------------------------------------------------- + * Function: H5F__set_paged_aggr + * + * Purpose: Quick and dirty routine to set the file's paged_aggr mode + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * June 19, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__set_paged_aggr(const H5F_t *f, hbool_t paged) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(f); + HDassert(f->shared); + + /* Dispatch to driver */ + if(H5FD_set_paged_aggr(f->shared->lf, paged) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "driver set paged aggr mode failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__set_paged_aggr() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Fio.c b/src/H5Fio.c index d40483f..6d23995 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -39,6 +39,7 @@ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ #include "H5Iprivate.h" /* IDs */ +#include "H5PBprivate.h" /* Page Buffer */ /****************/ @@ -129,9 +130,9 @@ H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") } /* end else */ - /* Pass through metadata accumulator layer */ - if(H5F__accum_read(&fio_info, map_type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through metadata accumulator failed") + /* Pass through page buffer layer */ + if(H5PB_read(&fio_info, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through page buffer failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -191,9 +192,9 @@ H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") } /* end else */ - /* Pass through metadata accumulator layer */ - if(H5F__accum_write(&fio_info, map_type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") + /* Pass through page buffer layer */ + if(H5PB_write(&fio_info, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through page buffer failed") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index aaa19d9..4dd9e20 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -42,6 +42,7 @@ #include "H5FSprivate.h" /* File free space */ #include "H5Gprivate.h" /* Groups */ #include "H5Oprivate.h" /* Object header messages */ +#include "H5PBprivate.h" /* Page buffer */ #include "H5UCprivate.h" /* Reference counted object functions */ @@ -68,8 +69,8 @@ /* Macro to abstract checking whether file is using a free space manager */ #define H5F_HAVE_FREE_SPACE_MANAGER(F) \ - ((F)->shared->fs_strategy == H5F_FILE_SPACE_ALL || \ - (F)->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) + ((F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || \ + (F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) /* Macros for encoding/decoding superblock */ #define H5F_MAX_DRVINFOBLOCK_SIZE 1024 /* Maximum size of superblock driver info buffer */ @@ -200,13 +201,6 @@ typedef struct H5F_meta_accum_t { hbool_t dirty; /* Flag to indicate that the accumulated metadata is dirty */ } H5F_meta_accum_t; -/* Enum for free space manager state */ -typedef enum H5F_fs_state_t { - H5F_FS_STATE_CLOSED, /* Free space manager is closed */ - H5F_FS_STATE_OPEN, /* Free space manager has been opened */ - H5F_FS_STATE_DELETING /* Free space manager is being deleted */ -} H5F_fs_state_t; - /* A record of the mount table */ typedef struct H5F_mount_t { struct H5G_t *group; /* Mount point group held open */ @@ -281,6 +275,7 @@ struct H5F_file_t { unsigned long feature_flags; /* VFL Driver feature Flags */ haddr_t maxaddr; /* Maximum address for file */ + H5PB_t *page_buf; /* The page buffer cache */ H5AC_t *cache; /* The object cache */ H5AC_cache_config_t mdc_initCacheCfg; /* initial configuration for the */ @@ -316,19 +311,38 @@ struct H5F_file_t { H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */ /* File space allocation information */ - H5F_file_space_type_t fs_strategy; /* File space handling strategy */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ hsize_t fs_threshold; /* Free space section threshold */ + hbool_t fs_persist; /* Free-space persist or not */ hbool_t use_tmp_space; /* Whether temp. file space allocation is allowed */ haddr_t tmp_addr; /* Next address to use for temp. space in the file */ + hbool_t point_of_no_return; /* flag to indicate that we can't go back and delete a freespace header when it's used up */ + + H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */ + haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */ + H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */ + hbool_t first_alloc_dealloc; /* TRUE iff free space managers */ + /* are persistant and have not */ + /* been used accessed for either */ + /* allocation or deallocation */ + /* since file open. */ + haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_mdci_fsalloc; /* eoa past file space allocation */ + /* for metadata cache image, or */ + /* HADDR_UNDEF if no cache image. */ + + /* Free-space aggregation info */ unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */ - H5F_fs_state_t fs_state[H5FD_MEM_NTYPES]; /* State of free space manager for each type */ - haddr_t fs_addr[H5FD_MEM_NTYPES]; /* Address of free space manager info for each type */ - H5FS_t *fs_man[H5FD_MEM_NTYPES]; /* Free space manager for each file space type */ - H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ - H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info */ - /* (if aggregating metadata allocations) */ - H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info */ - /* (if aggregating "small data" allocations) */ + H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ + H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */ + H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */ + + /* Paged aggregation info */ + hsize_t fs_page_size; /* File space page size */ + size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */ /* Metadata accumulator information */ H5F_meta_accum_t accum; /* Metadata accumulator info */ @@ -383,7 +397,7 @@ H5FL_EXTERN(H5F_file_t); /* General routines */ H5F_t *H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); -herr_t H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush); +H5_DLL herr_t H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush); H5_DLL herr_t H5F__flush(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing); H5_DLL htri_t H5F__is_hdf5(const char *name, hid_t meta_dxpl_id, hid_t raw_dxpl_id); H5_DLL herr_t H5F_get_objects(const H5F_t *f, unsigned types, size_t max_index, hid_t *obj_id_list, hbool_t app_ref, size_t *obj_id_count_ptr); @@ -433,9 +447,16 @@ H5_DLL herr_t H5F_efc_release(H5F_efc_t *efc); H5_DLL herr_t H5F_efc_destroy(H5F_efc_t *efc); H5_DLL herr_t H5F_efc_try_close(H5F_t *f); +/* Space allocation routines */ +H5_DLL haddr_t H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size); +H5_DLL herr_t H5F_free(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, haddr_t addr, hsize_t size); +H5_DLL htri_t H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, + haddr_t blk_end, hsize_t extra_requested); + /* Functions that get/retrieve values from VFD layer */ H5_DLL herr_t H5F__set_eoa(const H5F_t *f, H5F_mem_t type, haddr_t addr); H5_DLL herr_t H5F__set_base_addr(const H5F_t *f, haddr_t addr); +H5_DLL herr_t H5F__set_paged_aggr(const H5F_t *f, hbool_t paged); /* Functions that flush or evict */ H5_DLL herr_t H5F__evict_cache_entries(H5F_t *f, hid_t dxpl_id); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index f980347..886063a 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -315,8 +315,8 @@ #define H5F_SET_STORE_MSG_CRT_IDX(F, FL) ((F)->shared->store_msg_crt_idx = (FL)) #define H5F_GRP_BTREE_SHARED(F) ((F)->shared->grp_btree_shared) #define H5F_SET_GRP_BTREE_SHARED(F, RC) (((F)->shared->grp_btree_shared = (RC)) ? SUCCEED : FAIL) -#define H5F_USE_TMP_SPACE(F) ((F)->shared->use_tmp_space) -#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->tmp_addr, (ADDR))) +#define H5F_USE_TMP_SPACE(F) ((F)->shared->fs.use_tmp_space) +#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->fs.tmp_addr, (ADDR))) #define H5F_SET_LATEST_FLAGS(F, FL) ((F)->shared->latest_flags = (FL)) #ifdef H5_HAVE_PARALLEL #define H5F_COLL_MD_READ(F) ((F)->coll_md_read) @@ -324,6 +324,12 @@ #define H5F_USE_MDC_LOGGING(F) ((F)->shared->use_mdc_logging) #define H5F_START_MDC_LOG_ON_ACCESS(F) ((F)->shared->start_mdc_log_on_access) #define H5F_MDC_LOG_LOCATION(F) ((F)->shared->mdc_log_location) +#define H5F_ALIGNMENT(F) ((F)->shared->alignment) +#define H5F_THRESHOLD(F) ((F)->shared->threshold) +#define H5F_PGEND_META_THRES(F) ((F)->shared->fs.pgend_meta_thres) +#define H5F_POINT_OF_NO_RETURN(F) ((F)->shared->fs.point_of_no_return) +#define H5F_FIRST_ALLOC_DEALLOC(F) ((F)->shared->first_alloc_dealloc) +#define H5F_EOA_PRE_FSM_FSALLOC(F) ((F)->shared->eoa_pre_fsm_fsalloc) #else /* H5F_MODULE */ #define H5F_INTENT(F) (H5F_get_intent(F)) #define H5F_OPEN_NAME(F) (H5F_get_open_name(F)) @@ -375,6 +381,12 @@ #define H5F_USE_MDC_LOGGING(F) (H5F_use_mdc_logging(F)) #define H5F_START_MDC_LOG_ON_ACCESS(F) (H5F_start_mdc_log_on_access(F)) #define H5F_MDC_LOG_LOCATION(F) (H5F_mdc_log_location(F)) +#define H5F_ALIGNMENT(F) (H5F_get_alignment(F)) +#define H5F_THRESHOLD(F) (H5F_get_threshold(F)) +#define H5F_PGEND_META_THRES(F) (H5F_get_pgend_meta_thres(F)) +#define H5F_POINT_OF_NO_RETURN(F) (H5F_get_point_of_no_return(F)) +#define H5F_FIRST_ALLOC_DEALLOC(F) (H5F_get_first_alloc_dealloc(F)) +#define H5F_EOA_PRE_FSM_FSALLOC(F) (H5F_get_eoa_pre_fsm_fsalloc(F)) #endif /* H5F_MODULE */ @@ -448,7 +460,9 @@ #define H5F_CRT_SHMSG_LIST_MAX_NAME "shmsg_list_max" /* Shared message list maximum size */ #define H5F_CRT_SHMSG_BTREE_MIN_NAME "shmsg_btree_min" /* Shared message B-tree minimum size */ #define H5F_CRT_FILE_SPACE_STRATEGY_NAME "file_space_strategy" /* File space handling strategy */ +#define H5F_CRT_FREE_SPACE_PERSIST_NAME "free_space_persist" /* Free-space persisting status */ #define H5F_CRT_FREE_SPACE_THRESHOLD_NAME "free_space_threshold" /* Free space section threshold */ +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME "file_space_page_size" /* File space page size */ @@ -484,6 +498,9 @@ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ #define H5F_ACS_COLL_MD_WRITE_FLAG_NAME "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */ #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME "mdc_initCacheImageCfg" /* Initial metadata cache image creation configuration */ +#define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */ +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -522,10 +539,34 @@ /* See format specification on version 1 B-trees */ /* Default file space handling strategy */ -#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_ALL +#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR + +/* Default free space section threshold used by free-space managers */ +#define H5F_FREE_SPACE_PERSIST_DEF FALSE + /* Default free space section threshold used by free-space managers */ #define H5F_FREE_SPACE_THRESHOLD_DEF 1 +/* For paged aggregation: default file space page size when not set */ +#define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096 +/* For paged aggregation: minimum value for file space page size */ +#define H5F_FILE_SPACE_PAGE_SIZE_MIN 512 + +/* For paged aggregation: drop free-space with size <= this threshold for small meta section */ +#define H5F_FILE_SPACE_PGEND_META_THRES 10 + +/* Default for threshold for alignment (can be set via H5Pset_alignment()) */ +#define H5F_ALIGN_DEF 1 +/* Default for alignment (can be set via H5Pset_alignment()) */ +#define H5F_ALIGN_THRHD_DEF 1 +/* Default size for meta data aggregation block (can be set via H5Pset_meta_block_size()) */ +#define H5F_META_BLOCK_SIZE_DEF 2048 +/* Default size for small data aggregation block (can be set via H5Pset_small_data_block_size()) */ +#define H5F_SDATA_BLOCK_SIZE_DEF 2048 + +/* Check for file using paged aggregation */ +#define H5F_PAGED_AGGR(F) (F->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE && F->shared->fs_page_size) + /* Metadata read attempt values */ #define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */ #define H5F_SWMR_METADATA_READ_ATTEMPTS 100 /* Default # of read attempts for SWMR access */ @@ -648,6 +689,34 @@ typedef struct H5F_block_t { hsize_t length; /* Length of the block in the file */ } H5F_block_t; +/* Enum for free space manager state */ +typedef enum H5F_fs_state_t { + H5F_FS_STATE_CLOSED = 0, /* Free space manager is closed */ + H5F_FS_STATE_OPEN = 1, /* Free space manager has been opened */ + H5F_FS_STATE_DELETING = 2 /* Free space manager is being deleted */ +} H5F_fs_state_t; + +/* For paged aggregation */ +/* The values 0 to 6 is the same as H5F_mem_t */ +typedef enum H5F_mem_page_t { + H5F_MEM_PAGE_DEFAULT = 0, /* Not used */ + H5F_MEM_PAGE_SUPER = 1, + H5F_MEM_PAGE_BTREE = 2, + H5F_MEM_PAGE_DRAW = 3, + H5F_MEM_PAGE_GHEAP = 4, + H5F_MEM_PAGE_LHEAP = 5, + H5F_MEM_PAGE_OHDR = 6, + H5F_MEM_PAGE_LARGE_SUPER = 7, + H5F_MEM_PAGE_LARGE_BTREE = 8, + H5F_MEM_PAGE_LARGE_DRAW = 9, + H5F_MEM_PAGE_LARGE_GHEAP = 10, + H5F_MEM_PAGE_LARGE_LHEAP = 11, + H5F_MEM_PAGE_LARGE_OHDR = 12, + H5F_MEM_PAGE_NTYPES = 13 /* Sentinel value - must be last */ +} H5F_mem_page_t; + +#define H5F_MEM_PAGE_META H5F_MEM_PAGE_SUPER /* Small-sized meta data */ +#define H5F_MEM_PAGE_GENERIC H5F_MEM_PAGE_LARGE_SUPER /* Large-sized generic: meta and raw */ /*****************************/ /* Library-private Variables */ @@ -682,6 +751,10 @@ H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref); H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref); H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, size_t *obj_id_count_ptr); H5_DLL herr_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *oid_list, hbool_t app_ref, size_t *obj_id_count_ptr); +H5_DLL hsize_t H5F_get_pgend_meta_thres(const H5F_t *f); +H5_DLL hbool_t H5F_get_point_of_no_return(const H5F_t *f); +H5_DLL hbool_t H5F_get_first_alloc_dealloc(const H5F_t *f); +H5_DLL hbool_t H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); @@ -712,6 +785,8 @@ H5_DLL herr_t H5F_set_grp_btree_shared(H5F_t *f, struct H5UC_t *rc); H5_DLL hbool_t H5F_use_tmp_space(const H5F_t *f); H5_DLL hbool_t H5F_is_tmp_addr(const H5F_t *f, haddr_t addr); H5_DLL herr_t H5F_set_latest_flags(H5F_t *f, unsigned flags); +H5_DLL hsize_t H5F_get_alignment(const H5F_t *f); +H5_DLL hsize_t H5F_get_threshold(const H5F_t *f); #ifdef H5_HAVE_PARALLEL H5_DLL H5P_coll_md_read_flag_t H5F_coll_md_read(const H5F_t *f); H5_DLL void H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t flag); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index c57a821..f87aaad 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -182,6 +182,17 @@ typedef enum H5F_libver_t { } H5F_libver_t; /* File space handling strategy */ +typedef enum H5F_fspace_strategy_t { + H5F_FSPACE_STRATEGY_FSM_AGGR = 0, /* Mechanisms: free-space managers, aggregators, and virtual file drivers */ + /* This is the library default when not set */ + H5F_FSPACE_STRATEGY_PAGE = 1, /* Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */ + H5F_FSPACE_STRATEGY_AGGR = 2, /* Mechanisms: aggregators and virtual file drivers */ + H5F_FSPACE_STRATEGY_NONE = 3, /* Mechanisms: virtual file drivers */ + H5F_FSPACE_STRATEGY_NTYPES /* must be last */ +} H5F_fspace_strategy_t; + +/* Deprecated: File space handling strategy for release 1.10.0 */ +/* They are mapped to H5F_fspace_strategy_t as defined above from release 1.10.1 onwards */ typedef enum H5F_file_space_type_t { H5F_FILE_SPACE_DEFAULT = 0, /* Default (or current) free space strategy setting */ H5F_FILE_SPACE_ALL_PERSIST = 1, /* Persistent free space managers, aggregators, virtual file driver */ @@ -253,6 +264,10 @@ H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, /*OUT*/ hbool_t *is_enabled, /*OUT*/ hbool_t *is_currently_logging); H5_DLL herr_t H5Fformat_convert(hid_t fid); +H5_DLL herr_t H5Freset_page_buffering_stats(hid_t file_id); +H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], + unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); + #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag); H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 14dd655..34fe8f9 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1235,3 +1235,150 @@ H5F_mdc_log_location(const H5F_t *f) FUNC_LEAVE_NOAPI(f->shared->mdc_log_location) } /* end H5F_mdc_log_location() */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_alignment + * + * Purpose: Retrieve the 'alignment' for the file. + * + * Return: Success: Non-negative, the 'alignment' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_alignment(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->alignment) +} /* end H5F_get_alignment() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_threshold + * + * Purpose: Retrieve the 'threshold' for alignment in the file. + * + * Return: Success: Non-negative, the 'threshold' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_threshold(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->threshold) +} /* end H5F_get_threshold() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_pgend_meta_thres + * + * Purpose: Retrieve the 'page end meta threshold size' for the file. + * + * Return: Success: Non-negative, the 'pgend_meta_thres' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_pgend_meta_thres(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->pgend_meta_thres) +} /* end H5F_get_pgend_meta_thres() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_point_of_no_return + * + * Purpose: Retrieve the 'point of no return' value for the file. + * + * Return: Success: Non-negative, the 'point_of_no_return' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_point_of_no_return(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->point_of_no_return) +} /* end H5F_get_point_of_no_return() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_first_alloc_dealloc + * + * Purpose: Retrieve the 'first alloc / dealloc' value for the file. + * + * Return: Success: Non-negative, the 'first_alloc_dealloc' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_first_alloc_dealloc(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->first_alloc_dealloc) +} /* end H5F_get_first_alloc_dealloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_eoa_pre_fsm_fsalloc + * + * Purpose: Retrieve the 'EOA pre-FSM fsalloc' value for the file. + * + * Return: Success: Non-negative, the 'EOA pre-FSM fsalloc' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->eoa_pre_fsm_fsalloc) +} /* end H5F_get_eoa_pre_fsm_fsalloc() */ + diff --git a/src/H5Fspace.c b/src/H5Fspace.c new file mode 100644 index 0000000..53570f6 --- /dev/null +++ b/src/H5Fspace.c @@ -0,0 +1,226 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Fspace.c + * Dec 30 2013 + * Quincey Koziol <koziol@hdfgroup.org> + * + * Purpose: Space allocation routines for the file. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Fmodule.h" /* This source code file is part of the H5F module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5F_alloc + * + * Purpose: Wrapper for H5FD_alloc, to make certain EOA changes are + * reflected in superblock. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: The format address of the new file memory. + * Failure: The undefined address HADDR_UNDEF + * + * Programmer: Quincey Koziol + * Monday, December 30, 2013 + * + *------------------------------------------------------------------------- + */ +haddr_t +H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +{ + haddr_t ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI(HADDR_UNDEF) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(size > 0); + + /* Check whether the file can use temporary addresses */ + if(f->shared->use_tmp_space) { + haddr_t eoa; /* Current EOA for the file */ + + /* Get the EOA for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") + + /* Check for overlapping into file's temporary allocation space */ + if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) + HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") + } /* end if */ + + /* Call the file driver 'alloc' routine */ + ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, frag_addr, frag_size); + if(!H5F_addr_defined(ret_value)) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, HADDR_UNDEF, "file driver 'alloc' request failed") + + /* Mark EOA dirty */ + if(H5F_eoa_dirty(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark EOA as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_free + * + * Purpose: Wrapper for H5FD_free, to make certain EOA changes are + * reflected in superblock. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, December 30, 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, hsize_t size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(size > 0); + + /* Call the file driver 'free' routine */ + if(H5FD_free(f->shared->lf, dxpl_id, type, f, addr, size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "file driver 'free' request failed") + + /* Mark EOA dirty */ + if(H5F_eoa_dirty(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_try_extend + * + * Purpose: Extend a block at the end of the file, if possible. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: TRUE(1) - Block was extended + * FALSE(0) - Block could not be extended + * Failure: FAIL + * + * Programmer: Quincey Koziol + * Monday, 30 December, 2013 + * + *------------------------------------------------------------------------- + */ +htri_t +H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t blk_end, hsize_t extra_requested) +{ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(extra_requested > 0); + + /* Extend the object by extending the underlying file */ + if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, blk_end, extra_requested)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTEXTEND, FAIL, "driver try extend request failed") + + /* H5FD_try_extend() updates driver message and marks the superblock + * dirty, so no need to do it again here. + */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_try_extend() */ + diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 3050a28..58ef9bb 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -29,6 +29,7 @@ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ #include "H5Iprivate.h" /* IDs */ +#include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ #include "H5SMprivate.h" /* Shared Object Header Messages */ @@ -671,32 +672,98 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { - H5O_fsinfo_t fsinfo; /* Free-space manager info message from superblock extension */ + H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */ + uint8_t flags; /* Message flags */ - /* Retrieve the 'free-space manager info' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, meta_dxpl_id)) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") + /* Get message flags */ + if(H5O_msg_get_flags(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id, &flags) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to message flags for free-space manager info message") - /* Check for non-default info */ - if(f->shared->fs_strategy != fsinfo.strategy) { - f->shared->fs_strategy = fsinfo.strategy; + /* If message is NOT marked "unknown"--set up file space info */ + if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) { - /* Set non-default strategy in the property list */ - if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") - } /* end if */ - if(f->shared->fs_threshold != fsinfo.threshold) { - f->shared->fs_threshold = fsinfo.threshold; + /* Retrieve the 'file space info' structure */ + if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, meta_dxpl_id)) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") - /* Set non-default threshold in the property list */ - if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") - } /* end if */ + /* Update changed values */ + if(f->shared->fs_strategy != fsinfo.strategy) { + f->shared->fs_strategy = fsinfo.strategy; + + /* Set non-default strategy in the property list */ + if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + if(f->shared->fs_persist != fsinfo.persist) { + f->shared->fs_persist = fsinfo.persist; + + /* Set non-default strategy in the property list */ + if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &fsinfo.persist) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + if(f->shared->fs_threshold != fsinfo.threshold) { + f->shared->fs_threshold = fsinfo.threshold; + + /* Set non-default threshold in the property list */ + if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + HDassert(fsinfo.page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + if(f->shared->fs_page_size != fsinfo.page_size) { + f->shared->fs_page_size = fsinfo.page_size; + + /* Set file space page size in the property list */ + if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsinfo.page_size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space page size") + } /* end if */ + if(f->shared->pgend_meta_thres != fsinfo.pgend_meta_thres) + /* Initialize page end meta threshold */ + f->shared->pgend_meta_thres = fsinfo.pgend_meta_thres; + + if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc) + f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc; + + /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF + * in the absence of persistant free space managers. + */ + HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF)); + HDassert(!f->shared->first_alloc_dealloc); + + if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF) && + (H5F_INTENT(f) & H5F_ACC_RDWR)) + f->shared->first_alloc_dealloc = TRUE; + + f->shared->fs_addr[0] = HADDR_UNDEF; + for(u = 1; u < NELMTS(f->shared->fs_addr); u++) + f->shared->fs_addr[u] = fsinfo.fs_addr[u - 1]; + + if(fsinfo.mapped && (rw_flags & H5AC__READ_ONLY_FLAG) == 0) { - /* Set free-space manager addresses */ - f->shared->fs_addr[0] = HADDR_UNDEF; - for(u = 1; u < NELMTS(f->shared->fs_addr); u++) - f->shared->fs_addr[u] = fsinfo.fs_addr[u-1]; + /* Do the same kluge until we know for sure. VC */ +#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ + /* KLUGE ALERT!! + * + * H5F_super_ext_write_msg() expects f->shared->sblock to + * be set -- verify that it is NULL, and then set it. + * Set it back to NULL when we are done. + */ + HDassert(f->shared->sblock == NULL); + f->shared->sblock = sblock; +#endif /* JRM */ + + if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_FSINFO_ID) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension") + + if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") +#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ + f->shared->sblock = NULL; +#endif /* JRM */ + + } + } /* end if not marked "unknown" */ } /* end if */ /* Check for the extension having a 'metadata cache image' message */ @@ -796,6 +863,10 @@ H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial /* Set the pointer to the pinned superblock */ f->shared->sblock = sblock; + /* Set the page aggregation mode */ + if(H5F__set_paged_aggr(f, (hbool_t)H5F_PAGED_AGGR(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "failed to set paged_aggr status for file driver") + done: /* Reset the ring in the DXPL */ if(H5AC_reset_ring(dxpl, orig_ring) < 0) @@ -897,7 +968,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Check for non-default free-space settings */ if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF && - f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF)) + f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF && + f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF && + f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF)) non_default_fs_settings = TRUE; /* Bump superblock version if latest superblock version support is enabled */ @@ -906,8 +979,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Bump superblock version to create superblock extension for SOHM info */ else if(f->shared->sohm_nindexes > 0) super_vers = HDF5_SUPERBLOCK_VERSION_2; - /* Bump superblock version to create superblock extension for - * non-default file space strategy or non-default free-space threshold + /* + * Bump superblock version to create superblock extension for: + * -- non-default file space strategy or + * -- non-default persisting free-space or + * -- non-default free-space threshold or + * -- non-default page_size */ else if(non_default_fs_settings) super_vers = HDF5_SUPERBLOCK_VERSION_2; @@ -928,6 +1005,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set superblock version") } /* end if */ + if(H5FD_set_paged_aggr(f->shared->lf, (hbool_t)H5F_PAGED_AGGR(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "failed to set paged_aggr status for file driver") + /* * The superblock starts immediately after the user-defined * header, which we have already insured is a proper size. The @@ -939,9 +1019,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Sanity check the userblock size vs. the file's allocation alignment */ if(userblock_size > 0) { - if(userblock_size < f->shared->alignment) + /* Set up the alignment to use for page or aggr fs */ + hsize_t alignment = H5F_PAGED_AGGR(f) ? f->shared->fs_page_size : f->shared->alignment; + + if(userblock_size < alignment) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be > file object alignment") - if(0 != (userblock_size % f->shared->alignment)) + if(0 != (userblock_size % alignment)) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be an integral multiple of file object alignment") } /* end if */ @@ -997,10 +1080,6 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) if(super_vers < HDF5_SUPERBLOCK_VERSION_2) superblock_size += driver_size; - /* Reserve space in the file for the superblock, instead of allocating it */ - if(H5F__set_eoa(f, H5FD_MEM_SUPER, superblock_size) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to set EOA value for superblock") - /* Set the ring type in the DXPL */ if(H5AC_set_ring(dxpl_id, H5AC_RING_SB, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set ring value") @@ -1013,6 +1092,10 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Keep a copy of the superblock info */ f->shared->sblock = sblock; + /* Allocate space for the superblock */ + if(HADDR_UNDEF == H5MF_alloc(f, H5FD_MEM_SUPER, dxpl_id, superblock_size)) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for superblock") + /* set the drvinfo filed to NULL -- will overwrite this later if needed */ f->shared->drvinfo = NULL; @@ -1115,17 +1198,23 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) } /* end if */ /* Check for non-default free-space info settings */ - if(non_default_fs_settings) { - H5FD_mem_t type; /* Memory type for iteration */ - H5O_fsinfo_t fsinfo; /* Free space manager info message */ - - /* Write free-space manager info message to superblock extension object header if needed */ - fsinfo.strategy = f->shared->fs_strategy; - fsinfo.threshold = f->shared->fs_threshold; - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type-1] = HADDR_UNDEF; - - if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) + if(non_default_fs_settings) { + H5F_mem_page_t ptype; + H5O_fsinfo_t fsinfo; /* Free space manager info message */ + + /* Write free-space manager info message to superblock extension object header if needed */ + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + fsinfo.mapped = FALSE; + + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + + if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE | H5O_MSG_FLAG_MARK_IF_UNKNOWN, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update free-space info header message") } /* end if */ } /* end if */ diff --git a/src/H5HFsection.c b/src/H5HFsection.c index 37ff8f4..b113ca1 100644 --- a/src/H5HFsection.c +++ b/src/H5HFsection.c @@ -85,14 +85,14 @@ static herr_t H5HF_sect_single_full_dblock(H5HF_hdr_t *hdr, hid_t dxpl_id, H5HF_free_section_t *sect); /* 'single' section callbacks */ -static herr_t H5HF_sect_single_add(H5FS_section_info_t *sect, unsigned *flags, +static herr_t H5HF_sect_single_add(H5FS_section_info_t **sect, unsigned *flags, void *udata); static H5FS_section_info_t *H5HF_sect_single_deserialize(const H5FS_section_class_t *cls, hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size, unsigned *des_flags); static htri_t H5HF_sect_single_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5HF_sect_single_merge(H5FS_section_info_t *sect1, +static herr_t H5HF_sect_single_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); static htri_t H5HF_sect_single_can_shrink(const H5FS_section_info_t *sect, void *udata); @@ -121,7 +121,7 @@ static H5FS_section_info_t *H5HF_sect_row_deserialize(const H5FS_section_class_t unsigned *des_flags); static htri_t H5HF_sect_row_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5HF_sect_row_merge(H5FS_section_info_t *sect1, +static herr_t H5HF_sect_row_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); static htri_t H5HF_sect_row_can_shrink(const H5FS_section_info_t *sect, void *udata); @@ -811,7 +811,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) +H5HF_sect_single_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) { herr_t ret_value = SUCCEED; /* Return value */ @@ -821,7 +821,7 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) * have already been checked when it was first added */ if(!(*flags & H5FS_ADD_DESERIALIZING)) { - H5HF_free_section_t *sect = (H5HF_free_section_t *)_sect; /* Fractal heap free section */ + H5HF_free_section_t **sect = (H5HF_free_section_t **)_sect; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ hid_t dxpl_id = udata->dxpl_id; /* DXPL ID for operation */ @@ -832,14 +832,14 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) /* Check if single section covers entire direct block it's in */ /* (converts to row section possibly) */ - if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect) < 0) + if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section") /* Set the "returned space" flag if the single section was changed * into a row section, so the "merging & shrinking" algorithm * gets executed in the free space manager */ - if(sect->sect_info.type != H5HF_FSPACE_SECT_SINGLE) + if((*sect)->sect_info.type != H5HF_FSPACE_SECT_SINGLE) *flags |= H5FS_ADD_RETURNED_SPACE; } /* end if */ @@ -949,10 +949,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5HF_sect_single_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void *_udata) { - H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */ + H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */ H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ @@ -963,26 +963,26 @@ H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_SINGLE); + HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_SINGLE); HDassert(sect2); HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_SINGLE); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ if(H5HF_sect_single_free((H5FS_section_info_t *)sect2) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTRELEASE, FAIL, "can't free section node") /* Check to see if we should revive first section */ - if(sect1->sect_info.state != H5FS_SECT_LIVE) - if(H5HF_sect_single_revive(hdr, dxpl_id, sect1) < 0) + if((*sect1)->sect_info.state != H5FS_SECT_LIVE) + if(H5HF_sect_single_revive(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Check if single section covers entire direct block it's in */ /* (converts to row section possibly) */ - if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect1) < 0) + if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section") done: @@ -1771,10 +1771,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5HF_sect_row_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void *_udata) { - H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */ + H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */ H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ @@ -1785,7 +1785,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); + HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); HDassert(sect2); HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); @@ -1802,8 +1802,8 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, } /* end if */ else { /* Check to see if we should revive first section */ - if(sect1->sect_info.state != H5FS_SECT_LIVE) - if(H5HF_sect_row_revive(hdr, dxpl_id, sect1) < 0) + if((*sect1)->sect_info.state != H5FS_SECT_LIVE) + if(H5HF_sect_row_revive(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Check to see if we should revive second section */ @@ -1812,7 +1812,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Merge rows' underlying indirect sections together */ - if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, sect1, sect2) < 0) + if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, (*sect1), sect2) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTMERGE, FAIL, "can't merge underlying indirect sections") } /* end else */ @@ -29,6 +29,7 @@ /****************/ #define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5FS_FRIEND /*suppress error about including H5Fpkg */ #include "H5MFmodule.h" /* This source code file is part of the H5MF module */ @@ -38,6 +39,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ +#include "H5FSpkg.h" /* File access */ #include "H5Iprivate.h" /* IDs */ #include "H5MFpkg.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -50,11 +52,6 @@ #define H5MF_FSPACE_SHRINK 80 /* Percent of "normal" size to shrink serialized free space size */ #define H5MF_FSPACE_EXPAND 120 /* Percent of "normal" size to expand serialized free space size */ -/* Map an allocation request type to a free list */ -#define H5MF_ALLOC_TO_FS_TYPE(F, T) ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) \ - ? (T) : (F)->shared->fs_type_map[T]) - - /******************/ /* Local Typedefs */ /******************/ @@ -84,11 +81,24 @@ typedef struct { /********************/ /* Allocator routines */ -static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -static herr_t H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -static herr_t H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl); +static haddr_t H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size); + +/* "File closing" routines */ +static herr_t H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id); +static herr_t H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id); static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id); +/* General routines */ +static herr_t H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums); +static hbool_t H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type); +static hbool_t H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace); + +/* Free-space type manager routines */ +static herr_t H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); + /*********************/ /* Package Variables */ @@ -223,10 +233,54 @@ done: /*------------------------------------------------------------------------- - * Function: H5MF__alloc_open + * Function: H5MF_alloc_to_fs_type + * + * Purpose: Map "alloc_type" to the free-space manager type + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Nov 2016 + * + *------------------------------------------------------------------------- + */ +void +H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(fs_type); + + if(H5F_PAGED_AGGR(f)) { /* paged aggregation */ + if(size >= f->shared->fs_page_size) { + if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) { /* multi or split driver */ + /* For non-contiguous address space, map to large size free-space manager for each alloc_type */ + if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[alloc_type]) + *fs_type = (H5F_mem_page_t) (alloc_type + (H5FD_MEM_NTYPES - 1)); + else + *fs_type = (H5F_mem_page_t) (f->shared->fs_type_map[alloc_type] + (H5FD_MEM_NTYPES - 1)); + } /* end if */ + else + /* For contiguous address space, map to generic large size free-space manager */ + *fs_type = H5F_MEM_PAGE_GENERIC; /* H5F_MEM_PAGE_SUPER */ + } /* end if */ + else + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + } /* end if */ + else /* non-paged aggregation */ + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + + FUNC_LEAVE_NOAPI_VOID +} /* end H5MF_alloc_to_fs_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_open_fstype * * Purpose: Open an existing free space manager of TYPE for file by - * creating a free-space structure + * creating a free-space structure. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -238,31 +292,48 @@ done: *------------------------------------------------------------------------- */ herr_t -H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */ - H5MF_FSPACE_SECT_CLS_SIMPLE}; - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5MF_FSPACE_SECT_CLS_SIMPLE, + H5MF_FSPACE_SECT_CLS_SMALL, + H5MF_FSPACE_SECT_CLS_LARGE }; + hsize_t alignment; /* Alignment to use */ + hsize_t threshold; /* Threshold to use */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(H5F_addr_defined(f->shared->fs_addr[type])); HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED); - HDassert(type == H5MF_ALLOC_TO_FS_TYPE(f, type)); + + /* Set up the aligment and threshold to use depending on the manager type */ + if(H5F_PAGED_AGGR(f)) { + alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF; + threshold = H5F_ALIGN_THRHD_DEF; + } /* end if */ + else { + alignment = f->shared->alignment; + threshold = f->shared->threshold; + } /* end else */ /* Set the ring type in the DXPL */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -272,7 +343,7 @@ H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) /* Open an existing free space structure for the file */ if(NULL == (f->shared->fs_man[type] = H5FS_open(f, dxpl_id, f->shared->fs_addr[type], - NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold))) + NELMTS(classes), classes, f, alignment, threshold))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") /* Set the state for the free space manager to "open", if it is now */ @@ -286,14 +357,15 @@ done: HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF__alloc_open() */ +} /* end H5MF_open_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF_alloc_create + * Function: H5MF__create_fstype * * Purpose: Create free space manager of TYPE for the file by creating - * a free-space structure + * a free-space structure + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -305,21 +377,34 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */ - H5MF_FSPACE_SECT_CLS_SIMPLE}; - herr_t ret_value = SUCCEED; /* Return value */ - H5FS_create_t fs_create; /* Free space creation parameters */ + H5MF_FSPACE_SECT_CLS_SIMPLE, + H5MF_FSPACE_SECT_CLS_SMALL, + H5MF_FSPACE_SECT_CLS_LARGE }; + H5FS_create_t fs_create; /* Free space creation parameters */ + hsize_t alignment; /* Alignment to use */ + hsize_t threshold; /* Threshold to use */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED); @@ -330,24 +415,48 @@ H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) fs_create.max_sect_addr = 1 + H5VM_log2_gen((uint64_t)f->shared->maxaddr); fs_create.max_sect_size = f->shared->maxaddr; - if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL, - &fs_create, NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") + /* Set up alignment and threshold to use depending on TYPE */ + if(H5F_PAGED_AGGR(f)) { + alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF; + threshold = H5F_ALIGN_THRHD_DEF; + } /* end if */ + else { + alignment = f->shared->alignment; + threshold = f->shared->threshold; + } /* end else */ + /* Set the ring type in the DXPL */ + if(H5MF__fsm_type_is_self_referential(f, type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + + if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL, + &fs_create, NELMTS(classes), classes, f, alignment, threshold))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") /* Set the state for the free space manager to "open", if it is now */ if(f->shared->fs_man[type]) f->shared->fs_state[type] = H5F_FS_STATE_OPEN; done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_alloc_create() */ +} /* end H5MF__create_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF__alloc_start + * Function: H5MF_start_fstype * - * Purpose: Open or create a free space manager of a given type + * Purpose: Open or create a free space manager of a given TYPE. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -359,40 +468,123 @@ done: *------------------------------------------------------------------------- */ herr_t -H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ /* Check if the free space manager exists already */ if(H5F_addr_defined(f->shared->fs_addr[type])) { /* Open existing free space manager */ - if(H5MF__alloc_open(f, dxpl_id, type) < 0) + if(H5MF_open_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space") } /* end if */ else { /* Create new free space manager */ - if(H5MF_alloc_create(f, dxpl_id, type) < 0) + if(H5MF__create_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCREATE, FAIL, "can't initialize file free space") } /* end else */ done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF__alloc_start() */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_start_fstype() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__delete_fstype + * + * Purpose: Delete the free-space manager as specified by TYPE. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + /* check args */ + HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert(H5F_addr_defined(f->shared->fs_addr[type])); + + /* Put address into temporary variable and reset it */ + /* (Avoids loopback in file space freeing routine) */ + tmp_fs_addr = f->shared->fs_addr[type]; + f->shared->fs_addr[type] = HADDR_UNDEF; + + /* Shift to "deleting" state, to make certain we don't track any + * file space freed as a result of deleting the free space manager. + */ + f->shared->fs_state[type] = H5F_FS_STATE_DELETING; + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_type_is_self_referential(f, type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Delete free space manager for this type */ + if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager") + + /* Shift [back] to closed state */ + HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + + /* Sanity check that the free space manager for this type wasn't started up again */ + HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__delete_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF__alloc_close + * Function: H5MF__close_fstype * - * Purpose: Close an existing free space manager of TYPE for file + * Purpose: Close the free space manager of TYPE for file + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -402,7 +594,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { herr_t ret_value = SUCCEED; /* Return value */ @@ -412,11 +604,18 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(f->shared->fs_man[type]); HDassert(f->shared->fs_state[type] != H5F_FS_STATE_CLOSED); +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Close an existing free space structure for the file */ if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info") @@ -425,7 +624,158 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) done: FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF__alloc_close() */ +} /* end H5MF__close_fstype() */ + + + +/*------------------------------------------------------------------------- + * Function: H5MF_add_sect + * + * Purpose: To add a section to the specified free-space manager. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + H5MF_sect_ud_t udata; /* User data for callback */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ + + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + HDassert(f); + HDassert(fspace); + HDassert(node); + + H5MF_alloc_to_fs_type(f, alloc_type, node->sect_info.size, &fs_type); + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = TRUE; + udata.allow_eoa_shrink_only = FALSE; + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_is_self_referential(f, fspace)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: adding node, node->sect_info.addr = %a, node->sect_info.size = %Hu\n", FUNC, node->sect_info.addr, node->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Add the section */ + if(H5FS_sect_add(f, dxpl_id, fspace, (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + +done: + /* Reset the ring in the DXPL */ + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_add_sect() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_find_sect + * + * Purpose: To find a section from the specified free-space manager to fulfill the request. + * If found, re-add the left-over space back to the manager. + * + * Return: TRUE if a section is found to fulfill the request + * FALSE if not + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +htri_t +H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + H5MF_free_section_t *node; /* Free space section pointer */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FAIL; /* Whether an existing free list node was found */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + HDassert(f); + HDassert(fspace); + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_is_self_referential(f, fspace)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + + /* Try to get a section from the free space manager */ + if((ret_value = H5FS_sect_find(f, dxpl_id, fspace, size, (H5FS_section_info_t **)&node)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "error locating free space in file") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section found = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Check for actually finding section */ + if(ret_value) { + /* Sanity check */ + HDassert(node); + + /* Retrieve return value */ + if(addr) + *addr = node->sect_info.addr; + + /* Check for eliminating the section */ + if(node->sect_info.size == size) { +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: freeing node\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Free section node */ + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + } /* end if */ + else { + /* Adjust information for section */ + node->sect_info.addr += size; + node->sect_info.size -= size; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Re-add the section to the free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, fspace, node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + } /* end else */ + } /* end if */ + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_find_sect() */ /*------------------------------------------------------------------------- @@ -451,7 +801,7 @@ H5MF_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size) H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* free space manager ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ @@ -466,12 +816,20 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(f->shared->lf); HDassert(size > 0); - /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); + if(f->shared->first_alloc_dealloc) { + HDassert(! H5AC_cache_image_pending(f)); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "tidy of self referential fsm hack failed") + } /* end if */ + + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 1.0\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -481,74 +839,46 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ /* Check if we are using the free space manager for this file */ if(H5F_HAVE_FREE_SPACE_MANAGER(f)) { + /* We are about to change the contents of the free space manager -- + * notify metadata cache that the associated fsm ring is + * unsettled + */ + if(H5AC_unsettle_ring(f, fsm_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, HADDR_UNDEF, "attempt to notify cache that ring is unsettled failed") + /* Check if the free space manager for the file has been initialized */ - if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) - if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0) + if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) { + /* Open the free-space manager */ + if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, HADDR_UNDEF, "can't initialize file free space") + HDassert(f->shared->fs_man[fs_type]); + } /* end if */ /* Search for large enough space in the free space manager */ - if(f->shared->fs_man[fs_type]) { - H5MF_free_section_t *node; /* Free space section pointer */ - htri_t node_found = FALSE; /* Whether an existing free list node was found */ - - /* Try to get a section from the free space manager */ - if((node_found = H5FS_sect_find(f, dxpl_id, f->shared->fs_man[fs_type], size, (H5FS_section_info_t **)&node)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating free space in file") -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.5, node_found = %t\n", FUNC, node_found); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* Check for actually finding section */ - if(node_found) { - /* Sanity check */ - HDassert(node); - - /* Retrieve return value */ - ret_value = node->sect_info.addr; - - /* Check for eliminating the section */ - if(node->sect_info.size == size) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.6, freeing node\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - /* Free section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free simple section node") - } /* end if */ - else { - H5MF_sect_ud_t udata; /* User data for callback */ - - /* Adjust information for section */ - node->sect_info.addr += size; - node->sect_info.size -= size; - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + if(f->shared->fs_man[fs_type]) + if(H5MF_find_sect(f, alloc_type, dxpl_id, size, f->shared->fs_man[fs_type], &ret_value) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating a node") + } /* end if */ + /* If no space is found from the free-space manager, continue further action */ + if(!H5F_addr_defined(ret_value)) { #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.7, re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size); +HDfprintf(stderr, "%s: Check 2.0\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ - /* Re-insert section node into file's free space */ - if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") - } /* end else */ - - /* Leave now */ - HGOTO_DONE(ret_value) - } /* end if */ + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) { + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + if(HADDR_UNDEF == (ret_value = H5MF__alloc_pagefs(f, alloc_type, dxpl_id, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from paged aggregation") } /* end if */ + else { /* For non-paged aggregation, continue further action */ + if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd") + } /* end else */ + } /* end if */ + HDassert(H5F_addr_defined(ret_value)); #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 2.0\n", FUNC); +HDfprintf(stderr, "%s: Check 3.0\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ - } /* end if */ - - /* Allocate from the metadata aggregator (or the VFD) */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd") done: /* Reset the ring in the DXPL */ @@ -568,6 +898,146 @@ H5MF_sects_dump(f, dxpl_id, stderr); /*------------------------------------------------------------------------- + * Function: H5MF__alloc_pagefs + * + * Purpose: Allocate space from either the large or small free-space manager. + * For "large" request: + * Allocate request from VFD + * Determine mis-aligned fragment and return the fragment to the + * appropriate manager + * For "small" request: + * Allocate a page from the large manager + * Determine whether space is available from a mis-aligned fragment + * being returned to the manager + * Return left-over space to the manager after fulfilling request + * + * Return: Success: The file address of new chunk. + * Failure: HADDR_UNDEF + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static haddr_t +H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size) +{ + H5F_mem_page_t ptype; /* Free-space mananger type */ + H5MF_free_section_t *node = NULL; /* Free space section pointer */ + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, HADDR_UNDEF) + +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); +#endif /* H5MF_ALLOC_DEBUG */ + + H5MF_alloc_to_fs_type(f, alloc_type, size, &ptype); + + switch(ptype) { + case H5F_MEM_PAGE_GENERIC: + case H5F_MEM_PAGE_LARGE_BTREE: + case H5F_MEM_PAGE_LARGE_DRAW: + case H5F_MEM_PAGE_LARGE_GHEAP: + case H5F_MEM_PAGE_LARGE_LHEAP: + case H5F_MEM_PAGE_LARGE_OHDR: + { + haddr_t eoa; /* EOA for the file */ + hsize_t frag_size = 0; /* Fragment size */ + + /* Get the EOA for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") + HDassert(!(eoa % f->shared->fs_page_size)); + + H5MF_EOA_MISALIGN(f, (eoa+size), f->shared->fs_page_size, frag_size); + + /* Allocate from VFD */ + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size + frag_size, NULL, NULL))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") + + /* If there is a mis-aligned fragment at EOA */ + if(frag_size) { + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[ptype])) + if(H5MF_start_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space") + + /* Create free space section for the fragment */ + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, ret_value + size, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section") + + /* Add the fragment to the large free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") + + node = NULL; + } /* end if */ + } + break; + + case H5F_MEM_PAGE_META: + case H5F_MEM_PAGE_DRAW: + case H5F_MEM_PAGE_BTREE: + case H5F_MEM_PAGE_GHEAP: + case H5F_MEM_PAGE_LHEAP: + case H5F_MEM_PAGE_OHDR: + { + haddr_t new_page; /* The address for the new file size page */ + + /* Allocate one file space page */ + if(HADDR_UNDEF == (new_page = H5MF_alloc(f, alloc_type, dxpl_id, f->shared->fs_page_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[ptype])) + if(H5MF_start_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space") + HDassert(f->shared->fs_man[ptype]); + + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_SMALL, (new_page + size), (f->shared->fs_page_size - size)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section") + + /* Add the remaining space in the page to the manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") + + node = NULL; + + /* Insert the new page into the Page Buffer list of new pages so + we don't read an empty page from disk */ + if(f->shared->page_buf != NULL && H5PB_add_new_page(f, alloc_type, new_page) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't add new page to Page Buffer new page list") + + ret_value = new_page; + } + break; + + case H5F_MEM_PAGE_NTYPES: + case H5F_MEM_PAGE_DEFAULT: + default: + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space: unrecognized type") + break; + } /* end switch */ + +done: +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); +#endif /* H5MF_ALLOC_DEBUG */ +#ifdef H5MF_ALLOC_DEBUG_DUMP +H5MF_sects_dump(f, dxpl_id, stderr); +#endif /* H5MF_ALLOC_DEBUG_DUMP */ + + /* Release section node, if allocated and not added to section list or merged */ + if(node) + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free section node") + + FUNC_LEAVE_NOAPI_TAG(ret_value, HADDR_UNDEF) +} /* end H5MF__alloc_pagefs() */ + + +/*------------------------------------------------------------------------- * Function: H5MF_alloc_tmp * * Purpose: Allocate temporary space in the file @@ -646,13 +1116,13 @@ H5MF_xfree(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, hsize_t size) { H5F_io_info2_t fio_info; /* I/O info for operation */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ H5MF_free_section_t *node = NULL; /* Free space section pointer */ - H5MF_sect_ud_t udata; /* User data for callback */ + unsigned ctype; /* section class type */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ - H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -666,19 +1136,16 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN HGOTO_DONE(SUCCEED) HDassert(addr != 0); /* Can't deallocate the superblock :-) */ - /* Check for attempting to free space that's a 'temporary' file address */ - if(H5F_addr_le(f->shared->tmp_addr, addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space") + if(f->shared->first_alloc_dealloc) { + HDassert(!H5AC_cache_image_pending(f)); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end if */ - /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); -#endif /* H5MF_ALLOC_DEBUG_MORE */ + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -686,6 +1153,19 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; + /* we are about to change the contents of the free space manager -- + * notify metadata cache that the associated fsm ring is + * unsettled + */ + /* Only do so for strategies that use free-space managers */ + if(H5F_HAVE_FREE_SPACE_MANAGER(f)) + if(H5AC_unsettle_ring(f, fsm_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, FAIL, "attempt to notify cache that ring is unsettled failed") + + /* Check for attempting to free space that's a 'temporary' file address */ + if(H5F_addr_le(f->shared->tmp_addr, addr)) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space") + /* Set up I/O info for operation */ fio_info.f = f; if(H5FD_MEM_DRAW == alloc_type) { @@ -712,7 +1192,7 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); * space is at the end of the file */ #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)fs_type, f->shared->fs_addr[fs_type]); +HDfprintf(stderr, "%s: fs_addr = %a\n", FUNC, f->shared->fs_addr[fs_type]); #endif /* H5MF_ALLOC_DEBUG_MORE */ if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) { htri_t status; /* "can absorb" status for section into */ @@ -736,8 +1216,8 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a /* If we are deleting the free space manager, leave now, to avoid * [re-]starting it. - * or if file space strategy type is not using a free space manager - * (H5F_FILE_SPACE_AGGR_VFD or H5F_FILE_SPACE_VFD), drop free space + * or if file space strategy type is not using a free space manager + * (H5F_FSPACE_STRATEGY_AGGR or H5F_FSPACE_STRATEGY_NONE), drop free space * section on the floor. * * Note: this drops the space to free on the floor... @@ -755,38 +1235,42 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a * space isn't at the end of the file, so start up (or create) * the file space manager */ - if(H5MF__alloc_start(f, dxpl_id, fs_type) < 0) + if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") } /* end if */ - /* Create free space section for block */ - if(NULL == (node = H5MF_sect_simple_new(addr, size))) + /* Create the free-space section for the freed section */ + ctype = H5MF_SECT_CLASS_TYPE(f, size); + if(NULL == (node = H5MF_sect_new(ctype, addr, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; - - /* If size of section freed is larger than threshold, add it to the free space manager */ + /* If size of the freed section is larger than threshold, add it to the free space manager */ if(size >= f->shared->fs_threshold) { HDassert(f->shared->fs_man[fs_type]); #ifdef H5MF_ALLOC_DEBUG_MORE HDfprintf(stderr, "%s: Before H5FS_sect_add()\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Add to the free space for the file */ - if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space") - node = NULL; + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space") + node = NULL; + #ifdef H5MF_ALLOC_DEBUG_MORE HDfprintf(stderr, "%s: After H5FS_sect_add()\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ else { htri_t merged; /* Whether node was merged */ + H5MF_sect_ud_t udata; /* User data for callback */ + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = TRUE; + udata.allow_eoa_shrink_only = FALSE; /* Try to merge the section that is smaller than threshold */ if((merged = H5FS_sect_try_merge(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata)) < 0) @@ -804,7 +1288,7 @@ done: /* Release section node, if allocated and not added to section list or merged */ if(node) - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") #ifdef H5MF_ALLOC_DEBUG @@ -821,6 +1305,14 @@ H5MF_sects_dump(f, dxpl_id, stderr); * Function: H5MF_try_extend * * Purpose: Extend a block in the file if possible. + * For non-paged aggregation: + * --try to extend at EOA + * --try to extend into the aggregators + * --try to extend into a free-space section if adjoined + * For paged aggregation: + * --try to extend at EOA + * --try to extend into a free-space section if adjoined + * --try to extend into the page end threshold if a metadata block * * Return: Success: TRUE(1) - Block was extended * FALSE(0) - Block could not be extended @@ -835,14 +1327,16 @@ htri_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsize_t extra_requested) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - haddr_t end; /* End of block to extend */ - H5FD_mem_t fs_type; /* Memory type of the free space manager */ - H5FD_mem_t map_type; /* Mapped type */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - htri_t ret_value = FAIL; /* Return value */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + haddr_t end; /* End of block to extend */ + H5FD_mem_t map_type; /* Mapped type */ + H5F_mem_page_t fs_type; /* free space type */ + htri_t allow_extend = TRUE; /* Possible to extend the block */ + hsize_t frag_size = 0; /* Size of mis-aligned fragment */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG @@ -859,12 +1353,33 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r /* Compute end of block to extend */ end = addr + size; + /* For paged aggregation: + * To extend a small block: can only extend if not crossing page boundary + * To extend a large block at EOA: calculate in advance mis-aligned fragment so EOA will still end at page boundary + */ + if(H5F_PAGED_AGGR(f)) { + if(size < f->shared->fs_page_size) { + /* To extend a small block: cannot cross page boundary */ + if((addr / f->shared->fs_page_size) != (((end + extra_requested) - 1) / f->shared->fs_page_size)) + allow_extend = FALSE; + } /* end if */ + else { + haddr_t eoa; /* EOA for the file */ + + /* To extend a large block: calculate in advance the mis-aligned fragment so EOA will end at page boundary if extended */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") + HDassert(!(eoa % f->shared->fs_page_size)); + + H5MF_EOA_MISALIGN(f, (eoa+extra_requested), f->shared->fs_page_size, frag_size); + } /* end else */ + } /* end if */ + /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -872,29 +1387,90 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - /* Check if the block is exactly at the end of the file */ - if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, dxpl_id, end, extra_requested)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") - else if(ret_value == FALSE) { - H5F_blk_aggr_t *aggr; /* Aggregator to use */ + if(allow_extend) { + /* Try extending the block at EOA */ + if((ret_value = H5F_try_extend(f, dxpl_id, map_type, end, extra_requested + frag_size)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: extended = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If extending at EOA succeeds: */ + /* for paged aggregation, put the fragment into the large-sized free-space manager */ + if(ret_value == TRUE && H5F_PAGED_AGGR(f) && frag_size) { + H5MF_free_section_t *node = NULL; /* Free space section pointer */ + + /* Should be large-sized block */ + HDassert(size >= f->shared->fs_page_size); + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[fs_type])) + if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + + /* Create free space section for the fragment */ + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, end + extra_requested, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") + + /* Add the fragment to the large-sized free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + + node = NULL; + } /* end if */ - /* Check for test block able to extend aggregation block */ - aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); - if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") - else if(ret_value == FALSE) { + /* For non-paged aggregation: try to extend into the aggregators */ + if(ret_value == FALSE && (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || + f->shared->fs_strategy == H5F_FSPACE_STRATEGY_AGGR) ) { + H5F_blk_aggr_t *aggr; /* Aggregator to use */ + + /* Check if the block is able to extend into aggregation block */ + aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); + if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: H5MF_aggr_try_extend = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + + /* If no extension so far, try to extend into a free-space section */ + if(ret_value == FALSE && ((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) || + (H5F_PAGED_AGGR(f))) ) { + H5MF_sect_ud_t udata; /* User data */ + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; /* Check if the free space for the file has been initialized */ if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) - if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0) + /* Open the free-space manager */ + if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") - /* Check for test block able to block in free space manager */ - if(f->shared->fs_man[fs_type]) - if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested)) < 0) + /* Try to extend the block into a free-space section */ + if(f->shared->fs_man[fs_type]) { + if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested, H5FS_ADD_RETURNED_SPACE, &udata)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending block in free space manager") +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Try to H5FS_sect_try_extend = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + + /* For paged aggregation and a metadata block: try to extend into page end threshold */ + if(ret_value == FALSE && H5F_PAGED_AGGR(f) && map_type != H5FD_MEM_DRAW) { + H5MF_EOA_MISALIGN(f, end, f->shared->fs_page_size, frag_size); + + if(frag_size <= H5F_PGEND_META_THRES(f) && extra_requested <= frag_size) + ret_value = TRUE; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Try to extend into the page end threshold = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ } /* end if */ - } /* end else-if */ + } /* allow_extend */ done: /* Reset the ring in the DXPL */ @@ -914,174 +1490,316 @@ H5MF_sects_dump(f, dxpl_id, stderr); /*------------------------------------------------------------------------- - * Function: H5MF_get_freespace + * Function: H5MF_try_shrink * - * Purpose: Retrieve the amount of free space in a file. + * Purpose: Try to shrink the size of a file with a block or absorb it + * into a block aggregator. * - * Return: Success: Amount of free space in file - * Failure: Negative + * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * Monday, October 6, 2003 + * koziol@hdfgroup.org + * Feb 14 2008 * - * Modifications: - * Vailin Choi; July 2012 - * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY, - * checks are added to account for the last section of each free-space manager - * and the remaining space in the two aggregators are at EOF. *------------------------------------------------------------------------- */ -herr_t -H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size) +htri_t +H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, + hsize_t size) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t needed_ring; /* Ring value needed for loop iteration */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - haddr_t eoa; /* End of allocated space in the file */ - haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ - hsize_t ma_size = 0; /* Size of "metadata aggregator" */ - haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ - hsize_t sda_size = 0; /* Size of "small data aggregator" */ - hsize_t tot_fs_size = 0; /* Amount of all free space managed */ - hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */ - H5FD_mem_t type; /* Memory type for iteration */ - hbool_t fs_started[H5FD_MEM_NTYPES]; /* Indicate whether the free-space manager has been started */ - hbool_t eoa_shrank; /* Whether an EOA shrink occurs */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5MF_free_section_t *node = NULL; /* Free space section pointer */ + H5MF_sect_ud_t udata; /* User data for callback */ + H5FS_section_class_t *sect_cls; /* Section class */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ + H5F_mem_page_t fs_type; /* Free space type */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size); +#endif /* H5MF_ALLOC_DEBUG */ - /* check args */ + /* check arguments */ HDassert(f); HDassert(f->shared); HDassert(f->shared->lf); + HDassert(H5F_addr_defined(addr)); + HDassert(size > 0); - /* Retrieve the 'eoa' for the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_DEFAULT))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + /* Set up free-space section class information */ + sect_cls = H5MF_SECT_CLS_TYPE(f, size); + HDassert(sect_cls); + + /* Get free space type from allocation type */ + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - curr_ring = H5AC_RING_RDFSM; - /* Retrieve metadata aggregator info, if available */ - if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") + /* Create free-space section for block */ + if(NULL == (node = H5MF_sect_new(sect_cls->type, addr, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") - /* Retrieve 'small data' aggregator info, if available */ - if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */ + udata.allow_eoa_shrink_only = FALSE; - /* Iterate over all the free space types that have managers and get each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + /* Check if the block can shrink the container */ + if(sect_cls->can_shrink) { + if((ret_value = (*sect_cls->can_shrink)((const H5FS_section_info_t *)node, &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container") + if(ret_value > 0) { + HDassert(sect_cls->shrink); - fs_started[type] = FALSE; + if((*sect_cls->shrink)((H5FS_section_info_t **)&node, &udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container") + } /* end if */ + } /* end if */ - /* test to see if we need to switch rings -- do so if required */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; - } /* end if */ + /* Free section node allocated */ + if(node && H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") - /* Check if the free space for the file has been initialized */ - if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) { - if(H5MF__alloc_open(f, dxpl_id, type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") - HDassert(f->shared->fs_man[type]); - fs_started[type] = TRUE; - } /* end if */ +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_try_shrink() */ - /* Check if there's free space of this type */ - if(f->shared->fs_man[type]) { - hsize_t type_fs_size = 0; /* Amount of free space managed for each type */ - hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */ + +/*------------------------------------------------------------------------- + * Function: H5MF_close + * + * Purpose: Close the free space tracker(s) for a file: + * paged or non-paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_close(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ - /* Retrieve free space size from free space manager */ - if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") - if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats") + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ - /* Increment total free space for types */ - tot_fs_size += type_fs_size; - tot_meta_size += type_meta_size; - } /* end if */ - } /* end for */ + /* check args */ + HDassert(f); + HDassert(f->shared); - /* Iterate until no more EOA shrink occurs */ - do { - eoa_shrank = FALSE; + if(H5F_PAGED_AGGR(f)) { + if((ret_value = H5MF__close_pagefs(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'page' file space") + } /* end if */ + else { + if((ret_value = H5MF__close_aggrfs(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'aggr' file space") + } /* end else */ - /* Check the last section of each free-space manager */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - haddr_t sect_addr = HADDR_UNDEF; - hsize_t sect_size = 0; - - if(f->shared->fs_man[type]) { - if(H5FS_sect_query_last_sect(f->shared->fs_man[type], §_addr, §_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query last section on merge list") - - /* Deduct space from previous accumulation if the section is at EOA */ - if(H5F_addr_eq(sect_addr + sect_size, eoa)) { - eoa = sect_addr; - eoa_shrank = TRUE; - tot_fs_size -= sect_size; - } /* end if */ - } /* end if */ - } /* end for */ - - /* Check the metadata and raw data aggregators */ - if(ma_size > 0 && H5F_addr_eq(ma_addr + ma_size, eoa)) { - eoa = ma_addr; - eoa_shrank = TRUE; - ma_size = 0; - } /* end if */ - if(sda_size > 0 && H5F_addr_eq(sda_addr + sda_size, eoa)) { - eoa = sda_addr; - eoa_shrank = TRUE; - sda_size = 0; - } /* end if */ - } while(eoa_shrank); +done: - /* Close the free-space managers if they were opened earlier in this routine */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - if(fs_started[type]) { +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__close_delete_fstype + * + * Purpose: Common code for closing and deleting the freespace manager + * of TYPE for file. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi + * Jan 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + HDassert(f->shared); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If the free space manager for this type is open, close it */ + if(f->shared->fs_man[type]) + if(H5MF__close_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If there is free space manager info for this type, delete it */ + if(H5F_addr_defined(f->shared->fs_addr[type])) + if(H5MF__delete_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't delete the free space manager") + +done: +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* H5MF__close_delete() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_try_close + * + * Purpose: This is called by H5Fformat_convert() to close and delete + * free-space managers when downgrading persistent free-space + * to non-persistent. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi + * Jan 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_try_close(H5F_t *f, hid_t dxpl_id) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + /* If there have been no file space allocations / deallocation so + * far, must call H5MF_tidy_self_referential_fsm_hack() to float + * all self referential FSMs and release file space allocated to + * them. Otherwise, the function will be called after the format + * conversion, and will become very confused. + * + * The situation is further complicated if a cache image exists + * and had not yet been loaded into the metadata cache. In this + * case, call H5AC_force_cache_image_load() instead of + * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load() + * will load the cache image, and then call + * H5MF_tidy_self_referential_fsm_hack() to discard the cache image + * block. + */ + if(f->shared->first_alloc_dealloc) { + if(H5AC_cache_image_pending(f)) { + if(H5AC_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed") + } /* end if */ + else { + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end else */ + } /* end if */ + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + if(H5F_PAGED_AGGR(f)) { + H5F_mem_page_t ptype; /* Memory type for iteration */ + + /* Iterate over all the free space types that have managers and + * get each free list's space + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring ) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + } /* end for */ + } /* end if */ + else { + H5FD_mem_t type; /* Memory type for iteration */ + + /* Iterate over all the free space types that have managers and + * get each free list's space + */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { /* test to see if we need to switch rings -- do so if required */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; if(needed_ring != curr_ring) { if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)") + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") curr_ring = needed_ring; } /* end if */ - if(H5MF__alloc_close(f, dxpl_id, type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space") - } /* end if */ - } /* end for */ - - /* Set the value(s) to return */ - /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */ - if(tot_space) - *tot_space = tot_fs_size + ma_size + sda_size; - if(meta_size) - *meta_size = tot_meta_size; + if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + } /* end for */ + } /* end else */ done: /* Reset the ring in the DXPL */ @@ -1089,75 +1807,370 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_get_freespace() */ +} /* H5MF_try_close() */ /*------------------------------------------------------------------------- - * Function: H5MF_try_shrink + * Function: H5MF__close_aggrfs * - * Purpose: Try to shrink the size of a file with a block or absorb it - * into a block aggregator. + * Purpose: Close the free space tracker(s) for a file: non-paged aggregation * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Feb 14 2008 + * Tuesday, January 22, 2008 * *------------------------------------------------------------------------- */ -htri_t -H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, - hsize_t size) +static herr_t +H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id) { - H5MF_free_section_t *node = NULL; /* Free space section pointer */ - H5MF_sect_ud_t udata; /* User data for callback */ - htri_t ret_value = FAIL; /* Return value */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5FD_mem_t type; /* Memory type for iteration */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size); +HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - /* check arguments */ + /* check args */ HDassert(f); HDassert(f->shared); HDassert(f->shared->lf); - HDassert(H5F_addr_defined(addr)); - HDassert(size > 0); + HDassert(f->shared->sblock); - /* Create free space section for block */ - if(NULL == (node = H5MF_sect_simple_new(addr, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */ - udata.allow_eoa_shrink_only = FALSE; - - /* Call the "can shrink" callback for the section */ - if((ret_value = H5MF_sect_simple_can_shrink((const H5FS_section_info_t *)node, &udata)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container") - else if(ret_value > 0) { - /* Shrink or absorb the section */ - if(H5MF_sect_simple_shrink((H5FS_section_info_t **)&node, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container") + /* Free the space in aggregators */ + /* (for space not at EOA, it may be put into free space managers) */ + if(H5MF_free_aggrs(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + + /* Trying shrinking the EOA for the file */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* Making free-space managers persistent for superblock version >= 2 */ + if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 + && f->shared->fs_persist) { + H5O_fsinfo_t fsinfo; /* File space info message */ + haddr_t final_eoa; /* Final eoa -- for sanity check */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + + /* superblock extension and free space manager message should + * exist at this point -- verify at least the former. + */ + HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + + /* file space for all non-empty free space managers should be + * allocated at this point, and these free space managers should + * be written to file and thus their headers and section info + * entries in the metadata cache should be clean. + */ + + /* gather data for the free space manager superblock extension message. + * + * In passing, verify that all the free space managers are closed. + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + fsinfo.fs_addr[type-1] = f->shared->fs_addr[type]; + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + + /* Write the free space manager message -- message must already exist */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + + /* Close the free space managers */ + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(f->shared->fs_man[type]) { + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN); + + if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") + f->shared->fs_man[type] = NULL; + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + } /* end if */ + f->shared->fs_addr[type] = HADDR_UNDEF; + } /* end for */ + + /* verify that we haven't dirtied any metadata cache entries + * from the metadata free space manager ring out. + */ + HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + + /* verify that the aggregators are still shutdown. */ + HDassert(f->shared->sdata_aggr.tot_size == 0); + HDassert(f->shared->sdata_aggr.addr == 0); + HDassert(f->shared->sdata_aggr.size == 0); + + HDassert(f->shared->meta_aggr.tot_size == 0); + HDassert(f->shared->meta_aggr.addr == 0); + HDassert(f->shared->meta_aggr.size == 0); + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* get the eoa, and verify that it has the expected value */ + if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* f->shared->eoa_post_fsm_fsalloc is undefined if there has + * been no file space allocation or deallocation since file + * open. + */ + HDassert((f->shared->first_alloc_dealloc) || (final_eoa == f->shared->eoa_post_fsm_fsalloc)); } /* end if */ + else { /* super_vers can be 0, 1, 2 */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + } /* end else */ + + /* Free the space in aggregators (again) */ + /* (in case any free space information re-started them) */ + if(H5MF_free_aggrs(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - /* Free section node allocated */ - if(node && H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__close_aggrfs() */ + +/*------------------------------------------------------------------------- + * Function: H5MF__close_pagefs + * + * Purpose: Close the free space tracker(s) for a file: paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this + * iteration. + */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + H5O_fsinfo_t fsinfo; /* File space info message */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); +HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_try_shrink() */ + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(f->shared->sblock); + HDassert(f->shared->fs_page_size); + HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2); + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Trying shrinking the EOA for the file */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* Set up file space info message */ + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + + if(f->shared->fs_persist) { + haddr_t final_eoa; /* final eoa -- for sanity check */ + + /* superblock extension and free space manager message should + * exist at this point -- verify at least the former. + */ + HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + + /* file space for all non-empty free space managers should be + * allocated at this point, and these free space managers should + * be written to file and thus their headers and section info + * entries in the metadata cache should be clean. + */ + + /* gather data for the free space manager superblock extension message. + * Only need addresses of FSMs and eoa prior to allocation of + * file space for the self referential free space managers. Other + * data was gathered above. + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype-1] = f->shared->fs_addr[ptype]; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + + /* Write the free space manager message -- message must already exist */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + + /* Close the free space managers */ + /* use H5MF__close_fstype() for this? */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + if(f->shared->fs_man[ptype]) { + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + HDassert(f->shared->fs_state[ptype] == H5F_FS_STATE_OPEN); + + if(H5FS_close(f, dxpl_id, f->shared->fs_man[ptype]) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") + f->shared->fs_man[ptype] = NULL; + f->shared->fs_state[ptype] = H5F_FS_STATE_CLOSED; + } /* end if */ + f->shared->fs_addr[ptype] = HADDR_UNDEF; + } /* end for */ + + /* verify that we haven't dirtied any metadata cache entries + * from the metadata free space manager ring out. + */ + HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* get the eoa, and verify that it has the expected value */ + if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* f->shared->eoa_post_fsm_fsalloc is undefined if there has + * been no file space allocation or deallocation since file + * open. + * + * If there is a cache image in the file at file open, + * f->shared->first_alloc_dealloc will always be FALSE unless + * the file is opened R/O, as otherwise, the image will have been + * read and discarded by this point. + * + * If a cache image was created on file close, the actual EOA + * should be in f->shared->eoa_post_mdci_fsalloc. Note that in + * this case, it is conceivable that f->shared->first_alloc_dealloc + * will still be TRUE, as the cache image is allocated directly from + * the file driver layer. However, as this possibility seems remote, + * it is ignored in the following assert. + */ + HDassert((f->shared->first_alloc_dealloc) || + (final_eoa == f->shared->eoa_post_fsm_fsalloc) || + ((H5F_addr_defined(f->shared->eoa_post_mdci_fsalloc)) && + (final_eoa == f->shared->eoa_post_mdci_fsalloc))); + } /* end if */ + else { + /* Iterate over all the free space types that have managers + * and get each free list's space + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + + /* Write file space info message to superblock extension object header */ + /* Create the superblock extension object header in advance if needed */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + } /* end else */ + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__close_pagefs() */ /*------------------------------------------------------------------------- @@ -1175,17 +2188,16 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) { - H5FD_mem_t type; /* Memory type for iteration */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ - H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this - * iteration. - */ + H5F_mem_t type; + H5F_mem_page_t ptype; /* Memory type for iteration */ hbool_t eoa_shrank; /* Whether an EOA shrink occurs */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ htri_t status; /* Status value */ H5MF_sect_ud_t udata; /* User data for callback */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -1197,13 +2209,10 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) /* Construct user data for callbacks */ udata.f = f; udata.dxpl_id = dxpl_id; - udata.allow_sect_absorb = FALSE; - udata.allow_eoa_shrink_only = TRUE; + udata.allow_sect_absorb = FALSE; + udata.allow_eoa_shrink_only = TRUE; - /* Set the ring type in the DXPL. In most cases, we will - * need H5AC_RING_RDFSM, so initialy set the ring type in - * the DXPL to that value. We will alter this later if needed. - */ + /* Set the ring type in the DXPL */ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(1)") reset_ring = TRUE; @@ -1213,36 +2222,62 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) do { eoa_shrank = FALSE; - /* Check the last section of each free-space manager */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(H5F_PAGED_AGGR(f)) { + /* Check the last section of each free-space manager */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + if(f->shared->fs_man[ptype]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; - } /* end if */ + udata.alloc_type = (H5FD_mem_t)((H5FD_mem_t)ptype < H5FD_MEM_NTYPES ? ptype : ((ptype % H5FD_MEM_NTYPES) + 1)); - if(f->shared->fs_man[type]) { - udata.alloc_type = type; - if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") - else if(status > 0) - eoa_shrank = TRUE; - } /* end if */ - } /* end for */ + if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[ptype], &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end if */ + } /* end for */ + } /* end if */ + else { + /* Check the last section of each free-space manager */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(f->shared->fs_man[type]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + udata.alloc_type = type; - /* check the two aggregators */ - if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") - else if(status > 0) - eoa_shrank = TRUE; + if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end if */ + } /* end for */ + + /* check the two aggregators */ + if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end else */ } while(eoa_shrank); done: @@ -1256,6 +2291,370 @@ done: /*------------------------------------------------------------------------- + * Function: H5MF_get_freespace + * + * Purpose: Retrieve the amount of free space in the file + * + * Return: Success: Amount of free space in file + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, October 6, 2003 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size) +{ + haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ + hsize_t ma_size = 0; /* Size of "metadata aggregator" */ + haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ + hsize_t sda_size = 0; /* Size of "small data aggregator" */ + hsize_t tot_fs_size = 0; /* Amount of all free space managed */ + hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */ + H5FD_mem_t tt; /* Memory type for iteration */ + H5F_mem_page_t type; /* Memory type for iteration */ + H5F_mem_page_t start_type; /* Memory type for iteration */ + H5F_mem_page_t end_type; /* Memory type for iteration */ + htri_t fs_started[H5F_MEM_PAGE_NTYPES]; /* Indicate whether the free-space manager has been started */ + haddr_t fs_eoa[H5FD_MEM_NTYPES]; /* EAO for each free-space manager */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Determine start/end points for loop */ + if(H5F_PAGED_AGGR(f)) { + start_type = H5F_MEM_PAGE_META; + end_type = H5F_MEM_PAGE_NTYPES; + } /* end if */ + else { + start_type = (H5F_mem_page_t)H5FD_MEM_SUPER; + end_type = (H5F_mem_page_t)H5FD_MEM_NTYPES; + } /* end else */ + + for(tt = H5FD_MEM_SUPER; tt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, tt)) + if(HADDR_UNDEF == (fs_eoa[tt] = H5F_get_eoa(f, tt))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + if(!H5F_PAGED_AGGR(f)) { + /* Retrieve metadata aggregator info, if available */ + if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") + + /* Retrieve 'small data' aggregator info, if available */ + if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") + } /* end if */ + + /* Iterate over all the free space types that have managers and get each free list's space */ + for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) { + fs_started[type] = FALSE; + + /* Check if the free space for the file has been initialized */ + if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) { + if(H5MF_open_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + HDassert(f->shared->fs_man[type]); + fs_started[type] = TRUE; + } /* end if */ + + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ + + /* Check if there's free space of this type */ + if(f->shared->fs_man[type]) { + hsize_t type_fs_size = 0; /* Amount of free space managed for each type */ + hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */ + + /* Retrieve free space size from free space manager */ + if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") + if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats") + + /* Increment total free space for types */ + tot_fs_size += type_fs_size; + tot_meta_size += type_meta_size; + } /* end if */ + } /* end for */ + + /* Close the free-space managers if they were opened earlier in this routine */ + for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ + + if(fs_started[type]) + if(H5MF__close_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space") + } /* end for */ + + /* Set the value(s) to return */ + /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */ + if(tot_space) + *tot_space = tot_fs_size + ma_size + sda_size; + if(meta_size) + *meta_size = tot_meta_size; + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_get_freespace() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_get_free_sections() + * + * Purpose: To retrieve free-space section information for + * paged or non-paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +ssize_t +H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + size_t total_sects = 0; /* Total number of sections */ + H5MF_sect_iter_ud_t sect_udata; /* User data for callback */ + H5F_mem_page_t start_type, end_type; /* Memory types to iterate over */ + H5F_mem_page_t ty; /* Memory type for iteration */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + ssize_t ret_value = -1; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, -1) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + + /* H5MF_tidy_self_referential_fsm_hack() will fail if any self + * referential FSM is opened prior to the call to it. Thus call + * it here if necessary and if it hasn't been called already. + * + * The situation is further complicated if a cache image exists + * and had not yet been loaded into the metadata cache. In this + * case, call H5AC_force_cache_image_load() instead of + * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load() + * will load the cache image, and then call + * H5MF_tidy_self_referential_fsm_hack() to discard the cache image + * block. + */ + if(f->shared->first_alloc_dealloc) { + if(H5AC_cache_image_pending(f)) { + if(H5AC_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed") + } /* end if */ + else { + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end else */ + } /* end if */ + + if(type == H5FD_MEM_DEFAULT) { + start_type = H5F_MEM_PAGE_SUPER; + end_type = H5F_MEM_PAGE_NTYPES; + } /* end if */ + else { + start_type = end_type = (H5F_mem_page_t)type; + if(H5F_PAGED_AGGR(f)) /* set to the corresponding LARGE free-space manager */ + end_type = (H5F_mem_page_t)(end_type + H5FD_MEM_NTYPES); + else + H5_INC_ENUM(H5F_mem_page_t, end_type); + } /* end else */ + + /* Set up user data for section iteration */ + sect_udata.sects = sect_info; + sect_udata.sect_count = nsects; + sect_udata.sect_idx = 0; + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(0)") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Iterate over memory types, retrieving the number of sections of each type */ + for(ty = start_type; ty < end_type; H5_INC_ENUM(H5F_mem_page_t, ty)) { + hbool_t fs_started = FALSE; /* The free-space manager is opened or not */ + size_t nums = 0; /* The number of free-space sections */ + + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ty)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) { + if(H5MF_open_fstype(f, dxpl_id, ty) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't open the free space manager") + HDassert(f->shared->fs_man[ty]); + fs_started = TRUE; + } /* end if */ + + /* Check if there's free space sections of this type */ + if(f->shared->fs_man[ty]) + if(H5MF__get_free_sects(f, dxpl_id, f->shared->fs_man[ty], §_udata, &nums) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get section info for the free space manager") + + /* Increment total # of sections */ + total_sects += nums; + + /* Close the free space manager of this type, if we started it here */ + if(fs_started) + if(H5MF__close_fstype(f, dxpl_id, ty) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space") + if((H5F_PAGED_AGGR(f)) && (type != H5FD_MEM_DEFAULT)) + ty = (H5F_mem_page_t)(ty + H5FD_MEM_NTYPES - 2); + } /* end for */ + + /* Set return value */ + ret_value = (ssize_t)total_sects; + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, -1) +} /* H5MF_get_free_sections() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sects_cb() + * + * Purpose: Iterator callback for each free-space section + * Retrieve address and size into user data + * + * Return: Always succeed + * + * Programmer: Vailin Choi + * July 1st, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata) +{ + H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; + H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(udata->sect_idx < udata->sect_count) { + udata->sects[udata->sect_idx].addr = sect->sect_info.addr; + udata->sects[udata->sect_idx].size = sect->sect_info.size; + udata->sect_idx++; + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sects_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__get_free_sects + * + * Purpose: Retrieve section information for the specified free-space manager. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums) +{ + hsize_t hnums = 0; /* # of sections */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(f); + HDassert(sect_udata); + HDassert(nums); + HDassert(fspace); + + /* Query how many sections of this type */ + if(H5FS_sect_stats(fspace, NULL, &hnums) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") + H5_CHECKED_ASSIGN(*nums, size_t, hnums, hsize_t); + + /* Check if we should retrieve the section info */ + if(sect_udata->sects && *nums > 0) + /* Iterate over all the free space sections of this type, adding them to the user's section info */ + if(H5FS_sect_iterate(f, dxpl_id, fspace, H5MF_sects_cb, sect_udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF__get_free_sects() */ + + +/*------------------------------------------------------------------------- * Function: H5MF_settle_raw_data_fsm() * * Purpose: Handle any tasks required before the metadata cache @@ -1263,19 +2662,48 @@ done: * and any metadata free space managers that reside in the * raw data free space manager ring. * - * Specifically, any metadata managers that DON'T handle - * space allocation for free space manager header or section - * info will reside in the raw data free space manager ring. - * As of this writing, the plan is to move to only two free space - * managers, one for raw data and one for metadata -- which - * means that only the raw data free space manager will reside - * in the free space manager ring. However, this has not been - * fully implemented yet, so this code must support the - * possibilty of multiple metadata free space managers, at most - * two of which handle free space manager header or section info, - * and thus reside in the metadata free space manager ring. - * - * At present, the task list is: + * Specifically, this means any metadata managers that DON'T + * handle space allocation for free space manager header or + * section info will reside in the raw data free space manager + * ring. + * + * In the absence of page allocation, there is at most one + * free space manager per memory type defined in H5F_mem_t. + * Of these, the one that allocates H5FD_MEM_DRAW will + * always reside in the raw data free space manager ring. + * If there is more than one metadata free space manager, + * all that don't handle H5FD_MEM_FSPACE_HDR or + * H5FD_MEM_FSPACE_SINFO (which map to H5FD_MEM_OHDR and + * H5FD_MEM_LHEAP respectively) will reside in the raw + * data free space manager ring as well + * + * With page allocation, the situation is conceptually + * identical, but more complex in practice. + * + * In the worst case (multi file driver) page allocation + * can result in two free space managers for each memory + * type -- one for small (less than on equal to one page) + * allocations, and one for large (greater than one page) + * allocations. + * + * In the more common one file case, page allocation will + * result in a total of three free space managers -- one for + * small (<= one page) raw data allocations, one for small + * metadata allocations (i.e, all memory types other than + * H5FD_MEM_DRAW), and one for all large (> one page) + * allocations. + * + * Despite these complications, the solution is the same in + * the page allocation case -- free space managers (be they + * small data or large) are assigned to the raw data free + * space manager ring if they don't allocate file space for + * free space managers. Note that in the one file case, the + * large free space manager must be assigned to the metadata + * free space manager ring, as it both allocates pages for + * the metadata free space manager, and allocates space for + * large (> 1 page) metadata cache entries. + * + * At present, the task list for this routine is: * * 1) Reduce the EOA to the extent possible. To do this: * @@ -1330,26 +2758,43 @@ done: herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) { + int pass_count; + hsize_t alloc_size; + H5F_mem_t mem_type; /* Memory type for iteration */ + H5F_mem_page_t fsm_type; /* FSM type for iteration */ + H5O_fsinfo_t fsinfo; /* Free space manager info message */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) - /* check args */ + /* Check args */ HDassert(f); HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info */ - if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5O_fsinfo_t fsinfo; /* Free space manager info message */ - H5FD_mem_t type; /* Memory type for iteration */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ - H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ - hbool_t fsm_opened[H5FD_MEM_NTYPES]; /* State of FSM */ - hbool_t fsm_visited[H5FD_MEM_NTYPES]; /* State of FSM */ + /* Only need to settle things if we are persisting the free space info + * and allocation/deallocation has occurred. + */ + if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + hbool_t fsm_opened[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ + hbool_t fsm_visited[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ + + /* Sanity check */ + HDassert(f->shared->sblock); + + /* should only be called if file is opened R/W */ + HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + + /* shouldn't be called unless we have a superblock supporting the + * superblock extension. + */ + HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2); /* Initialize fsm_opened and fsm_visited */ HDmemset(fsm_opened, 0, sizeof(fsm_opened)); @@ -1366,8 +2811,12 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * * Note that while the raw data aggregator should not be restarted during * the close process, this need not be the case for the metadata aggregator. + * + * Note also that the aggregators will not exist if page aggregation + * is enabled -- skip this if so. */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + /* Vailin -- is this correct? */ + if(!H5F_PAGED_AGGR(f) && (H5MF_free_aggrs(f, dxpl_id) < 0)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators") /* Set the ring type in the DXPL. In most cases, we will @@ -1389,7 +2838,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * In the case of the raw data free space manager, and any other free * space manager that does not allocate space for free space managers, * allocations should be complete at this point, as all raw data should - * have space allocated and be flushed to file at this point. Thus we + * have space allocated and be flushed to file by now. Thus we * can examine such free space managers and only re-allocate space for * them if they contain free space. Do this later in this function after * the EOA has been reduced to the extent possible. @@ -1400,79 +2849,100 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * are possible), the matter is more ticklish due to the self- * referential nature of the problem. These FSMs are dealt with in * H5MF_settle_meta_data_fsm(). + * + * Since paged allocation may be enabled, there may be up to two + * free space managers per memory type -- one for small and one for + * large allocation. Hence we must loop over the memory types twice + * setting the allocation size accordingly if paged allocation is + * enabled. */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5FS_stat_t fs_stat; /* Information for free-space manager */ - H5FD_mem_t fsm_type; /* File memory type for FSM */ - - /* There is potentially a many-to-one mapping from memory types to - * free space managers. Use the fsm_visited[] array to avoid visiting - * a given FSM more than once. Use fsm_opened[] to track which FSMs - * must be closed at the end of this function. - */ - fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type); - if(!fsm_visited[fsm_type]) { - fsm_visited[fsm_type] = TRUE; - - /* If there is no active FSM for this type, but such a FSM has - * space allocated in file, open it so that we can free its file - * space. - */ - if(NULL == f->shared->fs_man[fsm_type]) { - if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) { - /* Sanity check */ - HDassert(fsm_opened[fsm_type] == FALSE); - - /* Start up FSM for the file memory type */ - if(H5MF__alloc_open(f, dxpl_id, fsm_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - fsm_opened[fsm_type] = TRUE; - } /* end if */ + for(pass_count = 0; pass_count <= 1; pass_count++) { + if(pass_count == 0) + alloc_size = 1; + else if ( H5F_PAGED_AGGR(f) ) + alloc_size = f->shared->fs_page_size + 1; + else /* no need for a second pass */ + break; + + for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { + H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + + if(pass_count == 0) { /* this is the first pass */ + HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER); } /* end if */ + else if(H5F_PAGED_AGGR(f)) { /* page alloc active */ + HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); + } /* end else-if */ + else /* paged allocation disabled -- should be unreachable */ + HDassert(FALSE); + + if(!fsm_visited[fsm_type]) { + fsm_visited[fsm_type] = TRUE; + + /* If there is no active FSM for this type, but such a FSM has + * space allocated in file, open it so that we can free its file + * space. + */ + if(NULL == f->shared->fs_man[fsm_type]) { + if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) { + /* Sanity check */ + HDassert(fsm_opened[fsm_type] == FALSE); - /* Check for an actual FSM for this type now */ - /* (Possibly opened in previous step) */ - if(f->shared->fs_man[fsm_type]) { - /* Test to see if we need to switch rings -- do so if required */ - if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - curr_ring = needed_ring; + if(H5MF_open_fstype(f, dxpl_id, fsm_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + fsm_opened[fsm_type] = TRUE; + } /* end if */ } /* end if */ - /* Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + if(f->shared->fs_man[fsm_type]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ - /* Check if the free space manager has space in the file */ - if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) { - /* Delete the free space manager in the file. Will - * reallocate later if the free space manager contains - * any free space. - */ - if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") - f->shared->fs_addr[fsm_type] = HADDR_UNDEF; + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + /* Check if the free space manager has space in the file */ + if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) { + /* Delete the free space manager in the file. Will + * reallocate later if the free space manager contains + * any free space. + */ + if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id, TRUE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[fsm_type] = HADDR_UNDEF; + } /* end if */ } /* end if */ - } /* end if */ - /* note that we are tracking opened FSM -- we will close them - * at the end of the function. - */ - } /* end if */ + /* note that we are tracking opened FSM -- we will close them + * at the end of the function. + */ + } /* end if */ + } /* end for */ } /* end for */ + /* c) Delete the free space manager superblock extension message * if allocated. * * Must do this since the routine that writes / creates superblock * extension messages will choke if the target message is * unexpectedly either absent or present. + * + * Update: This is probably unecessary, as I gather that the + * file space manager info message is guaranteed to exist. + * Leave it in for now, but consider removing it. */ if(H5F_addr_defined(f->shared->sblock->ext_addr)) if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0) @@ -1496,11 +2966,16 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * those addresses are unknown. This is OK -- we will write the correct * values to the message at free space manager shutdown. */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type - 1] = HADDR_UNDEF; + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + fsinfo.fs_addr[fsm_type - 1] = HADDR_UNDEF; fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; fsinfo.threshold = f->shared->fs_threshold; - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_NO_FLAGS_SET) < 0) + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") @@ -1524,102 +2999,117 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) */ /* Reinitialize fsm_visited */ - HDmemset(fsm_visited, 0, sizeof(fsm_visited)); - - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5FD_mem_t fsm_type; /* File memory type for FSM */ - - fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type); - - /* test to see if we need to switch rings -- do so if required */ - if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - curr_ring = needed_ring; - } /* end if */ - - /* Since there can be a many-to-one mapping from memory types - * to free space managers, ensure that we don't visit any FSM - * more than once. - */ - if(!fsm_visited[fsm_type]) { - fsm_visited[fsm_type] = TRUE; - - if(f->shared->fs_man[fsm_type]) { - /* Only allocate file space if the target free space manager - * doesn't allocate file space for free space managers. Note - * that this is also the deciding factor as to whether a FSM - * in in the raw data FSM ring. - */ - if((fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - && (fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) { - H5FS_stat_t fs_stat; /* Information for free-space manager */ + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + fsm_visited[fsm_type] = FALSE; + + for(pass_count = 0; pass_count <= 1; pass_count++) { + if(pass_count == 0) + alloc_size = 1; + else if(H5F_PAGED_AGGR(f)) + alloc_size = f->shared->fs_page_size + 1; + else /* no need for a second pass */ + break; + + for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { + H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + + if(pass_count == 0) { /* this is the first pass */ + HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER); + } /* end if */ + else if(H5F_PAGED_AGGR(f)) { /* page alloc active */ + HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); + } /* end else-if */ + else /* paged allocation disabled -- should be unreachable */ + HDassert(FALSE); - /* The current ring should be H5AC_RING_RDFSM */ - HDassert(curr_ring == H5AC_RING_RDFSM); + /* test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; - /* Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 ) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ - /* If the free space manager contains section info, - * allocate space for the header and sinfo (note that - * space must not be allocated at present -- verify - * verify this with assertions). + /* Since there can be a many-to-one mapping from memory types + * to free space managers, ensure that we don't visit any FSM + * more than once. + */ + if(!fsm_visited[fsm_type]) { + fsm_visited[fsm_type] = TRUE; + + if(f->shared->fs_man[fsm_type]) { + /* Only allocate file space if the target free space manager + * doesn't allocate file space for free space managers. Note + * that this is also the deciding factor as to whether a FSM + * in in the raw data FSM ring. */ - if(fs_stat.serial_sect_count > 0) { - /* Sanity check */ - HDassert(!H5F_addr_defined(fs_stat.addr)); - - /* Allocate FSM header */ - if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header") - - /* Allocate FSM section info */ - HDassert(!H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.alloc_sect_size == 0); - if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 ) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info") + if(!H5MF__fsm_type_is_self_referential(f, fsm_type)) { + /* The current ring should be H5AC_RING_RDFSM */ + HDassert(curr_ring == H5AC_RING_RDFSM); + + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + + /* If the free space manager contains section info, + * allocate space for the header and sinfo (note that + * space must not be allocated at present -- verify + * verify this with assertions). + */ + if(fs_stat.serial_sect_count > 0) { + /* Sanity check */ + HDassert(!H5F_addr_defined(fs_stat.addr)); + + /* Allocate FSM header */ + if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header") + + /* Allocate FSM section info */ + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info") #ifndef NDEBUG - /* Re-Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - - HDassert(H5F_addr_defined(fs_stat.addr)); - HDassert(H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.serial_sect_count > 0); - HDassert(fs_stat.alloc_sect_size > 0); - HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size); + /* Re-Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.serial_sect_count > 0); + HDassert(fs_stat.alloc_sect_size > 0); + HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size); #endif /* NDEBUG */ + } /* end if */ + else { + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.serial_sect_count == 0); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end else */ } /* end if */ - else { - HDassert(!H5F_addr_defined(fs_stat.addr)); - HDassert(!H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.serial_sect_count == 0); - HDassert(fs_stat.alloc_sect_size == 0); - } /* end else */ } /* end if */ - } /* end if */ - /* Close any opened FSMs */ - if(fsm_opened[fsm_type]) { - if(H5MF__alloc_close(f, dxpl_id, fsm_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager") - fsm_opened[fsm_type] = FALSE; + /* Close any opened FSMs */ + if(fsm_opened[fsm_type]) { + if(H5MF__close_fstype(f, dxpl_id, fsm_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager") + fsm_opened[fsm_type] = FALSE; + } /* end if */ } /* end if */ - } /* end if */ + } /* end for */ } /* end for */ /* verify that all opened FSMs were closed */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - HDassert(!fsm_opened[type]); + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + HDassert(!fsm_opened[fsm_type]); /* Indicate that the FSM was settled successfully */ *fsm_settled = TRUE; @@ -1640,11 +3130,11 @@ done: * * Purpose: If the free space manager is persistent, handle any tasks * required before the metadata cache can serialize or flush - * the metadata free space manager(sI) that handle file space + * the metadata free space manager(s) that handle file space * allocation for free space managers. * * In most cases, there will be only one manager assigned - * to this role. However, since for reason or reason unknown, + * to this role. However, since for reasons unknown, * free space manager headers and section info blocks are * different classes of memory, it is possible that two free * space managers will be involved. @@ -1652,7 +3142,7 @@ done: * On entry to this function, the raw data settle routine * (H5MF_settle_raw_data_fsm()) should have: * - * 1) Freed the accumulators. + * 1) Freed the aggregators. * * 2) Freed all file space allocated to the free space managers. * @@ -1688,9 +3178,12 @@ done: * 1) Verify that the free space manager(s) involved in file * space allocation for free space managers are still floating. * - * 2) Free the accumulators. + * 2) Free the aggregators. * - * 3) Reduce the EOA to the extent possible. + * 3) Reduce the EOA to the extent possible, and make note + * of the resulting value. This value will be stored + * in the fsinfo superblock extension message and be used + * in the subsequent file open. * * 4) Re-allocate space for any free space manager(s) that: * @@ -1724,6 +3217,9 @@ done: * severe time pressure at the moment, the above brute * force solution is attractive. * + * 5) Make note of the EOA -- used for sanity checking on + * FSM shutdown. + * * Return: SUCCEED/FAIL * * Programmer: John Mainzer @@ -1734,10 +3230,23 @@ done: herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) { + H5F_mem_page_t sm_fshdr_fs_type; /* small fs hdr fsm */ + H5F_mem_page_t sm_fssinfo_fs_type; /* small fs sinfo fsm */ + H5F_mem_page_t lg_fshdr_fs_type; /* large fs hdr fsm */ + H5F_mem_page_t lg_fssinfo_fs_type; /* large fs sinfo fsm */ + H5FS_t *sm_hdr_fspace = NULL; /* ptr to sm FSM hdr alloc FSM */ + H5FS_t *sm_sinfo_fspace = NULL; /* ptr to sm FSM sinfo alloc FSM */ + H5FS_t *lg_hdr_fspace = NULL; /* ptr to lg FSM hdr alloc FSM */ + H5FS_t *lg_sinfo_fspace = NULL; /* ptr to lg FSM sinfo alloc FSM */ + haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + /* for self referential FSMs */ + H5FS_stat_t fs_stat; /* Information for hdr FSM */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ hbool_t reset_ring = FALSE; /* Whether we set the ring */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -1746,29 +3255,49 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info */ - if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5FS_t *hdr_fspace; /* Ptr to FSM hdr alloc FSM */ - H5FS_t *sinfo_fspace; /* Ptr to FSM sinfo alloc FSM */ - H5FS_stat_t fs_stat; /* Information for FSM */ - H5FD_mem_t hdr_fsm_alloc_type; /* FSM hdr alloc type */ - H5FD_mem_t sinfo_fsm_alloc_type; /* FSM info alloc type */ - - /* Get the file memory types for the FSM header & section info */ - hdr_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR); - sinfo_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO); - - /* Sanity checks */ - HDassert(hdr_fsm_alloc_type > H5FD_MEM_NOLIST); - HDassert(hdr_fsm_alloc_type < H5FD_MEM_NTYPES); - HDassert(sinfo_fsm_alloc_type > H5FD_MEM_NOLIST); - HDassert(sinfo_fsm_alloc_type < H5FD_MEM_NTYPES); - HDassert(!H5F_addr_defined(f->shared->fs_addr[hdr_fsm_alloc_type])); - HDassert(!H5F_addr_defined(f->shared->fs_addr[sinfo_fsm_alloc_type])); - - /* Note that in most cases, hdr_fspace will equal sinfo_fspace. */ - hdr_fspace = f->shared->fs_man[hdr_fsm_alloc_type]; - sinfo_fspace = f->shared->fs_man[sinfo_fsm_alloc_type]; + /* Only need to settle things if we are persisting the free space info + * and allocation/deallocation has occurred. + */ + if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + /* Sanity check */ + HDassert(f->shared->lf); + + /* should only be called if file is opened R/W */ + HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); + + HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); + + HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER); + + HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])); + HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type])); + + /* Note that in most cases, sm_hdr_fspace will equal sm_sinfo_fspace. */ + sm_hdr_fspace = f->shared->fs_man[sm_fshdr_fs_type]; + sm_sinfo_fspace = f->shared->fs_man[sm_fssinfo_fs_type]; + + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); + + HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); + + HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES); + + HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])); + HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])); + + /* Note that in most cases, lg_hdr_fspace will equal lg_sinfo_fspace. */ + lg_hdr_fspace = f->shared->fs_man[lg_fshdr_fs_type]; + lg_sinfo_fspace = f->shared->fs_man[lg_fssinfo_fs_type]; + } /* end if */ /* Set the ring in the dxpl appropriately for subsequent calls */ if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0) @@ -1776,41 +3305,109 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) reset_ring = TRUE; #ifndef NDEBUG - /* Verify that hdr_fspace is floating if it exists */ - if(hdr_fspace) { + /* Verify that sm_hdr_fspace is floating if it exists */ + if(sm_hdr_fspace) { /* Query free space manager info for this type */ - if(H5FS_stat_info(f, hdr_fspace, &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + if(H5FS_stat_info(f, sm_hdr_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") HDassert(!H5F_addr_defined(fs_stat.addr)); HDassert(!H5F_addr_defined(fs_stat.sect_addr)); HDassert(fs_stat.alloc_sect_size == 0); } /* end if */ - /* Verify that sinfo_fspace is floating if it exists */ - if((sinfo_fspace) && (hdr_fspace != sinfo_fspace)) { + /* Verify that sm_sinfo_fspace is floating if it exists and is distinct */ + if((sm_sinfo_fspace) && (sm_hdr_fspace != sm_sinfo_fspace)) { /* Query free space manager info for this type */ - if(H5FS_stat_info(f, sinfo_fspace, &fs_stat) < 0) + if(H5FS_stat_info(f, sm_sinfo_fspace, &fs_stat) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") HDassert(!H5F_addr_defined(fs_stat.addr)); HDassert(!H5F_addr_defined(fs_stat.sect_addr)); HDassert(fs_stat.alloc_sect_size == 0); } /* end if */ + + if(H5F_PAGED_AGGR(f)) { + /* Verify that lg_hdr_fspace is floating if it exists */ + if(lg_hdr_fspace) { + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, lg_hdr_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (3)") + + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end if */ + + /* Verify that lg_sinfo_fspace is floating if it + * exists and is distinct + */ + if((lg_sinfo_fspace) && (lg_hdr_fspace != lg_sinfo_fspace)) { + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, lg_sinfo_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (4)") + + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end if */ + } /* end if */ #endif /* NDEBUG */ /* Free the space in the metadata aggregator. Do this via the * H5MF_free_aggrs() call. Note that the raw data aggregator must * have already been freed. Sanity checks for this? + * + * Note that the aggregators will not exist if paged aggregation + * is enabled -- don't attempt to free if this is the case. */ + /* Vailin -- is this correct? */ /* (for space not at EOF, it may be put into free space managers) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + if((!H5F_PAGED_AGGR(f)) && (H5MF_free_aggrs(f, dxpl_id) < 0)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators") /* Trying shrinking the EOA for the file */ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + /* At this point, the EOA should be set to a value that contains + * the allocation for all user data, all non self referential FSMs, + * the superblock and all superblock extension messages. + * + * Make note of the current EOA. We will store this value in the + * free space manager superblock extension message. Since space for + * everything other than the self referential FSMs (and possibly the + * cache image) has been allocated at this point, this allows us to + * to float the self referential FSMs on the first file space allocation / + * deallocaiton and then set the EOA to this value before we handle + * the allocation / deallocation. (If a cache image exists, the + * first allocation / deallocation will be the deallocation of space + * for the cache image). + * + * WARNING: This approach settling the self referential free space + * managers and allocating space for them in the file will + * not work as currently implemented with the split and + * multi file drivers, as the self referential free space + * manager header and section info can be stored in up to + * two different files -- requiring that up to two EOA's + * be stored in the the free space managers super block + * extension message. + * + * As of this writing, we are solving this problem by + * simply not supporting persistant FSMs with the split + * and multi file drivers. + * + * Current plans are to do away with the multi file + * driver, so this should be a non-issue in this case. + * + * We should be able to support the split file driver + * without a file format change. However, the code to + * do so does not exist at present. + */ + if(HADDR_UNDEF == (eoa_pre_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + + /* ******************* PROBLEM: ******************** * * If the file has an alignement other than 1, and if @@ -1859,15 +3456,39 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * This is isn't good, but due to schedule pressure, we will just drop * the fragment on the floor for now. */ - if(hdr_fspace) - if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, hdr_fspace, - &(f->shared->fs_addr[hdr_fsm_alloc_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate hdr FSM file space") + if(sm_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space") + + if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space") + + if(H5F_PAGED_AGGR(f)) { + if(lg_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space") + + if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space") + } /* end if */ - if(sinfo_fspace && (sinfo_fspace != hdr_fspace)) - if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, sinfo_fspace, - &(f->shared->fs_addr[sinfo_fsm_alloc_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sinfo FSM file space") + /* Get the eoa after allocation of file space for the self referential + * free space managers. Assuming no cache image, this should be the + * final EOA of the file. + */ + if(HADDR_UNDEF == (eoa_post_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* All free space managers should have file space allocated for them + * now, and should see no further allocations / deallocations. Store + * the pre and post file space allocaton for self referential FSMs EOA + * for use when we actually write the free space manager superblock + * extension message. + */ + f->shared->eoa_pre_fsm_fsalloc = eoa_pre_fsm_fsalloc; + f->shared->eoa_post_fsm_fsalloc = eoa_post_fsm_fsalloc; /* Indicate that the FSM was settled successfully */ *fsm_settled = TRUE; @@ -1878,479 +3499,410 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* H5MF_settle_meta_data_fsm() */ /*------------------------------------------------------------------------- - * Function: H5MF__close_delete + * Function: H5MF__fsm_type_is_self_referential() * - * Purpose: Common code for closing and deleting freespace managers from - * the file. + * Purpose: Return TRUE if the indicated free space manager allocates + * file space for free space managers. Return FALSE otherwise. * - * Return: SUCCEED/FAIL + * Return: TRUE/FALSE * - * Programmer: Vailin Choi - * Jan 2016 + * Programmer: John Mainzer + * 12/6/16 * *------------------------------------------------------------------------- */ -static herr_t -H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl) +hbool_t +H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type) { - H5FD_mem_t type; /* Memory type for iteration */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ - herr_t ret_value = SUCCEED; /* Return value */ + H5F_mem_page_t sm_fshdr_fsm; + H5F_mem_page_t sm_fssinfo_fsm; + H5F_mem_page_t lg_fshdr_fsm; + H5F_mem_page_t lg_fssinfo_fsm; + hbool_t result = FALSE; - FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ + FUNC_ENTER_NOAPI_NOINIT_NOERR - /* check args */ + /* Sanity check */ HDassert(f); HDassert(f->shared); + HDassert(fsm_type >= H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); - /* Iterate over all the free space types that have managers and get each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; + result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm) + || (fsm_type == lg_fshdr_fsm) || (fsm_type == lg_fssinfo_fsm); + } /* end if */ + else { + /* In principle, fsm_type should always be less than + * H5F_MEM_PAGE_LARGE_SUPER whenever paged aggregation + * is not enabled. However, since there is code that does + * not observe this prinicple, force the result to FALSE if + * fsm_type is greater than or equal to H5F_MEM_PAGE_LARGE_SUPER. + */ + if(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER) + result = FALSE; else - needed_ring = H5AC_RING_RDFSM; - - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (4)") - curr_ring = needed_ring; - } /* end if */ - - /* If the free space manager for this type is open, close it */ - if(f->shared->fs_man[type]) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info") - f->shared->fs_man[type] = NULL; - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - } /* end if */ -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* If there is free space manager info for this type, delete it */ - if(H5F_addr_defined(f->shared->fs_addr[type])) { - haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ - - /* Put address into temporary variable and reset it */ - /* (Avoids loopback in file space freeing routine) */ - tmp_fs_addr = f->shared->fs_addr[type]; - f->shared->fs_addr[type] = HADDR_UNDEF; - - /* Shift to "deleting" state, to make certain we don't track any - * file space freed as a result of deleting the free space manager. - */ - f->shared->fs_state[type] = H5F_FS_STATE_DELETING; - -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* Delete free space manager for this type */ - if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager") - - /* Shift [back] to closed state */ - HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - - /* Sanity check that the free space manager for this type wasn't started up again */ - HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); - } /* end if */ - } /* end for */ + result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm); + } /* end else */ -done: -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5MF__close_delete() */ + FUNC_LEAVE_NOAPI(result) +} /* H5MF__fsm_type_is_self_referential() */ /*------------------------------------------------------------------------- - * Function: H5MF_try_close + * Function: H5MF__fsm_is_self_referential() * - * Purpose: This is called by H5Fformat_convert() to close and delete - * free-space managers when downgrading persistent free-space - * to non-persistent. + * Purpose: Return TRUE if the indicated free space manager allocates + * file space for free space managers. Return FALSE otherwise. * - * Return: SUCCEED/FAIL + * Return: TRUE/FALSE * - * Programmer: Vailin Choi - * Jan 2016 + * Programmer: John Mainzer + * 12/6/16 * *------------------------------------------------------------------------- */ -herr_t -H5MF_try_close(H5F_t *f, hid_t dxpl_id) +static hbool_t +H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5F_mem_page_t sm_fshdr_fsm; + H5F_mem_page_t sm_fssinfo_fsm; + hbool_t result = FALSE; - FUNC_ENTER_NOAPI(FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ + FUNC_ENTER_STATIC_NOERR - /* check args */ + /* Sanity check */ HDassert(f); + HDassert(f->shared); + HDassert(fspace); - /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - reset_ring = TRUE; + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); - /* Close and delete freespace managers from the file */ - if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to close delete free-space managers") + if(H5F_PAGED_AGGR(f)) { + H5F_mem_page_t lg_fshdr_fsm; + H5F_mem_page_t lg_fssinfo_fsm; -done: - /* Reset the ring in the DXPL */ - if(reset_ring) - if(H5AC_reset_ring(dxpl, orig_ring) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_try_close() */ + result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || + (fspace == f->shared->fs_man[sm_fssinfo_fsm]) || + (fspace == f->shared->fs_man[lg_fshdr_fsm]) || + (fspace == f->shared->fs_man[lg_fssinfo_fsm]); + } /* end if */ + else + result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || + (fspace == f->shared->fs_man[sm_fssinfo_fsm]); + + FUNC_LEAVE_NOAPI(result) +} /* H5MF__fsm_is_self_referential() */ /*------------------------------------------------------------------------- - * Function: H5MF_close - * - * Purpose: Close the free space tracker(s) for a file + * Function: H5MF_tidy_self_referential_fsm_hack + * + * Purpose: As discussed in the comments of the settle routines above, + * the existence of self referential free space managers + * as currently implemented creates the possibility of + * infinite loops at file close. + * + * As a hack to avoid this, we have added code to settle + * self referential free space managers, and then allocate + * space for them directly from the file driver. + * + * To avoid dropping ever increasing amounts of file space + * on the floor with each subsequent file close/open cycle, + * we need to clean this up on file open. To avoid this, + * this function is called on the first file space allocation + * or deallocation after file open to float the self referential + * free space managers and reduce the EOA to the value it + * had before the direct allocation of space for the self + * referential free space managers. + * + * The function proceeds as follows: + * + * 1) Verify that f->shared->first_alloc_dealloc is TRUE, + * and then set it to FALSE. + * + * 2) Get the current EOA. Verify that it is greater than + * or equal to f->shared->eoa_pre_fsm_fsalloc. If the + * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, + * no self referential FSMs were stored, and we are done. + * + * NOTE: This will have to be reworked somewhat for + * cache image. + * + * 3) Load the self referential FSMs. In passing verify that + * the lowest address of a FSM header is equal to + * f->shared->eoa_pre_fsm_fsalloc.' + * + * Note that we don't have to use any special I/O for + * this -- we can use the regular I/O methods even if + * paged aggregation and page buffering is enabled. + * + * 4) Float the FSMs. Ensure that the file space is NOT + * released. + * + * 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, + * and then set f->shared->eoa_pre_fsm_fsalloc to + * HADDR_UNDEF. + * + * If page buffering, verify that the new EOA is + * on a page boundary, and expunge any pages in the + * page buffer after the new EOA. + * + * Note that this function is also called from test code + * when it is necessary to startup a self referential + * free space manager prior to the first file space + * allocation / deallocation. Failure to do so will + * result in assertion failures in this function on + * the first file space allocation / deallocation. + * + * Return: SUCCEED/FAIL * - * Return: SUCCEED/FAIL - * - * Programmer: Quincey Koziol - * Tuesday, January 22, 2008 + * Programmer: John Mainzer + * 12/11/16 * - * Modifications: - * Vailin Choi; July 2012 - * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY, - * modifications are needed to shrink EOA if the last section of each free-space manager - * and the remaining space in the two aggregators are at EOA. - *------------------------------------------------------------------------- */ herr_t -H5MF_close(H5F_t *f, hid_t dxpl_id) +H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id) { + haddr_t eoa; /* EOA of file */ + hsize_t tail_size = 0; /* Size of chunk to free */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t type; /* Memory type for iteration */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + haddr_t first_srfsm_hdr = HADDR_UNDEF; /* Addr of first self referential */ + /* fsm header in file */ + H5FS_stat_t fs_stat; /* Information for hdr FSM */ + H5F_mem_page_t sm_fshdr_fs_type; /* Small fs hdr fsm */ + H5F_mem_page_t sm_fssinfo_fs_type; /* Small fs sinfo fsm */ + H5F_mem_page_t lg_fshdr_fs_type; /* Large fs hdr fsm */ + H5F_mem_page_t lg_fssinfo_fs_type; /* Large fs sinfo fsm */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ /* check args */ HDassert(f); HDassert(f->shared); - HDassert(f->shared->lf); - HDassert(f->shared->sblock); + HDassert(f->shared->fs_persist); + HDassert(f->shared->first_alloc_dealloc); - /* Set the ring type in the DXPL. In most cases, we will - * need H5AC_RING_RDFSM, so initialy set the ring in - * the DXPL to that value. We will alter this later if - * needed. + /* Set the ring type in the DXPL. Since we are only dealing with + * self referential FSMs, we will only need H5AC_RING_MDFSM. */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - curr_ring = H5AC_RING_RDFSM; - /* Free the space in aggregators */ - /* (for space not at EOF, it may be put into free space managers) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + /* 1) Verify that f->shared->first_alloc_dealloc is TRUE, + * and then set it to FALSE. + */ + HDassert(f->shared->first_alloc_dealloc); + f->shared->first_alloc_dealloc = FALSE; - /* Trying shrinking the EOA for the file */ - if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") - /* Making free-space managers persistent for superblock version >= 2 */ - if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 - && f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5O_fsinfo_t fsinfo; /* Free space manager info message */ + /* 2) Get the current EOA. Verify that it is greater than + * or equal to f->shared->eoa_pre_fsm_fsalloc. If the + * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, + * no self referential FSMs were stored, and we are done. + * + * NOTE: This will have to be reworked somewhat for + * cache image. + */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + HDassert(H5F_addr_le(f->shared->eoa_pre_fsm_fsalloc, eoa)); - /* Superblock extension and free space manager message should - * exist at this point -- verify at least the former. - */ - HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + if(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)) + HGOTO_DONE(SUCCEED) - /* Note that unlike the previous version of this code, we do not - * delete free space managers that have no section to store. - * - * Can't do this, as that would involve freeing file space, which would - * dirty the free space manager in question. - * - * Fortunately, the code doesn't seem to care about this. - */ - /* Gather data for the free space manager superblock extension message. - * - * In passing, verify that all the free space managers are closed. - */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type - 1] = f->shared->fs_addr[type]; - fsinfo.strategy = f->shared->fs_strategy; - fsinfo.threshold = f->shared->fs_threshold; + /* 3) Load the self referential FSMs. In passing verify that + * the lowest address of a FSM header is equal to + * f->shared->eoa_pre_fsm_fsalloc.' + * + * Note that we don't have to use any special I/O for + * this -- we can use the regular I/O methods even if + * paged aggregation and page buffering is enabled. + */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); + HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - /* Write the free space manager message -- message must already exist */ - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - /* Final close of free-space managers */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - if(f->shared->fs_man[type]) { - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ + HDassert(NULL == f->shared->fs_man[sm_fshdr_fs_type]); + HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])) { + first_srfsm_hdr = f->shared->fs_addr[sm_fshdr_fs_type]; - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)") - curr_ring = needed_ring; - } /* end if */ + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, sm_fshdr_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN); - if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") - f->shared->fs_man[type] = NULL; - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - } /* end if */ - f->shared->fs_addr[type] = HADDR_UNDEF; - } /* end for */ + HDassert(f->shared->fs_man[sm_fshdr_fs_type]); + } /* end if */ - /* Verify that we haven't dirtied any metadata cache entries - * from the metadata free space manager ring out. - */ - HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + if((sm_fshdr_fs_type != sm_fssinfo_fs_type) && + (H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type]))) { - /* Verify that the aggregators are still shutdown. */ - HDassert(f->shared->sdata_aggr.tot_size == 0); - HDassert(f->shared->sdata_aggr.addr == 0); - HDassert(f->shared->sdata_aggr.size == 0); - HDassert(f->shared->meta_aggr.tot_size == 0); - HDassert(f->shared->meta_aggr.addr == 0); - HDassert(f->shared->meta_aggr.size == 0); - } /* end if */ - else { /* super_vers can be 0, 1, 2 */ - /* Close and delete freespace managers from the file */ - if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[sm_fssinfo_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[sm_fssinfo_fs_type]; - /* moved code that was for both the persistant and non persistant free - * space managers to the non-persistant case. In the persistant - * case, the EOA should already be as shrunked as it can be, and the - * aggregators should alread be shut down. - */ + HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - /* Free the space in aggregators (again) */ - /* (in case any free space information re-started them) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, sm_fssinfo_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - /* Trying shrinking the EOA for the file */ - /* (in case any free space is now at the EOA) */ - if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") - } /* end else */ + HDassert(f->shared->fs_man[sm_fssinfo_fs_type]); + } /* end if */ -done: - /* Reset the ring in the DXPL */ - if(reset_ring) - if(H5AC_reset_ring(dxpl, orig_ring) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_close() */ + HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); - -/*------------------------------------------------------------------------- - * Function: H5MF_sects_cb() - * - * Purpose: Iterator callback for each free-space section - * Retrieve address and size into user data - * - * Return: Always succeed - * - * Programmer: Vailin Choi - * July 1st, 2009 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata) -{ - H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; - H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata; + HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES); - FUNC_ENTER_NOAPI_NOINIT_NOERR + HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); + HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - if(udata->sect_idx < udata->sect_count) { - udata->sects[udata->sect_idx].addr = sect->sect_info.addr; - udata->sects[udata->sect_idx].size = sect->sect_info.size; - udata->sect_idx++; - } /* end if */ + if(H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])) { + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[lg_fshdr_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[lg_fshdr_fs_type]; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sects_cb() */ + HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); - -/*------------------------------------------------------------------------- - * Function: H5MF_get_free_sections() - * - * Purpose: To iterate over one or all free-space managers for: - * # of sections - * section info as defined in H5F_sect_info_t - * - * Return: SUCCEED/FAIL - * - * Programmer: Vailin Choi - * July 1st, 2009 - * - *------------------------------------------------------------------------- - */ -ssize_t -H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info) -{ - size_t total_sects = 0; /* total number of sections */ - H5MF_sect_iter_ud_t sect_udata; /* User data for callback */ - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t start_type, end_type; /* Memory types to iterate over */ - H5FD_mem_t ty; /* Memory type for iteration */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - ssize_t ret_value = -1; /* Return value */ + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, lg_fshdr_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + HDassert(f->shared->fs_man[lg_fshdr_fs_type]); + } /* end if */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + if(lg_fshdr_fs_type != lg_fssinfo_fs_type && H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])) { + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[lg_fssinfo_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[lg_fssinfo_fs_type]; - /* check args */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); + HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - /* Determine start/end points for loop */ - if(type == H5FD_MEM_DEFAULT) { - start_type = H5FD_MEM_SUPER; - end_type = H5FD_MEM_NTYPES; + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, lg_fssinfo_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + HDassert(f->shared->fs_man[lg_fssinfo_fs_type]); + } /* end if */ + } /* end if */ + HDassert(H5F_addr_eq(first_srfsm_hdr, f->shared->eoa_pre_fsm_fsalloc)); + + /* 4) Float the FSMs. Ensure that the file space is NOT released. */ + if(f->shared->fs_man[sm_fshdr_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[sm_fshdr_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[sm_fshdr_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[sm_fshdr_fs_type] = HADDR_UNDEF; } /* end if */ - else { - start_type = end_type = type; - H5_INC_ENUM(H5FD_mem_t, end_type); - } /* end else */ - - /* Set up user data for section iteration */ - sect_udata.sects = sect_info; - sect_udata.sect_count = nsects; - sect_udata.sect_idx = 0; - /* Set the ring type in the DXPL. Note that if we are - * scanning a number of different free space managers, - * we may have to change the ring - */ - if((H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - curr_ring = needed_ring; - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &orig_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - reset_ring = TRUE; + if(sm_fshdr_fs_type != sm_fssinfo_fs_type && f->shared->fs_man[sm_fssinfo_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[sm_fssinfo_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - /* Iterate over memory types, retrieving the number of sections of each type */ - for(ty = start_type; ty < end_type; H5_INC_ENUM(H5FD_mem_t, ty)) { - hbool_t fs_started = FALSE; + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[sm_fssinfo_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[sm_fssinfo_fs_type] = HADDR_UNDEF; + } /* end if */ - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(H5F_PAGED_AGGR(f)) { + if(f->shared->fs_man[lg_fshdr_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[lg_fshdr_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[lg_fshdr_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") + f->shared->fs_addr[lg_fshdr_fs_type] = HADDR_UNDEF; } /* end if */ - /* Open free space manager of this type, if it isn't already */ - if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) { - if(H5MF__alloc_open(f, dxpl_id, ty) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space") - HDassert(f->shared->fs_man[ty]); - fs_started = TRUE; - } /* end if */ - - /* Check if f there's free space sections of this type */ - if(f->shared->fs_man[ty]) { - hsize_t hnums = 0; /* Total # of sections */ - size_t nums; /* Total # of sections, cast to a size_t */ - - /* Query how many sections of this type */ - if(H5FS_sect_stats(f->shared->fs_man[ty], NULL, &hnums) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") - H5_CHECKED_ASSIGN(nums, size_t, hnums, hsize_t); - - /* Increment total # of sections */ - total_sects += nums; - - /* Check if we should retrieve the section info */ - if(sect_info && nums > 0) { - /* Iterate over all the free space sections of this type, adding them to the user's section info */ - if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ty], H5MF_sects_cb, §_udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections") - } /* end if */ + if(lg_fshdr_fs_type != lg_fssinfo_fs_type && f->shared->fs_man[lg_fssinfo_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[lg_fssinfo_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[lg_fssinfo_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") + f->shared->fs_addr[lg_fssinfo_fs_type] = HADDR_UNDEF; } /* end if */ + } /* end if */ - /* Close the free space manager of this type, if we started it here */ - if(fs_started) - if(H5MF__alloc_close(f, dxpl_id, ty) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space") - } /* end for */ + /* 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, + * and then set f->shared->eoa_pre_fsm_fsalloc to + * HADDR_UNDEF. + * + * If page buffering, verify that the new EOA is + * on a page boundary, and expunge any pages in the + * page buffer after the new EOA. + */ + if(!H5F_PAGED_AGGR(f)) { + /* Verify that the aggregators are still shutdown. */ + HDassert(f->shared->sdata_aggr.tot_size == 0); + HDassert(f->shared->sdata_aggr.addr == 0); + HDassert(f->shared->sdata_aggr.size == 0); - /* Set return value */ - ret_value = (ssize_t)total_sects; + HDassert(f->shared->meta_aggr.tot_size == 0); + HDassert(f->shared->meta_aggr.addr == 0); + HDassert(f->shared->meta_aggr.size == 0); + } /* end if */ + + tail_size = (hsize_t)(eoa - f->shared->eoa_pre_fsm_fsalloc); + + /* Release file space allocated to self referential FSMs */ + if(H5F_free(f, dxpl_id, H5FD_MEM_DEFAULT, f->shared->eoa_pre_fsm_fsalloc, tail_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + HDassert(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)); + + f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + HDassert((!H5F_PAGED_AGGR(f)) || (0 == (eoa % f->shared->fs_page_size))); done: /* Reset the ring in the DXPL */ @@ -2359,5 +3911,5 @@ done: HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5MF_get_free_sections() */ +} /* H5MF_tidy_self_referential_fsm_hack() */ diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c index 1510645..de98bf1 100644 --- a/src/H5MFaggr.c +++ b/src/H5MFaggr.c @@ -57,8 +57,10 @@ /********************/ /* Local Prototypes */ /********************/ -static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, +static herr_t H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr); +static haddr_t H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, + H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size); /*********************/ @@ -78,86 +80,9 @@ static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, /*------------------------------------------------------------------------- - * Function: H5MF_vfd_alloc - * - * Purpose: Allocate SIZE bytes of file memory via H5FD_alloc() and return - * the relative address where that contiguous chunk of file memory - * exists. - * The TYPE argument describes the purpose for which the storage - * is being requested. - * - * Return: Success: The file address of new chunk. - * Failure: HADDR_UNDEF - * - * Programmer: Quincey Koziol - * January 2, 2017 - * - *------------------------------------------------------------------------- - */ -haddr_t -H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, hsize_t size, - hbool_t keep_fragment) -{ - haddr_t eoa; /* Initial EOA for the file */ - haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ - hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */ - haddr_t ret_value = HADDR_UNDEF; /* Return value */ - - FUNC_ENTER_NOAPI(HADDR_UNDEF) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); -#endif /* H5MF_ALLOC_DEBUG */ - - /* check arguments */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); - HDassert(size > 0); - - /* Get the EOA for the file -- need for sanity check below */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") - - /* Check for overlap into temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "hdr file space alloc will overlap into 'temporary' file space") - - /* Allocate space for the header */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space for hdr") - - /* Sanity check for overlapping into file's temporary allocation space */ - HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); - - /* If the file alignment is 1, there should be no eoa fragment */ - HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); - - /* Check if fragment was generated and we want to keep it */ - if(keep_fragment && eoa_frag_size > 0) { - /* Sanity check */ - HDassert(H5F_addr_defined(eoa_frag_addr)); - - /* Put fragment on the free list */ - if(H5MF_xfree(f, alloc_type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment") - } /* end if */ - -done: -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); -#endif /* H5MF_ALLOC_DEBUG */ -#ifdef H5MF_ALLOC_DEBUG_DUMP -H5MF_sects_dump(f, dxpl_id, stderr); -#endif /* H5MF_ALLOC_DEBUG_DUMP */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_vfd_alloc() */ - - -/*------------------------------------------------------------------------- * Function: H5MF_aggr_vfd_alloc * - * Purpose: Allocate SIZE bytes of file memory via H5MF_aggr_alloc() + * Purpose: Allocate SIZE bytes of file memory via H5MF__aggr_alloc() * and return the relative address where that contiguous chunk * of file memory exists. * The TYPE argument describes the purpose for which the storage @@ -177,9 +102,9 @@ H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size haddr_t ret_value = HADDR_UNDEF; /* Return value */ FUNC_ENTER_NOAPI(HADDR_UNDEF) -#ifdef H5MF_ALLOC_DEBUG +#ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); -#endif /* H5MF_ALLOC_DEBUG */ +#endif /* H5MF_AGGR_DEBUG */ /* check arguments */ HDassert(f); @@ -190,12 +115,12 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ /* Couldn't find anything from the free space manager, go allocate some */ if(alloc_type != H5FD_MEM_DRAW && alloc_type != H5FD_MEM_GHEAP) { /* Handle metadata differently from "raw" data */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size))) + if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate metadata") } /* end if */ else { /* Allocate "raw" data: H5FD_MEM_DRAW and H5FD_MEM_GHEAP */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size))) + if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate raw data") } /* end else */ @@ -203,19 +128,16 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); done: -#ifdef H5MF_ALLOC_DEBUG +#ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); -#endif /* H5MF_ALLOC_DEBUG */ -#ifdef H5MF_ALLOC_DEBUG_DUMP -H5MF_sects_dump(f, dxpl_id, stderr); -#endif /* H5MF_ALLOC_DEBUG_DUMP */ +#endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5MF_aggr_vfd_alloc() */ /*------------------------------------------------------------------------- - * Function: H5MF_aggr_alloc + * Function: H5MF__aggr_alloc * * Purpose: Try to allocate SIZE bytes of memory from an aggregator * block if possible. @@ -228,8 +150,8 @@ H5MF_sects_dump(f, dxpl_id, stderr); * *------------------------------------------------------------------------- */ -haddr_t -H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, +static haddr_t +H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size) { haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ @@ -237,7 +159,7 @@ H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ - FUNC_ENTER_NOAPI(HADDR_UNDEF) + FUNC_ENTER_STATIC #ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); #endif /* H5MF_AGGR_DEBUG */ @@ -257,11 +179,11 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") /* - * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_VFD, + * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_NONE, * allocate "generic" space and sub-allocate out of that, if possible. - * Otherwise just allocate through H5FD_alloc(). + * Otherwise just allocate through H5F_alloc(). */ - if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FILE_SPACE_VFD) { + if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE) { haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */ hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */ hsize_t alignment; /* Alignment of this section */ @@ -273,12 +195,12 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz #endif /* H5MF_AGGR_DEBUG */ /* Turn off alignment if allocation < threshold */ - alignment = f->shared->alignment; - if(!((alignment > 1) && (size >= f->shared->threshold))) + alignment = H5F_ALIGNMENT(f); + if(!((alignment > 1) && (size >= H5F_THRESHOLD(f)))) alignment = 0; /* no alignment */ /* Generate fragment if aggregator is mis-aligned */ - if(alignment && aggr->addr > 0 && (aggr_mis_align = (aggr->addr + H5FD_get_base_addr(f->shared->lf)) % alignment)) { + if(alignment && H5F_addr_gt(aggr->addr, 0) && (aggr_mis_align = (aggr->addr + H5F_BASE_ADDR(f)) % alignment)) { aggr_frag_addr = aggr->addr; aggr_frag_size = alignment - aggr_mis_align; } /* end if */ @@ -288,7 +210,7 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz /* Check if the space requested is larger than the space left in the block */ if((size + aggr_frag_size) > aggr->size) { - htri_t was_extended = FALSE; /* Whether the file was extended */ + htri_t extended = FALSE; /* Whether the file was extended */ /* Check if the block asked for is too large for 'normal' aggregator block */ if(size >= aggr->alloc_size) { @@ -298,32 +220,27 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr)) HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0) + if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space") - else if(was_extended) { + else if (extended) { /* aggr->size is unchanged */ ret_value = aggr->addr + aggr_frag_size; aggr->addr += ext_size; aggr->tot_size += ext_size; - } /* end else-if */ - else { - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - + } else { /* Release "other" aggregator, if it exists, is at the end of the allocated space, * has allocated more than one block and the unallocated space is greater than its * allocation block size. */ if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) && (other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) { - if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) + if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block") } /* end if */ /* Allocate space from the VFD (i.e. at the end of the file) */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block") + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size, &eoa_frag_addr, &eoa_frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") } /* end else */ } /* end if */ else { @@ -341,9 +258,9 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr)) HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0) + if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space") - else if(was_extended) { + else if(extended) { aggr->addr += aggr_frag_size; aggr->size += (ext_size - aggr_frag_size); aggr->tot_size += ext_size; @@ -351,23 +268,19 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); else { haddr_t new_space; /* Address of new space allocated */ - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + aggr->alloc_size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - /* Release "other" aggregator, if it exists, is at the end of the allocated space, * has allocated more than one block and the unallocated space is greater than its * allocation block size. */ if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) && (other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) { - if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) + if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block") } /* end if */ /* Allocate space from the VFD (i.e. at the end of the file) */ - if(HADDR_UNDEF == (new_space = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block") + if(HADDR_UNDEF == (new_space = H5F_alloc(f, dxpl_id, alloc_type, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") /* Return the unused portion of the block to a free list */ if(aggr->size > 0) @@ -408,7 +321,7 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment") /* Freeing any possible fragment due to alignment in the block after extension */ - if(was_extended && aggr_frag_size) + if(extended && aggr_frag_size) if(H5MF_xfree(f, alloc_type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation fragment") } /* end if */ @@ -425,12 +338,8 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); } /* end else */ } /* end if */ else { - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - /* Allocate data from the file */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, &eoa_frag_addr, &eoa_frag_size))) + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, type, size, &eoa_frag_addr, &eoa_frag_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") /* Check if fragment was generated */ @@ -444,15 +353,15 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); /* Post-condition sanity check */ - if(f->shared->alignment && size >= f->shared->threshold) - HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % f->shared->alignment)); + if(H5F_ALIGNMENT(f) && size >= H5F_THRESHOLD(f)) + HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % H5F_ALIGNMENT(f))); done: #ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: ret_value = %a\n", FUNC, ret_value); #endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_aggr_alloc() */ +} /* end H5MF__aggr_alloc() */ /*------------------------------------------------------------------------- @@ -524,7 +433,7 @@ H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, else { hsize_t extra = (extra_requested < aggr->alloc_size) ? aggr->alloc_size : extra_requested; - if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, (aggr->addr + aggr->size), extra)) < 0) + if((ret_value = H5F_try_extend(f, dxpl_id, type, (aggr->addr + aggr->size), extra)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") else if(ret_value == TRUE) { /* Shift the aggregator block by the extra requested */ @@ -905,7 +814,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5MF_aggr_free + * Function: H5MF__aggr_free * * Purpose: Free the aggregator's space in the file. * @@ -919,11 +828,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) +H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC /* Sanity check */ HDassert(f); @@ -936,7 +845,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) HDassert(f->shared->feature_flags & aggr->feature_flag); /* Free the remaining space at EOA in the aggregator */ - if(H5FD_free(f->shared->lf, dxpl_id, type, f, aggr->addr, aggr->size) < 0) + if(H5F_free(f, dxpl_id, type, aggr->addr, aggr->size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregation block") /* Reset the aggregator */ @@ -946,7 +855,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_aggr_free() */ +} /* H5MF__aggr_free() */ /*------------------------------------------------------------------------- @@ -978,13 +887,13 @@ H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id) if((ma_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr))) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") if(ma_status > 0) - if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0) + if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") if((sda_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DRAW, &(f->shared->sdata_aggr))) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") if(sda_status > 0) - if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0) + if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") ret_value = (ma_status || sda_status); diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c index 57bac9e..817ea00 100644 --- a/src/H5MFdbg.c +++ b/src/H5MFdbg.c @@ -116,7 +116,9 @@ H5MF_sects_debug_cb(H5FS_section_info_t *_sect, void *_udata) /* Print generic section information */ HDfprintf(udata->stream, "%*s%-*s %s\n", udata->indent, "", udata->fwidth, "Section type:", - (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" : "unknown")); + (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" : + (sect->sect_info.type == H5MF_FSPACE_SECT_SMALL ? "small" : + (sect->sect_info.type == H5MF_FSPACE_SECT_LARGE ? "large" : "unknown")))); HDfprintf(udata->stream, "%*s%-*s %a\n", udata->indent, "", udata->fwidth, "Section address:", sect->sect_info.addr); @@ -171,7 +173,7 @@ H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int ind for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) if(H5F_addr_eq(f->shared->fs_addr[type], fs_addr)) { if(!f->shared->fs_man[type]) - if(H5MF__alloc_open(f, dxpl_id, type) < 0) + if(H5MF_open_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") if(f->shared->fs_man[type]) { @@ -217,11 +219,6 @@ herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream) { haddr_t eoa; /* End of allocated space in the file */ - haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ - hsize_t ma_size = 0; /* Size of "metadata aggregator" */ - haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ - hsize_t sda_size = 0; /* Size of "small data aggregator" */ - H5FD_mem_t type; /* Memory type for iteration */ int indent = 0; /* Amount to indent */ int fwidth = 50; /* Field width */ herr_t ret_value = SUCCEED; /* Return value */ @@ -244,56 +241,90 @@ HDfprintf(stderr, "%s: Dumping file free space sections\n", FUNC); HDfprintf(stderr, "%s: for type = H5FD_MEM_DEFAULT, eoa = %a\n", FUNC, eoa); #endif /* H5MF_ALLOC_DEBUG */ - /* Retrieve metadata aggregator info, if available */ - H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1)); -#endif /* H5MF_ALLOC_DEBUG */ + if(H5F_PAGED_AGGR(f)) { /* File space paging */ + H5F_mem_page_t ptype; /* Memory type for iteration -- page fs */ - /* Retrieve 'small data' aggregator info, if available */ - H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1)); -#endif /* H5MF_ALLOC_DEBUG */ - - /* Iterate over all the free space types that have managers and dump each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - /* Print header for type */ - HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)type); - - /* Check for this type being mapped to another type */ - if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[type] || - type == f->shared->fs_type_map[type]) { - /* Retrieve the 'eoa' for this file memory type */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") - HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa); + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + /* Print header for type */ + HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)ptype); /* Print header for sections */ HDfprintf(stream, "%*sSections:\n", indent + 3, ""); /* If there is a free space manager for this type, iterate over them */ - if(f->shared->fs_man[type]) { + if(f->shared->fs_man[ptype]) { H5MF_debug_iter_ud_t udata; /* User data for callbacks */ /* Prepare user data for section iteration callback */ - udata.fspace = f->shared->fs_man[type]; + udata.fspace = f->shared->fs_man[ptype]; udata.stream = stream; udata.indent = indent + 6; udata.fwidth = MAX(0, fwidth - 6); /* Iterate over all the free space sections */ - if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[type], H5MF_sects_debug_cb, &udata) < 0) + if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ptype], H5MF_sects_debug_cb, &udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") } /* end if */ else /* No sections of this type */ HDfprintf(stream, "%*s<none>\n", indent + 6, ""); - } /* end if */ - else - HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[type]); + } /* end for */ + } /* end if */ + else { /* not file space paging */ + H5FD_mem_t atype; /* Memory type for iteration -- aggr fs */ + haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ + hsize_t ma_size = 0; /* Size of "metadata aggregator" */ + haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ + hsize_t sda_size = 0; /* Size of "small data aggregator" */ + + /* Retrieve metadata aggregator info, if available */ + H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1)); +#endif /* H5MF_ALLOC_DEBUG */ + + /* Retrieve 'small data' aggregator info, if available */ + H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size); +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1)); +#endif /* H5MF_ALLOC_DEBUG */ - } /* end for */ + /* Iterate over all the free space types that have managers and dump each free list's space */ + for(atype = H5FD_MEM_DEFAULT; atype < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, atype)) { + /* Print header for type */ + HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)atype); + + /* Check for this type being mapped to another type */ + if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[atype] || atype == f->shared->fs_type_map[atype]) { + /* Retrieve the 'eoa' for this file memory type */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, atype))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa); + + /* Print header for sections */ + HDfprintf(stream, "%*sSections:\n", indent + 3, ""); + + /* If there is a free space manager for this type, iterate over them */ + if(f->shared->fs.aggr.fs_man[atype]) { + H5MF_debug_iter_ud_t udata; /* User data for callbacks */ + + /* Prepare user data for section iteration callback */ + udata.fspace = f->shared->fs_man[atype]; + udata.stream = stream; + udata.indent = indent + 6; + udata.fwidth = MAX(0, fwidth - 6); + + /* Iterate over all the free space sections */ + if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[atype], H5MF_sects_debug_cb, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") + } /* end if */ + else /* No sections of this type */ + HDfprintf(stream, "%*s<none>\n", indent + 6, ""); + } /* end if */ + else + HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[atype]); + } /* end for */ + } /* end else */ done: HDfprintf(stderr, "%s: Done dumping file free space sections\n", FUNC); diff --git a/src/H5MFpkg.h b/src/H5MFpkg.h index 43fc1cd..c4a0819 100644 --- a/src/H5MFpkg.h +++ b/src/H5MFpkg.h @@ -51,9 +51,38 @@ /* Define this to dump free space tracker contents after they've been modified */ /* #define H5MF_ALLOC_DEBUG_DUMP */ -/* Free space section types for file */ +/* Free-space section types for file */ /* (values stored in free space data structures in file) */ -#define H5MF_FSPACE_SECT_SIMPLE 0 /* Section is a range of actual bytes in file */ +#define H5MF_FSPACE_SECT_SIMPLE 0 /* For non-paged aggregation: section is a range of actual bytes in file */ +#define H5MF_FSPACE_SECT_SMALL 1 /* For paged aggregation: "small" meta/raw data section which is < fsp_size) */ +#define H5MF_FSPACE_SECT_LARGE 2 /* For paged aggregation: "large" Section which is >= fsp_size) */ + +/* For non-paged aggregation: map allocation request type to tracked free-space type */ +/* F -- pointer to H5F_t; T -- H5FD_mem_t */ +#define H5MF_ALLOC_TO_FS_AGGR_TYPE(F, T) \ + ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) ? (T) : (F)->shared->fs_type_map[T]) + +/* Get section class type based on size */ +#define H5MF_SECT_CLASS_TYPE(F, S) \ + ((H5F_PAGED_AGGR(F)) ? \ + ((S >= (F)->shared->fs_page_size) ? H5MF_FSPACE_SECT_LARGE : H5MF_FSPACE_SECT_SMALL) : H5MF_FSPACE_SECT_SIMPLE) + +/* Get section class cls */ +#define H5MF_SECT_CLS_TYPE(F, S) \ + ((H5F_PAGED_AGGR(F)) ? \ + ((S >= (F)->shared->fs_page_size) ? \ + H5MF_FSPACE_SECT_CLS_LARGE : H5MF_FSPACE_SECT_CLS_SMALL) : H5MF_FSPACE_SECT_CLS_SIMPLE) + +/* Calculate the mis-aligned fragment */ +#define H5MF_EOA_MISALIGN(F, E, A, FR) \ +{ \ + hsize_t m; \ + \ + if(H5F_addr_gt((E), 0) && ((m) = ((E) + H5F_BASE_ADDR(F)) % (A))) \ + (FR) = (A) - m; \ + else \ + (FR) = 0; \ +} /****************************/ @@ -129,6 +158,15 @@ typedef struct H5MF_sect_ud_t { H5F_blk_aggr_t *aggr; /* Aggregator block to operate on */ } H5MF_sect_ud_t; +/* Information about the current free-space manager to use */ +typedef struct H5MF_fs_t { + H5F_fs_state_t *fs_state; + haddr_t *fs_addr; + H5FS_t **fs_man; + hsize_t align_thres; /* Threshold for alignment */ + hsize_t alignment; /* Alignment */ +} H5MF_fs_t; + /*****************************/ /* Package Private Variables */ @@ -136,6 +174,8 @@ typedef struct H5MF_sect_ud_t { /* H5MF single section inherits serializable properties from H5FS_section_class_t */ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1]; +H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1]; +H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1]; /******************************/ @@ -143,22 +183,23 @@ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1]; /******************************/ /* Allocator routines */ -H5_DLL herr_t H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -H5_DLL herr_t H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); +H5_DLL herr_t H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +H5_DLL herr_t H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); + +H5_DLL htri_t H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr); +H5_DLL herr_t H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node); + H5_DLL herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream); -/* 'simple' section routines */ -H5_DLL H5MF_free_section_t *H5MF_sect_simple_new(haddr_t sect_off, +H5_DLL void H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type); + +/* 'simple/small/large' section routines */ +H5_DLL H5MF_free_section_t *H5MF_sect_new(unsigned ctype, haddr_t sect_off, hsize_t sect_size); -H5_DLL htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, - void *udata); -H5_DLL herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, - void *udata); -H5_DLL herr_t H5MF_sect_simple_free(H5FS_section_info_t *sect); +H5_DLL herr_t H5MF_sect_free(H5FS_section_info_t *sect); + /* Block aggregator routines */ -H5_DLL haddr_t H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, - H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size); H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, H5FD_mem_t type, haddr_t abs_blk_end, hsize_t extra_requested); H5_DLL htri_t H5MF_aggr_can_absorb(const H5F_t *f, const H5F_blk_aggr_t *aggr, diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h index e258677..70322c3 100644 --- a/src/H5MFprivate.h +++ b/src/H5MFprivate.h @@ -58,8 +58,6 @@ H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id); /* File space allocation routines */ H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); -H5_DLL haddr_t H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, - hsize_t size, hbool_t keep_fragment); H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); H5_DLL herr_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, @@ -80,6 +78,14 @@ H5_DLL htri_t H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled); H5_DLL herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled); +/* This function has to be declared in H5MFprivate.h as it is needed + * in our test code to allow us to manually start a self referential + * free space manager prior to the first file space allocations / + * deallocation without causing assertion failures on the first + * file space allocation / deallocation. + */ +H5_DLL herr_t H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id); + /* Debugging routines */ #ifdef H5MF_DEBUGGING H5_DLL herr_t H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, diff --git a/src/H5MFsection.c b/src/H5MFsection.c index e5a0cf0..617cb59 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -57,18 +57,47 @@ /* Local Prototypes */ /********************/ -/* 'simple' section callbacks */ -static H5FS_section_info_t *H5MF_sect_simple_deserialize(const H5FS_section_class_t *cls, +/* 'simple/small/large' section callbacks */ +static H5FS_section_info_t *H5MF_sect_deserialize(const H5FS_section_class_t *cls, hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size, unsigned *des_flags); +static herr_t H5MF_sect_valid(const H5FS_section_class_t *cls, + const H5FS_section_info_t *sect, hid_t dxpl_id); +static H5FS_section_info_t *H5MF_sect_split(H5FS_section_info_t *sect, + hsize_t frag_size); + + +/* 'simple' section callbacks */ static htri_t H5MF_sect_simple_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5MF_sect_simple_merge(H5FS_section_info_t *sect1, +static herr_t H5MF_sect_simple_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); -static herr_t H5MF_sect_simple_valid(const H5FS_section_class_t *cls, - const H5FS_section_info_t *sect, hid_t dxpl_id); -static H5FS_section_info_t *H5MF_sect_simple_split(H5FS_section_info_t *sect, - hsize_t frag_size); +static htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, + void *udata); + + +/* 'small' section callbacks */ +static herr_t H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata); +static htri_t H5MF_sect_small_can_merge(const H5FS_section_info_t *sect1, + const H5FS_section_info_t *sect2, void *udata); +static herr_t H5MF_sect_small_merge(H5FS_section_info_t **sect1, + H5FS_section_info_t *sect2, void *udata); +static htri_t H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_small_shrink(H5FS_section_info_t **_sect, + void *udata); + +/* 'large' section callbacks */ +static htri_t H5MF_sect_large_can_merge(const H5FS_section_info_t *sect1, + const H5FS_section_info_t *sect2, void *udata); +static herr_t H5MF_sect_large_merge(H5FS_section_info_t **sect1, + H5FS_section_info_t *sect2, void *udata); +static htri_t H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_large_shrink(H5FS_section_info_t **_sect, + void *udata); /*********************/ /* Package Variables */ @@ -89,17 +118,68 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Object methods */ NULL, /* Add section */ NULL, /* Serialize section */ - H5MF_sect_simple_deserialize, /* Deserialize section */ + H5MF_sect_deserialize, /* Deserialize section */ H5MF_sect_simple_can_merge, /* Can sections merge? */ H5MF_sect_simple_merge, /* Merge sections */ H5MF_sect_simple_can_shrink, /* Can section shrink container?*/ H5MF_sect_simple_shrink, /* Shrink container w/section */ - H5MF_sect_simple_free, /* Free section */ - H5MF_sect_simple_valid, /* Check validity of section */ - H5MF_sect_simple_split, /* Split section node for alignment */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ NULL, /* Dump debugging for section */ }}; +/* Class info for "small" free space sections */ +H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{ + /* Class variables */ + H5MF_FSPACE_SECT_SMALL, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ + + /* Class methods */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ + + /* Object methods */ + H5MF_sect_small_add, /* Add section */ + NULL, /* Serialize section */ + H5MF_sect_deserialize, /* Deserialize section */ + H5MF_sect_small_can_merge, /* Can sections merge? */ + H5MF_sect_small_merge, /* Merge sections */ + H5MF_sect_small_can_shrink, /* Can section shrink container?*/ + H5MF_sect_small_shrink, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ +}}; + +/* Class info for "large" free space sections */ +H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1] = {{ + /* Class variables */ + H5MF_FSPACE_SECT_LARGE, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ + + /* Class methods */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ + + /* Object methods */ + NULL, /* Add section */ + NULL, /* Serialize section */ + H5MF_sect_deserialize, /* Deserialize section */ + H5MF_sect_large_can_merge, /* Can sections merge? */ + H5MF_sect_large_merge, /* Merge sections */ + H5MF_sect_large_can_shrink, /* Can section shrink container?*/ + H5MF_sect_large_shrink, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ +}}; /*****************************/ /* Library Private Variables */ @@ -113,12 +193,15 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Declare a free list to manage the H5MF_free_section_t struct */ H5FL_DEFINE(H5MF_free_section_t); +/* + * "simple/small/large" section callbacks + */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_new + * Function: H5MF_sect_new * - * Purpose: Create a new 'simple' section and return it to the caller + * Purpose: Create a new section of "ctype" and return it to the caller * * Return: Pointer to new section on success/NULL on failure * @@ -129,9 +212,9 @@ H5FL_DEFINE(H5MF_free_section_t); *------------------------------------------------------------------------- */ H5MF_free_section_t * -H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) +H5MF_sect_new(unsigned ctype, haddr_t sect_off, hsize_t sect_size) { - H5MF_free_section_t *sect = NULL; /* 'Simple' free space section to add */ + H5MF_free_section_t *sect; /* 'Simple' free space section to add */ H5MF_free_section_t *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -148,7 +231,7 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) sect->sect_info.size = sect_size; /* Set the section's class & state */ - sect->sect_info.type = H5MF_FSPACE_SECT_SIMPLE; + sect->sect_info.type = ctype; sect->sect_info.state = H5FS_SECT_LIVE; /* Set return value */ @@ -156,13 +239,43 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_sect_simple_new() */ +} /* end H5MF_sect_new() */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_deserialize + * Function: H5MF_sect_free * - * Purpose: Deserialize a buffer into a "live" single section + * Purpose: Free a 'simple/small/large' section node + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 8, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_sect_free(H5FS_section_info_t *_sect) +{ + H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(sect); + + /* Release the section */ + sect = H5FL_FREE(H5MF_free_section_t, sect); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sect_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_deserialize + * + * Purpose: Deserialize a buffer into a "live" section * * Return: Success: non-negative * Failure: negative @@ -173,7 +286,7 @@ done: *------------------------------------------------------------------------- */ static H5FS_section_info_t * -H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, +H5MF_sect_deserialize(const H5FS_section_class_t *cls, hid_t H5_ATTR_UNUSED dxpl_id, const uint8_t H5_ATTR_UNUSED *buf, haddr_t sect_addr, hsize_t sect_size, unsigned H5_ATTR_UNUSED *des_flags) { @@ -183,11 +296,12 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, FUNC_ENTER_NOAPI_NOINIT /* Check arguments. */ + HDassert(cls); HDassert(H5F_addr_defined(sect_addr)); HDassert(sect_size); /* Create free space section for block */ - if(NULL == (sect = H5MF_sect_simple_new(sect_addr, sect_size))) + if(NULL == (sect = H5MF_sect_new(cls->type, sect_addr, sect_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") /* Set return value */ @@ -195,10 +309,80 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_sect_simple_deserialize() */ +} /* H5MF_sect_deserialize() */ /*------------------------------------------------------------------------- + * Function: H5MF_sect_valid + * + * Purpose: Check the validity of a section + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 8, 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls, + const H5FS_section_info_t +#ifdef NDEBUG + H5_ATTR_UNUSED +#endif /* NDEBUG */ + *_sect, hid_t H5_ATTR_UNUSED dxpl_id) +{ +#ifndef NDEBUG + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ +#endif /* NDEBUG */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(sect); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sect_valid() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_split + * + * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section + * SECT's addr and size are updated to point to the aligned section + * + * Return: Success: the fragment for aligning sect + * Failure: null + * + * Programmer: Vailin Choi, July 29, 2008 + * + *------------------------------------------------------------------------- + */ +static H5FS_section_info_t * +H5MF_sect_split(H5FS_section_info_t *sect, hsize_t frag_size) +{ + H5MF_free_section_t *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Allocate space for new section */ + if(NULL == (ret_value = H5MF_sect_new(sect->type, sect->addr, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") + + /* Set new section's info */ + sect->addr += frag_size; + sect->size -= frag_size; + +done: + FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value) +} /* end H5MF_sect_split() */ + +/* + * "simple" section callbacks + */ + +/*------------------------------------------------------------------------- * Function: H5MF_sect_simple_can_merge * * Purpose: Can two sections of this type merge? @@ -252,10 +436,10 @@ H5MF_sect_simple_can_merge(const H5FS_section_info_t *_sect1, *------------------------------------------------------------------------- */ static herr_t -H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5MF_sect_simple_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) { - H5MF_free_section_t *sect1 = (H5MF_free_section_t *)_sect1; /* File free section */ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ herr_t ret_value = SUCCEED; /* Return value */ @@ -263,16 +447,16 @@ H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); HDassert(sect2); HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)sect2) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") done: @@ -293,7 +477,7 @@ done: * *------------------------------------------------------------------------- */ -htri_t +static htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, void *_udata) { const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ @@ -392,7 +576,7 @@ done: * *------------------------------------------------------------------------- */ -herr_t +static herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) { H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ @@ -411,8 +595,8 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) /* Sanity check */ HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); - /* Release section's space at EOA with file driver */ - if(H5FD_free(udata->f->shared->lf, udata->dxpl_id, udata->alloc_type, udata->f, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) + /* Release section's space at EOA */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") } /* end if */ else { @@ -427,7 +611,7 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) /* Check for freeing section */ if(udata->shrink != H5MF_SHRINK_SECT_ABSORB_AGGR) { /* Free section */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)*sect) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") /* Mark section as freed, for free space manager */ @@ -438,100 +622,474 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5MF_sect_simple_shrink() */ +/* + * "small" section callbacks + */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_add + * + * Purpose: Perform actions on a small "meta" action before adding it to the free space manager: + * 1) Drop the section if it is at page end and its size <= page end threshold + * 2) Adjust section size to include page end threshold if + * (section size + threshold) is at page end + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) +{ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* Fractal heap free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t sect_end; + hsize_t rem, prem; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Do not adjust the section raw data or global heap data */ + if(udata->alloc_type == H5FD_MEM_DRAW || udata->alloc_type == H5FD_MEM_GHEAP) + HGOTO_DONE(ret_value); + + sect_end = (*sect)->sect_info.addr + (*sect)->sect_info.size; + rem = sect_end % udata->f->shared->fs_page_size; + prem = udata->f->shared->fs_page_size - rem; + + /* Drop the section if it is at page end and its size is <= pgend threshold */ + if(!rem && (*sect)->sect_info.size <= H5F_PGEND_META_THRES(udata->f) && (*flags & H5FS_ADD_RETURNED_SPACE)) { + if(H5MF_sect_free((H5FS_section_info_t *)(*sect)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + *sect = NULL; + *flags &= (unsigned)~H5FS_ADD_RETURNED_SPACE; + *flags |= H5FS_PAGE_END_NO_ADD; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section is dropped\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + /* Adjust the section if it is not at page end but its size + pgend threshold is at page end */ + else + if(prem <= H5F_PGEND_META_THRES(udata->f)) { + (*sect)->sect_info.size += prem; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section is adjusted {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_add() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_can_shrink + * + * Purpose: Can this section shrink the container? + * + * Note: A small section is allowed to shrink only at closing. + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect, void *_udata) +{ + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t eoa; /* End of address space in the file */ + haddr_t end; /* End of section to extend */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect); + HDassert(udata); + HDassert(udata->f); + + /* Retrieve the end of the file's address space */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* Compute address of end of section to check */ + end = sect->sect_info.addr + sect->sect_info.size; + + /* Check if the section is exactly at the end of the allocated space in the file */ + if(H5F_addr_eq(end, eoa) && sect->sect_info.size == udata->f->shared->fs_page_size) { + udata->shrink = H5MF_SHRINK_EOA; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Indicate shrinking can occur */ + HGOTO_DONE(TRUE) + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_can_shrink() */ + /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_free + * Function: H5MF_sect_small_shrink * - * Purpose: Free a 'single' section node + * Purpose: Shrink container with section * * Return: Success: non-negative - * Failure: negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 * - * Programmer: Quincey Koziol - * Tuesday, January 8, 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_small_shrink(H5FS_section_info_t **_sect, void *_udata) +{ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect); + HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(udata); + HDassert(udata->f); + HDassert(udata->shrink == H5MF_SHRINK_EOA); + HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); + + /* Release section's space at EOA */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + + /* Free section */ + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + + /* Mark section as freed, for free space manager */ + *sect = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_shrink() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_can_merge + * + * Purpose: Can two sections of this type merge? + * + * Note: Second section must be "after" first section + * The "merged" section cannot cross page boundary. + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ -herr_t -H5MF_sect_simple_free(H5FS_section_info_t *_sect) +static htri_t +H5MF_sect_small_can_merge(const H5FS_section_info_t *_sect1, + const H5FS_section_info_t *_sect2, void *_udata) { - H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */ + const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */ + const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR /* Check arguments. */ - HDassert(sect); + HDassert(sect1); + HDassert(sect2); + HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */ + HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)); - /* Release the section */ - sect = H5FL_FREE(H5MF_free_section_t, sect); + /* Check if second section adjoins first section */ + ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr); + if(ret_value > 0) + /* If they are on different pages, couldn't merge */ + if((sect1->sect_info.addr / udata->f->shared->fs_page_size) != (((sect2->sect_info.addr + sect2->sect_info.size - 1) / udata->f->shared->fs_page_size))) + ret_value = FALSE; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sect_simple_free() */ +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_can_merge() */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_valid + * Function: H5MF_sect_small_merge * - * Purpose: Check the validity of a section + * Purpose: Merge two sections of this type + * + * Note: Second section always merges into first node. + * If the size of the "merged" section is equal to file space page size, + * free the section. * * Return: Success: non-negative * Failure: negative * - * Programmer: Quincey Koziol - * Tuesday, January 8, 2008 + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ static herr_t -H5MF_sect_simple_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls, - const H5FS_section_info_t -#ifdef NDEBUG - H5_ATTR_UNUSED -#endif /* NDEBUG */ - *_sect, hid_t H5_ATTR_UNUSED dxpl_id) +H5MF_sect_small_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, + void *_udata) { -#ifndef NDEBUG - const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ -#endif /* NDEBUG */ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ + H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect1); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(sect2); + HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); + + /* Add second section's size to first section */ + (*sect1)->sect_info.size += sect2->sect_info.size; + + if((*sect1)->sect_info.size == udata->f->shared->fs_page_size) { + if(H5MF_xfree(udata->f, udata->alloc_type, udata->dxpl_id, (*sect1)->sect_info.addr, (*sect1)->sect_info.size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section") + + /* Need to free possible metadata page in the PB cache */ + /* This is in response to the data corruption bug from fheap.c with page buffering + page strategy */ + /* Note: Large metadata page bypasses the PB cache */ + /* Note: Update of raw data page (large or small sized) is handled by the PB cache */ + if(udata->f->shared->page_buf != NULL && udata->alloc_type != H5FD_MEM_DRAW) + if(H5PB_remove_entry(udata->f, (*sect1)->sect_info.addr) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section") + + if(H5MF_sect_free((H5FS_section_info_t *)(*sect1)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + *sect1 = NULL; + } /* end if */ + + /* Get rid of second section */ + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_merge() */ + +/* + * "Large" section callbacks + */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_can_merge (same as H5MF_sect_simple_can_merge) + * + * Purpose: Can two sections of this type merge? + * + * Note: Second section must be "after" first section + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_large_can_merge(const H5FS_section_info_t *_sect1, + const H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) +{ + const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */ + const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR /* Check arguments. */ + HDassert(sect1); + HDassert(sect2); + HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */ + HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)); + + ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_can_merge() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_merge (same as H5MF_sect_simple_merge) + * + * Purpose: Merge two sections of this type + * + * Note: Second section always merges into first node + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_large_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, + void H5_ATTR_UNUSED *_udata) +{ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ + H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect1); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(sect2); + HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); + + /* Add second section's size to first section */ + (*sect1)->sect_info.size += sect2->sect_info.size; + + /* Get rid of second section */ + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_merge() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_can_shrink + * + * Purpose: Can this section shrink the container? + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect, void *_udata) +{ + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t eoa; /* End of address space in the file */ + haddr_t end; /* End of section to extend */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ HDassert(sect); + HDassert(sect->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(udata); + HDassert(udata->f); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sect_simple_valid() */ + /* Retrieve the end of the file's address space */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* Compute address of end of section to check */ + end = sect->sect_info.addr + sect->sect_info.size; + + /* Check if the section is exactly at the end of the allocated space in the file */ + if(H5F_addr_eq(end, eoa) && sect->sect_info.size >= udata->f->shared->fs_page_size) { + /* Set the shrinking type */ + udata->shrink = H5MF_SHRINK_EOA; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Indicate shrinking can occur */ + HGOTO_DONE(TRUE) + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_can_shrink() */ + /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_split + * Function: H5MF_sect_large_shrink * - * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section - * SECT's addr and size are updated to point to the aligned section + * Purpose: Shrink a large-sized section * - * Return: Success: the fragment for aligning sect - * Failure: null + * Return: Success: non-negative + * Failure: negative * - * Programmer: Vailin Choi, July 29, 2008 + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ -static H5FS_section_info_t * -H5MF_sect_simple_split(H5FS_section_info_t *sect, hsize_t frag_size) +static herr_t +H5MF_sect_large_shrink(H5FS_section_info_t **_sect, void *_udata) { - H5MF_free_section_t *ret_value = NULL; /* Return value */ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + hsize_t frag_size = 0; /* Fragment size */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT - /* Allocate space for new section */ - if(NULL == (ret_value = H5MF_sect_simple_new(sect->addr, frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") + /* Check arguments. */ + HDassert(sect); + HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(udata); + HDassert(udata->f); + HDassert(udata->shrink == H5MF_SHRINK_EOA); + HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); + HDassert(H5F_PAGED_AGGR(udata->f)); - /* Set new section's info */ - sect->addr += frag_size; - sect->size -= frag_size; + /* Calculate possible mis-aligned fragment */ + H5MF_EOA_MISALIGN(udata->f, (*sect)->sect_info.addr, udata->f->shared->fs_page_size, frag_size); + + /* Free full pages from EOA */ + /* Retain partial page in the free-space manager so as to keep EOA at page boundary */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr+frag_size, (*sect)->sect_info.size-frag_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + + if(frag_size) /* Adjust section size for the partial page */ + (*sect)->sect_info.size = frag_size; + else { + /* Free section */ + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + + /* Mark section as freed, for free space manager */ + *sect = NULL; + } /* end else */ done: - FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value) -} /* end H5MF_sect_simple_split() */ + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_shrink() */ diff --git a/src/H5Oalloc.c b/src/H5Oalloc.c index 4f98cfa..40c9c38 100644 --- a/src/H5Oalloc.c +++ b/src/H5Oalloc.c @@ -2023,6 +2023,7 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh) /* Second message has been merged, delete it */ if(merged_msg) { H5O_chunk_proxy_t *curr_chk_proxy; /* Chunk that message is in */ + htri_t result; /* Release any information/memory for second message */ H5O_msg_free_mesg(curr_msg2); @@ -2050,6 +2051,13 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh) /* (Don't bother reducing size of message array for now -QAK) */ oh->nmesgs--; + /* The merge null message might span the entire chunk: scan for empty chunk to remove */ + if((result = H5O_remove_empty_chunks(f, dxpl_id, oh)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPACK, FAIL, "can't remove empty chunk") + else if(result > 0) + /* Get out of loop */ + break; + /* If the merged message is too large, shrink the chunk */ if(curr_msg->raw_size >= H5O_MESG_MAX_SIZE) if(H5O_alloc_shrink_chunk(f, dxpl_id, oh, curr_msg->chunkno) < 0) diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c index 65f6aa2..bea99a2 100644 --- a/src/H5Ocache_image.c +++ b/src/H5Ocache_image.c @@ -33,6 +33,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Opkg.h" /* Object headers */ #include "H5MFprivate.h" /* File space management */ @@ -290,9 +291,50 @@ H5O__mdci_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg) HDassert(mesg); /* Free file space for cache image */ - if(H5F_addr_defined(mesg->addr)) - if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block") + if(H5F_addr_defined(mesg->addr)) { + /* The space for the cache image block was allocated directly + * from the VFD layer at the end of file. As this was the + * last file space allocation before shutdown, the cache image + * should still be the last item in the file. + * + * If the hack to work around the self referential free space + * manager issue is in use, file space for the non-empty self + * referential free space managers was also allocated from VFD + * layer at the end of file. Since these allocations directly + * preceeded the cache image allocation they should be directly + * adjacent to the cache image block at the end of file. + * + * In this case, just call H5MF_tidy_self_referential_fsm_hack(). + * + * That routine will float the self referential free space + * managers, and reduce the eoa to its value just prior to + * allocation of space for same. Since the cache image appears + * just after the self referential free space managers, this + * will release the file space for the cache image as well. + * + * Note that in this case, there must not have been any file + * space allocations / deallocations prior to the free of the + * cache image. Verify this to the extent possible. + * + * If the hack to work around the persistant self referential + * free space manager issue is NOT in use, just call H5MF_xfree() + * to release the cache iamge. In principle, we should be able + * to just reduce the EOA to the base address of the cache + * image block, as there shouldn't be any file space allocation + * before the first metadata cache access. However, given + * time constraints, I don't want to go there now. + */ + if(H5F_FIRST_ALLOC_DEALLOC(f)) { + HDassert(HADDR_UNDEF !=H5F_EOA_PRE_FSM_FSALLOC(f)); + HDassert(H5F_addr_ge(mesg->addr, H5F_EOA_PRE_FSM_FSALLOC(f))); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end if */ + else { + if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block") + } /* end else */ + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c index 938d319..37ec627 100644 --- a/src/H5Ofsinfo.c +++ b/src/H5Ofsinfo.c @@ -23,12 +23,13 @@ * *------------------------------------------------------------------------- */ - +#define H5F_FRIEND /*suppress error about including H5Fpkg */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ #include "H5FLprivate.h" /* Free lists */ #include "H5Opkg.h" /* Object headers */ @@ -66,7 +67,8 @@ const H5O_msg_class_t H5O_MSG_FSINFO[1] = {{ }}; /* Current version of free-space manager info information */ -#define H5O_FSINFO_VERSION 0 +#define H5O_FSINFO_VERSION_0 0 +#define H5O_FSINFO_VERSION_1 1 /* Declare a free list to manage the H5O_fsinfo_t struct */ H5FL_DEFINE_STATIC(H5O_fsinfo_t); @@ -85,12 +87,13 @@ H5FL_DEFINE_STATIC(H5O_fsinfo_t); *------------------------------------------------------------------------- */ static void * -H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh, +H5O_fsinfo_decode(H5F_t *f, hid_t dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, const uint8_t *p) { - H5O_fsinfo_t *fsinfo = NULL; /* free-space manager info */ - H5FD_mem_t type; /* Memory type for iteration */ - void *ret_value = NULL; /* Return value */ + H5O_fsinfo_t *fsinfo = NULL; /* File space info message */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + unsigned vers; /* message version */ + void *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -98,26 +101,83 @@ H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED * HDassert(f); HDassert(p); - /* Version of message */ - if(*p++ != H5O_FSINFO_VERSION) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") - /* Allocate space for message */ if(NULL == (fsinfo = H5FL_CALLOC(H5O_fsinfo_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - fsinfo->strategy = (H5F_file_space_type_t)*p++; /* file space strategy */ - H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* free space section size threshold */ + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo->fs_addr[ptype - 1] = HADDR_UNDEF; - /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */ - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1])); - } /* end if */ - else { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo->fs_addr[type-1] = HADDR_UNDEF; - } /* end else */ + /* Version of message */ + vers = *p++; + + if(vers == H5O_FSINFO_VERSION_0) { + H5F_file_space_type_t strategy; /* Strategy */ + hsize_t threshold; /* Threshold */ + H5FD_mem_t type; /* Memory type for iteration */ + + fsinfo->persist = H5F_FREE_SPACE_PERSIST_DEF; + fsinfo->threshold = H5F_FREE_SPACE_THRESHOLD_DEF; + fsinfo->page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF; + fsinfo->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; + fsinfo->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + strategy = (H5F_file_space_type_t)*p++; /* File space strategy */ + H5F_DECODE_LENGTH(f, p, threshold); /* Free-space section threshold */ + + /* Map version 0 (deprecated) to version 1 message */ + switch(strategy) { + + case H5F_FILE_SPACE_ALL_PERSIST: + fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + fsinfo->persist = TRUE; + fsinfo->threshold = threshold; + if(HADDR_UNDEF == (fsinfo->eoa_pre_fsm_fsalloc = H5F_get_eoa(f, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file size") + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1])); + break; + + case H5F_FILE_SPACE_ALL: + fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + fsinfo->threshold = threshold; + break; + + case H5F_FILE_SPACE_AGGR_VFD: + fsinfo->strategy = H5F_FSPACE_STRATEGY_AGGR; + break; + + case H5F_FILE_SPACE_VFD: + fsinfo->strategy = H5F_FSPACE_STRATEGY_NONE; + break; + + case H5F_FILE_SPACE_NTYPES: + case H5F_FILE_SPACE_DEFAULT: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file space strategy") + } /* end switch */ + + fsinfo->mapped = TRUE; + + } else { + HDassert(vers == H5O_FSINFO_VERSION_1); + + fsinfo->strategy = (H5F_fspace_strategy_t)*p++; /* File space strategy */ + fsinfo->persist = *p++; /* Free-space persist or not */ + H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section threshold */ + + H5F_DECODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ + UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metdata threshold */ + H5F_addr_decode(f, &p, &(fsinfo->eoa_pre_fsm_fsalloc)); /* EOA before free-space header and section info */ + + /* Decode addresses of free space managers, if persisting */ + if(fsinfo->persist) { + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + H5F_addr_decode(f, &p, &(fsinfo->fs_addr[ptype - 1])); + } /* end if */ + + fsinfo->mapped = FALSE; + } /* Set return value */ ret_value = fsinfo; @@ -145,7 +205,7 @@ static herr_t H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, const void *_mesg) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg; - H5FD_mem_t type; /* Memory type for iteration */ + H5F_mem_page_t ptype; /* Memory type for iteration */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -154,14 +214,20 @@ H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, c HDassert(p); HDassert(fsinfo); - *p++ = H5O_FSINFO_VERSION; /* message version */ - *p++ = fsinfo->strategy; /* file space strategy */ - H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* free-space section size threshold */ + *p++ = H5O_FSINFO_VERSION_1; /* message version */ + *p++ = fsinfo->strategy; /* File space strategy */ + *p++ = (unsigned char)fsinfo->persist; /* Free-space persist or not */ + H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section size threshold */ + + H5F_ENCODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ + UINT16ENCODE(p, fsinfo->pgend_meta_thres); /* Page end metadata threshold */ + H5F_addr_encode(f, &p, fsinfo->eoa_pre_fsm_fsalloc); /* EOA before free-space header and section info */ - /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */ - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - H5F_addr_encode(f, &p, fsinfo->fs_addr[type-1]); + /* Store addresses of free-space managers, if persisting */ + if(fsinfo->persist) { + /* Addresses of free-space managers */ + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + H5F_addr_encode(f, &p, fsinfo->fs_addr[ptype - 1]); } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) @@ -224,19 +290,19 @@ static size_t H5O_fsinfo_size(const H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, const void *_mesg) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg; - size_t fs_addr_size = 0; size_t ret_value = 0; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Addresses of free-space managers exist only for H5F_FILE_SPACE_ALL_PERSIST type */ - if(H5F_FILE_SPACE_ALL_PERSIST == fsinfo->strategy) - fs_addr_size = (H5FD_MEM_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f); - - ret_value = 2 /* Version & strategy */ - + (size_t)H5F_SIZEOF_SIZE(f) /* Threshold */ - + fs_addr_size; /* Addresses of free-space managers */ + ret_value = 3 /* Version, strategy & persist */ + + (size_t)H5F_SIZEOF_SIZE(f) /* Free-space section threshold */ + + (size_t)H5F_SIZEOF_SIZE(f) /* File space page size */ + + 2 /* Page end meta threshold */ + + (size_t)H5F_SIZEOF_ADDR(f); + + /* Free-space manager addresses */ + if(fsinfo->persist) + ret_value += (H5F_MEM_PAGE_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_fsinfo_size() */ @@ -282,7 +348,7 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo int indent, int fwidth) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *) _mesg; - H5FD_mem_t type; /* Memory type for iteration */ + H5F_mem_page_t ptype; /* Free-space types for iteration */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -293,16 +359,48 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo HDassert(indent >= 0); HDassert(fwidth >= 0); - HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, - "File space strategy:", fsinfo->strategy); + HDfprintf(stream, "%*s%-*s ", indent, "", fwidth, "File space strategy:"); + switch(fsinfo->strategy) { + case H5F_FSPACE_STRATEGY_FSM_AGGR: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_FSM_AGGR"); + break; + + case H5F_FSPACE_STRATEGY_PAGE: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_PAGE"); + break; + + case H5F_FSPACE_STRATEGY_AGGR: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_AGGR"); + break; + + case H5F_FSPACE_STRATEGY_NONE: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_NONE"); + break; + + case H5F_FSPACE_STRATEGY_NTYPES: + default: + HDfprintf(stream, "%s\n", "unknown"); + } /* end switch */ + + HDfprintf(stream, "%*s%-*s %t\n", indent, "", fwidth, + "Free-space persist:", fsinfo->persist); HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth, - "Free space section threshold:", fsinfo->threshold); + "Free-space section threshold:", fsinfo->threshold); + + HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth, + "File space page size:", fsinfo->page_size); + + HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, + "Page end metadata threshold:", fsinfo->pgend_meta_thres); + + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "eoa_pre_fsm_fsalloc:", fsinfo->eoa_pre_fsm_fsalloc); - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, - "Free space manager address:", fsinfo->fs_addr[type-1]); + if(fsinfo->persist) { + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "Free space manager address:", fsinfo->fs_addr[ptype-1]); } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) diff --git a/src/H5Omessage.c b/src/H5Omessage.c index 376c888..7e6463a 100644 --- a/src/H5Omessage.c +++ b/src/H5Omessage.c @@ -2253,3 +2253,55 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_flush_msgs() */ + +/*------------------------------------------------------------------------- + * Function: H5O_msg_get_flags + * + * Purpose: Queries a message's message flags in the object header + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin; Jan 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags) +{ + H5O_t *oh = NULL; /* Object header to use */ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + H5O_mesg_t *idx_msg; /* Pointer to message to modify */ + unsigned idx; /* Index of message to modify */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(loc); + HDassert(loc->file); + HDassert(H5F_addr_defined(loc->addr)); + HDassert(type_id < NELMTS(H5O_msg_class_g)); + type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ + HDassert(type); + + /* Get the object header */ + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Locate message of correct type */ + for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++) + if(type == idx_msg->type) + break; + + if(idx == oh->nmesgs) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found") + + /* Set return value */ + *flags = idx_msg->flags; + +done: + if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_msg_get_flags() */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 71c512c..f0fbe72 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -791,9 +791,16 @@ typedef unsigned H5O_unknown_t; /* Original message type ID */ * (Data structure in memory) */ typedef struct H5O_fsinfo_t { - H5F_file_space_type_t strategy; /* File space strategy */ - hsize_t threshold; /* Free space section threshold */ - haddr_t fs_addr[H5FD_MEM_NTYPES-1]; /* Addresses of free space managers */ + H5F_fspace_strategy_t strategy; /* File space strategy */ + hbool_t persist; /* Persisting free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + hsize_t page_size; /* For paged aggregation: file space page size */ + size_t pgend_meta_thres; /* For paged aggregation: page end metadata threshold */ + haddr_t eoa_pre_fsm_fsalloc; /* For paged aggregation: the eoa before free-space headers & sinfo */ + haddr_t fs_addr[H5F_MEM_PAGE_NTYPES - 1]; /* 13 addresses of free-space managers */ + /* For non-paged aggregation: only 6 addresses are used */ + hbool_t mapped; /* Not stored */ + /* Indicate the message is mapped from version 0 to version 1 */ } H5O_fsinfo_t; /* @@ -920,6 +927,7 @@ H5_DLL void* H5O_msg_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, const unsigned char *buf); H5_DLL herr_t H5O_msg_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, void *mesg); +H5_DLL herr_t H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags); /* Object metadata flush/refresh routines */ H5_DLL herr_t H5O_flush_common(H5O_loc_t *oloc, hid_t obj_id, hid_t dxpl_id); diff --git a/src/H5PB.c b/src/H5PB.c new file mode 100644 index 0000000..c67ae59 --- /dev/null +++ b/src/H5PB.c @@ -0,0 +1,1533 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5PB.c + * + * Purpose: Page Buffer routines. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#include "H5PBmodule.h" /* This source code file is part of the H5PB module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5PBpkg.h" /* File access */ +#include "H5SLprivate.h" /* Skip List */ + + +/****************/ +/* Local Macros */ +/****************/ +#define H5PB__PREPEND(page_ptr, head_ptr, tail_ptr, len) { \ + if((head_ptr) == NULL) { \ + (head_ptr) = (page_ptr); \ + (tail_ptr) = (page_ptr); \ + } /* end if */ \ + else { \ + (head_ptr)->prev = (page_ptr); \ + (page_ptr)->next = (head_ptr); \ + (head_ptr) = (page_ptr); \ + } /* end else */ \ + (len)++; \ +} /* H5PB__PREPEND() */ + +#define H5PB__REMOVE(page_ptr, head_ptr, tail_ptr, len) { \ + if((head_ptr) == (page_ptr)) { \ + (head_ptr) = (page_ptr)->next; \ + if((head_ptr) != NULL) \ + (head_ptr)->prev = NULL; \ + } /* end if */ \ + else \ + (page_ptr)->prev->next = (page_ptr)->next; \ + if((tail_ptr) == (page_ptr)) { \ + (tail_ptr) = (page_ptr)->prev; \ + if((tail_ptr) != NULL) \ + (tail_ptr)->next = NULL; \ + } /* end if */ \ + else \ + (page_ptr)->next->prev = (page_ptr)->prev; \ + page_ptr->next = NULL; \ + page_ptr->prev = NULL; \ + (len)--; \ +} + +#define H5PB__INSERT_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* insert the entry at the head of the list. */ \ + H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + +#define H5PB__REMOVE_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* remove the entry from the list. */ \ + H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + +#define H5PB__MOVE_TO_TOP_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* Remove entry and insert at the head of the list. */ \ + H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ + H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + + +/******************/ +/* Local Typedefs */ +/******************/ + +/* Iteration context for destroying page buffer */ +typedef struct { + H5PB_t *page_buf; + hbool_t actual_slist; +} H5PB_ud1_t; + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ +static herr_t H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry); +static htri_t H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf, H5FD_mem_t inserted_type); +static herr_t H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ +/* Declare a free list to manage the H5PB_t struct */ +H5FL_DEFINE_STATIC(H5PB_t); + +/* Declare a free list to manage the H5PB_entry_t struct */ +H5FL_DEFINE_STATIC(H5PB_entry_t); + + + +/*------------------------------------------------------------------------- + * Function: H5PB_reset_stats + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Reset statistics collected for the page buffer layer. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_reset_stats(H5PB_t *page_buf) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + + page_buf->accesses[0] = 0; + page_buf->accesses[1] = 0; + page_buf->hits[0] = 0; + page_buf->hits[1] = 0; + page_buf->misses[0] = 0; + page_buf->misses[1] = 0; + page_buf->evictions[0] = 0; + page_buf->evictions[1] = 0; + page_buf->bypasses[0] = 0; + page_buf->bypasses[1] = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_reset_stats() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_get_stats + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Retrieve statistics collected about page accesses for the page buffer layer. + * --accesses: the number of metadata and raw data accesses to the page buffer layer + * --hits: the number of metadata and raw data hits in the page buffer layer + * --misses: the number of metadata and raw data misses in the page buffer layer + * --evictions: the number of metadata and raw data evictions from the page buffer layer + * --bypasses: the number of metadata and raw data accesses that bypass the page buffer layer + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2], unsigned hits[2], + unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + + accesses[0] = page_buf->accesses[0]; + accesses[1] = page_buf->accesses[1]; + hits[0] = page_buf->hits[0]; + hits[1] = page_buf->hits[1]; + misses[0] = page_buf->misses[0]; + misses[1] = page_buf->misses[1]; + evictions[0] = page_buf->evictions[0]; + evictions[1] = page_buf->evictions[1]; + bypasses[0] = page_buf->bypasses[0]; + bypasses[1] = page_buf->bypasses[1]; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_get_stats */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_print_stats() + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Print out statistics collected for the page buffer layer. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_print_stats(const H5PB_t *page_buf) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(page_buf); + + printf("PAGE BUFFER STATISTICS:\n"); + + printf("******* METADATA\n"); + printf("\t Total Accesses: %u\n", page_buf->accesses[0]); + printf("\t Hits: %u\n", page_buf->hits[0]); + printf("\t Misses: %u\n", page_buf->misses[0]); + printf("\t Evictions: %u\n", page_buf->evictions[0]); + printf("\t Bypasses: %u\n", page_buf->bypasses[0]); + printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[0]/(page_buf->accesses[0] - page_buf->bypasses[0]))*100); + printf("*****************\n\n"); + + printf("******* RAWDATA\n"); + printf("\t Total Accesses: %u\n", page_buf->accesses[1]); + printf("\t Hits: %u\n", page_buf->hits[1]); + printf("\t Misses: %u\n", page_buf->misses[1]); + printf("\t Evictions: %u\n", page_buf->evictions[1]); + printf("\t Bypasses: %u\n", page_buf->bypasses[1]); + printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[1]/(page_buf->accesses[1]-page_buf->bypasses[0]))*100); + printf("*****************\n\n"); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_print_stats */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_create + * + * Purpose: Create and setup the PB on the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc) +{ + H5PB_t *page_buf = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + + /* Check args */ + if(f->shared->fs_strategy != H5F_FSPACE_STRATEGY_PAGE) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "Enabling Page Buffering requires PAGE file space strategy") + /* round down the size if it is larger than the page size */ + else if(size > f->shared->fs_page_size) { + hsize_t temp_size; + + temp_size = (size / f->shared->fs_page_size) * f->shared->fs_page_size; + H5_CHECKED_ASSIGN(size, size_t, temp_size, hsize_t); + } /* end if */ + else if(0 != size % f->shared->fs_page_size) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "Page Buffer size must be >= to the page size") + + /* Allocate the new page buffering structure */ + if(NULL == (page_buf = H5FL_CALLOC(H5PB_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + page_buf->max_size = size; + H5_CHECKED_ASSIGN(page_buf->page_size, size_t, f->shared->fs_page_size, hsize_t); + page_buf->min_meta_perc = page_buf_min_meta_perc; + page_buf->min_raw_perc = page_buf_min_raw_perc; + + /* Calculate the minimum page count for metadata and raw data + * based on the fractions provided + */ + page_buf->min_meta_count = (unsigned)((size * page_buf_min_meta_perc) / (f->shared->fs_page_size * 100)); + page_buf->min_raw_count = (unsigned)((size * page_buf_min_raw_perc) / (f->shared->fs_page_size * 100)); + + if(NULL == (page_buf->slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list") + if(NULL == (page_buf->mf_slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list") + + if(NULL == (page_buf->page_fac = H5FL_fac_init(page_buf->page_size))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "can't create page factory") + + f->shared->page_buf = page_buf; + +done: + if(ret_value < 0) { + if(page_buf != NULL) { + if(page_buf->slist_ptr != NULL) + H5SL_close(page_buf->slist_ptr); + if(page_buf->mf_slist_ptr != NULL) + H5SL_close(page_buf->mf_slist_ptr); + if(page_buf->page_fac != NULL) + H5FL_fac_term(page_buf->page_fac); + page_buf = H5FL_FREE(H5PB_t, page_buf); + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_create */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__flush_cb + * + * Purpose: Callback to flush PB skiplist entries. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__flush_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) +{ + H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */ + const H5F_io_info2_t *fio_info = (const H5F_io_info2_t *)_op_data; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(page_entry); + HDassert(fio_info); + + /* Flush the page if it's dirty */ + if(page_entry->is_dirty) + if(H5PB__write_entry(fio_info, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB__flush_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_flush + * + * Purpose: Flush/Free all the PB entries to the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_flush(const H5F_io_info2_t *fio_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(fio_info); + HDassert(fio_info->f); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); + + /* Flush all the entries in the PB skiplist, if we have write access on the file */ + if(fio_info->f->shared->page_buf && (H5F_ACC_RDWR & H5F_INTENT(fio_info->f))) { + H5PB_t *page_buf = fio_info->f->shared->page_buf; + + /* Iterate over all entries in page buffer skip list */ + if(H5SL_iterate(page_buf->slist_ptr, H5PB__flush_cb, (void *)fio_info)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADITER, FAIL, "can't flush page buffer skip list") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_flush */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__dest_cb + * + * Purpose: Callback to free PB skiplist entries. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) +{ + H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */ + H5PB_ud1_t *op_data = (H5PB_ud1_t *)_op_data; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checking */ + HDassert(page_entry); + HDassert(op_data); + HDassert(op_data->page_buf); + + /* Remove entry from LRU list */ + if(op_data->actual_slist) { + H5PB__REMOVE_LRU(op_data->page_buf, page_entry) + page_entry->page_buf_ptr = H5FL_FAC_FREE(op_data->page_buf->page_fac, page_entry->page_buf_ptr); + } /* end if */ + + /* Free page entry */ + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB__dest_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_dest + * + * Purpose: destroy the PB on the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_dest(H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + + /* Destroy page buffer info, if there is any */ + if(f->shared->page_buf) { + H5PB_t *page_buf = f->shared->page_buf; + H5PB_ud1_t op_data; /* Iteration context */ + + /* Set up context info */ + op_data.page_buf = page_buf; + + /* Destroy the skip list containing all the entries in the PB */ + op_data.actual_slist = TRUE; + if(H5SL_destroy(page_buf->slist_ptr, H5PB__dest_cb, &op_data)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list") + + /* Destroy the skip list containing the new entries */ + op_data.actual_slist = FALSE; + if(H5SL_destroy(page_buf->mf_slist_ptr, H5PB__dest_cb, &op_data)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list") + + /* Destroy the page factory */ + if(H5FL_fac_term(page_buf->page_fac) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTRELEASE, FAIL, "can't destroy page buffer page factory") + +#ifdef QAK +H5PB_print_stats(page_buf); +#endif /* QAK */ + + f->shared->page_buf = H5FL_FREE(H5PB_t, page_buf); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_dest */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_add_new_page + * + * Purpose: Add a new page to the new page skip list. This is called + * from the MF layer when a new page is allocated to + * indicate to the page buffer layer that a read of the page + * from the file is not necessary since it's an empty page. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr) +{ + H5PB_t *page_buf = f->shared->page_buf; + H5PB_entry_t *page_entry = NULL; /* pointer to the corresponding page entry */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(page_buf); + + /* If there is an existing page, this means that at some point the + * file free space manager freed and re-allocated a page at the same + * address. No need to do anything here then... + */ + /* MSC - to be safe, might want to dig in the MF layer and remove + * the page when it is freed from this list if it still exists and + * remove this check + */ + if(NULL == H5SL_search(page_buf->mf_slist_ptr, &(page_addr))) { + /* Create the new PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Initialize page fields */ + page_entry->addr = page_addr; + page_entry->type = (H5F_mem_page_t)type; + page_entry->is_dirty = FALSE; + + /* Insert entry in skip list */ + if(H5SL_insert(page_buf->mf_slist_ptr, page_entry, &(page_entry->addr)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Can't insert entry in skip list") + } /* end if */ + +done: + if(ret_value < 0) + if(page_entry) + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_add_new_page */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_update_entry + * + * Purpose: In PHDF5, entries that are written by other processes and just + * marked clean by this process have to have their corresponding + * pages updated if they exist in the page buffer. + * This routine checks and update the pages. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf) +{ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + haddr_t page_addr; + + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + HDassert(size <= page_buf->page_size); + HDassert(buf); + + /* calculate the aligned address of the first page */ + page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* search for the page and update if found */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&page_addr)); + if(page_entry) { + haddr_t offset; + + HDassert(addr + size <= page_addr + page_buf->page_size); + offset = addr - page_addr; + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, size); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_update_entry */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_remove_entry + * + * Purpose: Remove possible metadata entry with ADDR from the PB cache. + * This is in response to the data corruption bug from fheap.c + * with page buffering + page strategy. + * Note: Large metadata page bypasses the PB cache. + * Note: Update of raw data page (large or small sized) is handled by the PB cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Feb 2017 + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_remove_entry(const H5F_t *f, haddr_t addr) +{ + H5PB_t *page_buf = f->shared->page_buf; + H5PB_entry_t *page_entry = NULL; /* pointer to the page entry being searched */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(page_buf); + + /* Search for address in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&addr)); + + /* If found, remove the entry from the PB cache */ + if(page_entry) { + HDassert(page_entry->type != H5F_MEM_PAGE_DRAW); + if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr))) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Page Entry is not in skip list") + + /* Remove from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len); + + page_buf->meta_count--; + + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_remove_entry */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_read + * + * Purpose: Reads in the data from the page containing it if it exists + * in the PB cache; otherwise reads in the page through the VFD. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, + size_t size, void *buf/*out*/) +{ + H5PB_t *page_buf; /* Page buffering info for this file */ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ + haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */ + haddr_t offset; + haddr_t search_addr; /* Address of current page */ + hsize_t num_touched_pages; /* Number of pages accessed */ + size_t access_size; + hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */ + hsize_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(fio_info); + + /* Get pointer to page buffer info for this file */ + page_buf = fio_info->f->shared->page_buf; + +#ifdef H5_HAVE_PARALLEL + if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) { +#if 1 + bypass_pb = TRUE; +#else + /* MSC - why this stopped working ? */ + int mpi_size; + + if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") + if(1 != mpi_size) + bypass_pb = TRUE; +#endif + } /* end if */ +#endif + + /* If page buffering is disabled, or the I/O size is larger than that of a + * single page, or if this is a parallel raw data access, bypass page + * buffering. + */ + if(NULL == page_buf || size >= page_buf->page_size || + (bypass_pb && H5FD_MEM_DRAW == type)) { + if(H5F__accum_read(fio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "read through metadata accumulator failed") + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->bypasses[1] ++; + else + page_buf->bypasses[0] ++; + } /* end if */ + + /* If page buffering is disabled, or if this is a large metadata access, + * or if this is parallel raw data access, we are done here + */ + if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) || + (bypass_pb && H5FD_MEM_DRAW == type)) + HGOTO_DONE(SUCCEED) + } /* end if */ + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->accesses[1]++; + else + page_buf->accesses[0]++; + } /* end if */ + + /* Calculate the aligned address of the first page */ + first_page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* For Raw data calculate the aligned address of the last page and + * the number of pages accessed if more than 1 page is accessed + */ + if(H5FD_MEM_DRAW == type) { + last_page_addr = ((addr + size - 1) / page_buf->page_size) * page_buf->page_size; + + /* How many pages does this write span */ + num_touched_pages = (last_page_addr / page_buf->page_size + 1) - + (first_page_addr / page_buf->page_size); + if(first_page_addr == last_page_addr) { + HDassert(1 == num_touched_pages); + last_page_addr = HADDR_UNDEF; + } /* end if */ + } /* end if */ + /* Otherwise set last page addr to HADDR_UNDEF */ + else { + num_touched_pages = 1; + last_page_addr = HADDR_UNDEF; + } /* end else */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + /* Copy raw data from dirty pages into the read buffer if the read + request spans pages in the page buffer*/ + if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) { + H5SL_node_t *node; + + /* For each touched page in the page buffer, check if it + * exists in the page Buffer and is dirty. If it does, we + * update the buffer with what's in the page so we get the up + * to date data into the buffer after the big read from the file. + */ + node = H5SL_find(page_buf->slist_ptr, (void *)(&first_page_addr)); + for(i = 0; i < num_touched_pages; i++) { + search_addr = i*page_buf->page_size + first_page_addr; + + /* if we still haven't located a starting page, search again */ + if(!node && i!=0) + node = H5SL_find(page_buf->slist_ptr, (void *)(&search_addr)); + + /* if the current page is in the Page Buffer, do the updates */ + if(node) { + page_entry = (H5PB_entry_t *)H5SL_item(node); + + HDassert(page_entry); + + /* If the current page address falls out of the access + block, then there are no more pages to go over */ + if(page_entry->addr >= addr + size) + break; + + HDassert(page_entry->addr == search_addr); + + if(page_entry->is_dirty) { + /* special handling for the first page if it is not a full page access */ + if(i == 0 && first_page_addr != addr) { + offset = addr - first_page_addr; + HDassert(page_buf->page_size > offset); + + HDmemcpy(buf, (uint8_t *)page_entry->page_buf_ptr + offset, + page_buf->page_size - (size_t)offset); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + /* special handling for the last page if it is not a full page access */ + else if(num_touched_pages > 1 && i == num_touched_pages-1 && search_addr < addr+size) { + offset = (num_touched_pages-2)*page_buf->page_size + + (page_buf->page_size - (addr - first_page_addr)); + + HDmemcpy((uint8_t *)buf + offset, page_entry->page_buf_ptr, + (size_t)((addr + size) - last_page_addr)); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end else-if */ + /* copy the entire fully accessed pages */ + else { + offset = i*page_buf->page_size; + + HDmemcpy((uint8_t *)buf+(i*page_buf->page_size) , page_entry->page_buf_ptr, + page_buf->page_size); + } /* end else */ + } /* end if */ + node = H5SL_next(node); + } /* end if */ + } /* end for */ + } /* end if */ + else { + /* A raw data access could span 1 or 2 PB entries at this point so + we need to handle that */ + HDassert(1 == num_touched_pages || 2 == num_touched_pages); + for(i = 0 ; i < num_touched_pages; i++) { + haddr_t buf_offset; + + /* Calculate the aligned address of the page to search for it in the skip list */ + search_addr = (0==i ? first_page_addr : last_page_addr); + + /* Calculate the access size if the access spans more than 1 page */ + if(1 == num_touched_pages) + access_size = size; + else + access_size = (0 == i ? (size_t)((first_page_addr + page_buf->page_size) - addr) : (size - access_size)); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + + /* if found */ + if(page_entry) { + offset = (0 == i ? addr - page_entry->addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* copy the requested data from the page into the input buffer */ + HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)page_entry->page_buf_ptr + offset, access_size); + + /* Update LRU */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* if not found */ + else { + void *new_page_buf = NULL; + size_t page_size = page_buf->page_size; + haddr_t eoa; + + /* make space for new entry */ + if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) { + htri_t can_make_space; + + /* check if we can make space in page buffer */ + if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") + + /* if make_space returns 0, then we can't use the page + buffer for this I/O and we need to bypass */ + if(0 == can_make_space) { + /* make space can't return FALSE on second touched page since the first is of the same type */ + HDassert(0 == i); + + /* read entire block from VFD and return */ + if(H5FD_read(&fdio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Break out of loop */ + break; + } /* end if */ + } /* end if */ + + /* Read page from VFD */ + if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + + /* Read page through the VFD layer, but make sure we don't read past the EOA. */ + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the entire page falls outside the EOA, then fail */ + if(search_addr > eoa) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "reading an entire page that is outside the file EOA") + + /* Adjust the read size to not go beyond the EOA */ + if(search_addr + page_size > eoa) + page_size = (size_t)(eoa - search_addr); + + /* Read page from VFD */ + if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Copy the requested data from the page into the input buffer */ + offset = (0 == i ? addr - search_addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)new_page_buf + offset, access_size); + + /* Create the new PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + page_entry->page_buf_ptr = new_page_buf; + page_entry->addr = search_addr; + page_entry->type = (H5F_mem_page_t)type; + page_entry->is_dirty = FALSE; + + /* Insert page into PB */ + if(H5PB__insert_entry(page_buf, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer") + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->misses[1]++; + else + page_buf->misses[0]++; + } /* end else */ + } /* end for */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_write + * + * Purpose: Write data into the Page Buffer. If the page exists in the + * cache, update it; otherwise read it from disk, update it, and + * insert into cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_write(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, + size_t size, const void *buf) +{ + H5PB_t *page_buf; /* Page buffering info for this file */ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ + haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */ + haddr_t offset; + haddr_t search_addr; /* Address of current page */ + hsize_t num_touched_pages; /* Number of pages accessed */ + size_t access_size; + hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */ + hsize_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(fio_info); + HDassert(fio_info->f); + + /* Get pointer to page buffer info for this file */ + page_buf = fio_info->f->shared->page_buf; + +#ifdef H5_HAVE_PARALLEL + if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) { +#if 1 + bypass_pb = TRUE; +#else + /* MSC - why this stopped working ? */ + int mpi_size; + + if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") + if(1 != mpi_size) + bypass_pb = TRUE; +#endif + } /* end if */ +#endif + + /* If page buffering is disabled, or the I/O size is larger than that of a + * single page, or if this is a parallel raw data access, bypass page + * buffering. + */ + if(NULL == page_buf || size >= page_buf->page_size || bypass_pb) { + if(H5F__accum_write(fio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->bypasses[1]++; + else + page_buf->bypasses[0]++; + } /* end if */ + + /* If page buffering is disabled, or if this is a large metadata access, + * or if this is a parallel raw data access, we are done here + */ + if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) || + (bypass_pb && H5FD_MEM_DRAW == type)) + HGOTO_DONE(SUCCEED) + +#ifdef H5_HAVE_PARALLEL + if(bypass_pb) { + if(H5PB_update_entry(page_buf, addr, size, buf) > 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTUPDATE, FAIL, "failed to update PB with metadata cache") + HGOTO_DONE(SUCCEED) + } /* end if */ +#endif + } /* end if */ + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->accesses[1]++; + else + page_buf->accesses[0]++; + } /* end if */ + + /* Calculate the aligned address of the first page */ + first_page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* For raw data calculate the aligned address of the last page and + * the number of pages accessed if more than 1 page is accessed + */ + if(H5FD_MEM_DRAW == type) { + last_page_addr = (addr + size - 1) / page_buf->page_size * page_buf->page_size; + + /* how many pages does this write span */ + num_touched_pages = (last_page_addr/page_buf->page_size + 1) - + (first_page_addr / page_buf->page_size); + if(first_page_addr == last_page_addr) { + HDassert(1 == num_touched_pages); + last_page_addr = HADDR_UNDEF; + } /* end if */ + } /* end if */ + /* Otherwise set last page addr to HADDR_UNDEF */ + else { + num_touched_pages = 1; + last_page_addr = HADDR_UNDEF; + } /* end else */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + /* Check if existing pages for raw data need to be updated since raw data access is not atomic */ + if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) { + /* For each touched page, check if it exists in the page buffer, and + * update it with the data in the buffer to keep it up to date + */ + for(i = 0; i < num_touched_pages; i++) { + search_addr = i * page_buf->page_size + first_page_addr; + + /* Special handling for the first page if it is not a full page update */ + if(i == 0 && first_page_addr != addr) { + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + offset = addr - first_page_addr; + HDassert(page_buf->page_size > offset); + + /* Update page's data */ + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, page_buf->page_size - (size_t)offset); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + } /* end if */ + /* Special handling for the last page if it is not a full page update */ + else if(num_touched_pages > 1 && i == (num_touched_pages - 1) && + (search_addr + page_buf->page_size) != (addr + size)) { + HDassert(search_addr+page_buf->page_size > addr+size); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + offset = (num_touched_pages - 2) * page_buf->page_size + + (page_buf->page_size - (addr - first_page_addr)); + + /* Update page's data */ + HDmemcpy(page_entry->page_buf_ptr, (const uint8_t *)buf + offset, + (size_t)((addr + size) - last_page_addr)); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + } /* end else-if */ + /* Discard all fully written pages from the page buffer */ + else { + page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + /* Remove from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + + /* Decrement page count of appropriate type */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count--; + else + page_buf->meta_count--; + + /* Free page info */ + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + } /* end if */ + } /* end else */ + } /* end for */ + } /* end if */ + else { + /* An access could span 1 or 2 PBs at this point so we need to handle that */ + HDassert(1 == num_touched_pages || 2 == num_touched_pages); + for(i = 0; i < num_touched_pages; i++) { + haddr_t buf_offset; + + /* Calculate the aligned address of the page to search for it in the skip list */ + search_addr = (0 == i ? first_page_addr : last_page_addr); + + /* Calculate the access size if the access spans more than 1 page */ + if(1 == num_touched_pages) + access_size = size; + else + access_size = (0 == i ? (size_t)(first_page_addr + page_buf->page_size - addr) : (size - access_size)); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + + /* If found */ + if(page_entry) { + offset = (0 == i ? addr - page_entry->addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* Copy the requested data from the input buffer into the page */ + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, (const uint8_t *)buf + buf_offset, access_size); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* If not found */ + else { + void *new_page_buf; + size_t page_size = page_buf->page_size; + + /* Make space for new entry */ + if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) { + htri_t can_make_space; + + /* Check if we can make space in page buffer */ + if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") + + /* If make_space returns 0, then we can't use the page + * buffer for this I/O and we need to bypass + */ + if(0 == can_make_space) { + HDassert(0 == i); + + /* Write to VFD and return */ + if(H5FD_write(&fdio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "driver write request failed") + + /* Break out of loop */ + break; + } /* end if */ + } /* end if */ + + /* Don't bother searching if there is no write access */ + if(H5F_ACC_RDWR & H5F_INTENT(fio_info->f)) + /* Lookup & remove the page from the new skip list page if + * it exists to see if this is a new page from the MF layer + */ + page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->mf_slist_ptr, (void *)(&search_addr)); + + /* Calculate offset into the buffer of the page and the user buffer */ + offset = (0 == i ? addr - search_addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* If found, then just update the buffer pointer to the newly allocate buffer */ + if(page_entry) { + /* Allocate space for the page buffer */ + if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + HDmemset(new_page_buf, 0, (size_t)offset); + HDmemset((uint8_t *)new_page_buf + offset + access_size, 0, page_size - ((size_t)offset + access_size)); + + page_entry->page_buf_ptr = new_page_buf; + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* Otherwise read page through the VFD layer, but make sure we don't read past the EOA. */ + else { + haddr_t eoa, eof = HADDR_UNDEF; + + /* Allocate space for the page buffer */ + if(NULL == (new_page_buf = H5FL_FAC_CALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + + /* Create the new loaded PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed") + + page_entry->page_buf_ptr = new_page_buf; + page_entry->addr = search_addr; + page_entry->type = (H5F_mem_page_t)type; + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the entire page falls outside the EOA, then fail */ + if(search_addr > eoa) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "writing to a page that is outside the file EOA") + + /* Retrieve the 'eof' for the file - The MPI-VFD EOF + * returned will most likely be HADDR_UNDEF, so skip + * that check. + */ + if(!H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) + if(HADDR_UNDEF == (eof = H5FD_get_eof(fio_info->f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eof request failed") + + /* Adjust the read size to not go beyond the EOA */ + if(search_addr + page_size > eoa) + page_size = (size_t)(eoa - search_addr); + + if(search_addr < eof) { + if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->misses[1]++; + else + page_buf->misses[0]++; + } /* end if */ + } /* end else */ + + /* Copy the requested data from the page into the input buffer */ + HDmemcpy((uint8_t *)new_page_buf + offset, (const uint8_t *)buf+buf_offset, access_size); + + /* Page is dirty now */ + page_entry->is_dirty = TRUE; + + /* Insert page into PB, evicting other pages as necessary */ + if(H5PB__insert_entry(page_buf, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer") + } /* end else */ + } /* end for */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__insert_entry() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Insert the supplied page into the page buffer, both the + * skip list and the LRU. + * + * As best I can tell, this function imposes no limit on the + * number of entries in the page buffer beyond an assertion + * failure it the page count exceeds the limit. + * + * JRM -- 12/22/16 + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Insert entry in skip list */ + if(H5SL_insert(page_buf->slist_ptr, page_entry, &(page_entry->addr)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINSERT, FAIL, "can't insert entry in skip list") + HDassert(H5SL_count(page_buf->slist_ptr) * page_buf->page_size <= page_buf->max_size); + + /* Increment appropriate page count */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count++; + else + page_buf->meta_count++; + + /* Insert entry in LRU */ + H5PB__INSERT_LRU(page_buf, page_entry) + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__insert_entry() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__make_space() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * If necessary and if possible, evict a page from the page + * buffer to make space for the supplied page. Depending on + * the page buffer configuration and contents, and the page + * supplied this may or may not be possible. + * + * JRM -- 12/22/16 + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static htri_t +H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf, + H5FD_mem_t inserted_type) +{ + H5PB_entry_t *page_entry; /* Pointer to page eviction candidate */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(fio_info); + HDassert(page_buf); + + /* Get oldest entry */ + page_entry = page_buf->LRU_tail_ptr; + + if(H5FD_MEM_DRAW == inserted_type) { + /* If threshould is 100% metadata and page buffer is full of + metadata, then we can't make space for raw data */ + if(0 == page_buf->raw_count && page_buf->min_meta_count == page_buf->meta_count) { + HDassert(page_buf->meta_count * page_buf->page_size == page_buf->max_size); + HGOTO_DONE(FALSE) + } /* end if */ + + /* check the metadata threshold before evicting metadata items */ + while(1) { + if(page_entry->prev && H5F_MEM_PAGE_META == page_entry->type && + page_buf->min_meta_count >= page_buf->meta_count) + page_entry = page_entry->prev; + else + break; + } /* end while */ + } /* end if */ + else { + /* If threshould is 100% raw data and page buffer is full of + raw data, then we can't make space for meta data */ + if(0 == page_buf->meta_count && page_buf->min_raw_count == page_buf->raw_count) { + HDassert(page_buf->raw_count * page_buf->page_size == page_buf->max_size); + HGOTO_DONE(FALSE) + } /* end if */ + + /* check the raw data threshold before evicting raw data items */ + while(1) { + if(page_entry->prev && (H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) && + page_buf->min_raw_count >= page_buf->raw_count) + page_entry = page_entry->prev; + else + break; + } /* end while */ + } /* end else */ + + /* Remove from page index */ + if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Tail Page Entry is not in skip list") + + /* Remove entry from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len); + + /* Decrement appropriate page type counter */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count--; + else + page_buf->meta_count--; + + /* Flush page if dirty */ + if(page_entry->is_dirty) + if(H5PB__write_entry(fio_info, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + + /* Update statistics */ + if(page_entry->type == H5F_MEM_PAGE_DRAW || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->evictions[1]++; + else + page_buf->evictions[0]++; + + /* Release page */ + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__make_space() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__write_entry() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry) +{ + haddr_t eoa; /* Current EOA for the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(fio_info); + HDassert(fio_info->f); + HDassert(page_entry); + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, page_entry->type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the starting address of the page is larger than + * the EOA, then the entire page is discarded without writing. + */ + if(page_entry->addr <= eoa) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ + size_t page_size = fio_info->f->shared->page_buf->page_size; + + /* Adjust the page length if it exceeds the EOA */ + if((page_entry->addr + page_size) > eoa) + page_size = (size_t)(eoa - page_entry->addr); + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + if(H5FD_write(&fdio_info, page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + } /* end if */ + + page_entry->is_dirty = FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__write_entry() */ + diff --git a/src/H5PBmodule.h b/src/H5PBmodule.h new file mode 100644 index 0000000..35da3f6 --- /dev/null +++ b/src/H5PBmodule.h @@ -0,0 +1,34 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Saturday, September 12, 2015 + * + * Purpose: This file contains declarations which define macros for the + * H5PB package. Including this header means that the source file + * is part of the H5PB package. + */ +#ifndef _H5PBmodule_H +#define _H5PBmodule_H + +/* Define the proper control macros for the generic FUNC_ENTER/LEAVE and error + * reporting macros. + */ +#define H5PB_MODULE +#define H5_MY_PKG H5PB +#define H5_MY_PKG_ERR H5E_RESOURCE +#define H5_MY_PKG_INIT NO + +#endif /* _H5PBmodule_H */ diff --git a/src/H5PBpkg.h b/src/H5PBpkg.h new file mode 100644 index 0000000..976f18d --- /dev/null +++ b/src/H5PBpkg.h @@ -0,0 +1,60 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if !(defined H5PB_FRIEND || defined H5PB_MODULE) +#error "Do not include this file outside the H5PB package!" +#endif + +#ifndef _H5PBpkg_H +#define _H5PBpkg_H + +/* Get package's private header */ +#include "H5PBprivate.h" + +/* Other private headers needed by this file */ + +/**************************/ +/* Package Private Macros */ +/**************************/ + + +/****************************/ +/* Package Private Typedefs */ +/****************************/ + +typedef struct H5PB_entry_t { + void *page_buf_ptr; /* Pointer to the buffer containing the data */ + haddr_t addr; /* Address of the page in the file */ + H5F_mem_page_t type; /* Type of the page entry (H5F_MEM_PAGE_RAW/META) */ + hbool_t is_dirty; /* Flag indicating whether the page has dirty data or not */ + + /* Fields supporting replacement policies */ + struct H5PB_entry_t *next; /* next pointer in the LRU list */ + struct H5PB_entry_t *prev; /* previous pointer in the LRU list */ +} H5PB_entry_t; + + +/*****************************/ +/* Package Private Variables */ +/*****************************/ + + +/******************************/ +/* Package Private Prototypes */ +/******************************/ + + +#endif /* _H5PBpkg_H */ + diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h new file mode 100644 index 0000000..3b5dcae --- /dev/null +++ b/src/H5PBprivate.h @@ -0,0 +1,108 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5PBprivate.h + * June 2014 + * Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ + +#ifndef _H5PBprivate_H +#define _H5PBprivate_H + +/* Include package's public header */ +#ifdef NOT_YET +#include "H5PBpublic.h" +#endif /* NOT_YET */ + +/* Private headers needed by this header */ +#include "H5private.h" /* Generic Functions */ +#include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5SLprivate.h" /* Skip List */ + + +/**************************/ +/* Library Private Macros */ +/**************************/ + + +/****************************/ +/* Library Private Typedefs */ +/****************************/ + +/* Forward declaration for a page buffer entry */ +struct H5PB_entry_t; + +/* Typedef for the main structure for the page buffer */ +typedef struct H5PB_t { + size_t max_size; /* The total page buffer size */ + size_t page_size; /* Size of a single page */ + unsigned min_meta_perc; /* Minimum ratio of metadata entries required before evicting meta entries */ + unsigned min_raw_perc; /* Minimum ratio of raw data entries required before evicting raw entries */ + unsigned meta_count; /* Number of entries for metadata */ + unsigned raw_count; /* Number of entries for raw data */ + unsigned min_meta_count; /* Minimum # of entries for metadata */ + unsigned min_raw_count; /* Minimum # of entries for raw data */ + + H5SL_t *slist_ptr; /* Skip list with all the active page entries */ + H5SL_t *mf_slist_ptr; /* Skip list containing newly allocated page entries inserted from the MF layer */ + + size_t LRU_list_len; /* Number of entries in the LRU (identical to slist_ptr count) */ + struct H5PB_entry_t *LRU_head_ptr; /* Head pointer of the LRU */ + struct H5PB_entry_t *LRU_tail_ptr; /* Tail pointer of the LRU */ + + H5FL_fac_head_t *page_fac; /* Factory for allocating pages */ + + /* Statistics */ + unsigned accesses[2]; + unsigned hits[2]; + unsigned misses[2]; + unsigned evictions[2]; + unsigned bypasses[2]; +} H5PB_t; + +/*****************************/ +/* Library-private Variables */ +/*****************************/ + + +/***************************************/ +/* Library-private Function Prototypes */ +/***************************************/ + +/* General routines */ +H5_DLL herr_t H5PB_create(H5F_t *file, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc); +H5_DLL herr_t H5PB_flush(const H5F_io_info2_t *fio_info); +H5_DLL herr_t H5PB_dest(H5F_t *file); +H5_DLL herr_t H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr); +H5_DLL herr_t H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf); +H5_DLL herr_t H5PB_remove_entry(const H5F_t *f, haddr_t addr); +H5_DLL herr_t H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, + haddr_t addr, size_t size, void *buf/*out*/); +H5_DLL herr_t H5PB_write(const H5F_io_info2_t *f, H5FD_mem_t type, haddr_t addr, + size_t size, const void *buf); + +/* Statistics routines */ +H5_DLL herr_t H5PB_reset_stats(H5PB_t *page_buf); +H5_DLL herr_t H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2], + unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); +H5_DLL herr_t H5PB_print_stats(const H5PB_t *page_buf); + +#endif /* !_H5PBprivate_H */ + diff --git a/src/H5Pdeprec.c b/src/H5Pdeprec.c index ecf2bea..d225dfa 100644 --- a/src/H5Pdeprec.c +++ b/src/H5Pdeprec.c @@ -488,5 +488,135 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_version() */ +/*------------------------------------------------------------------------- + * Function: H5Pset_file_space + * + * Purpose: It is mapped to H5Pset_file_space_strategy(). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Jan 2017 + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold) +{ + + H5F_fspace_strategy_t new_strategy; /* File space strategy type */ + hbool_t new_persist = H5F_FREE_SPACE_PERSIST_DEF; /* Persisting free-space or not */ + hsize_t new_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; /* Free-space section threshold */ + H5F_file_space_type_t in_strategy = strategy; /* Input strategy */ + hsize_t in_threshold = threshold; /* Input threshold */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "iFfh", plist_id, strategy, threshold); + + if((unsigned)in_strategy >= H5F_FILE_SPACE_NTYPES) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy") + /* + * For 1.10.0 H5Pset_file_space: + * If strategy is zero, the property is not changed; + * the existing strategy is retained. + * If threshold is zero, the property is not changed; + * the existing threshold is retained. + */ + if(!in_strategy) + H5Pget_file_space(plist_id, &in_strategy, NULL); + if(!in_threshold) + H5Pget_file_space(plist_id, NULL, &in_threshold); + + switch(in_strategy) { + case H5F_FILE_SPACE_ALL_PERSIST: + new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + new_persist = TRUE; + new_threshold = in_threshold; + break; + + case H5F_FILE_SPACE_ALL: + new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + new_threshold = in_threshold; + break; + + case H5F_FILE_SPACE_AGGR_VFD: + new_strategy = H5F_FSPACE_STRATEGY_AGGR; + break; + + case H5F_FILE_SPACE_VFD: + new_strategy = H5F_FSPACE_STRATEGY_NONE; + break; + + case H5F_FILE_SPACE_NTYPES: + case H5F_FILE_SPACE_DEFAULT: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy") + } + + if(H5Pset_file_space_strategy(plist_id, new_strategy, new_persist, new_threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_file_space() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_space + * + * Purpose: It is mapped to H5Pget_file_space_strategy(). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Jan 2017 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold) +{ + H5F_fspace_strategy_t new_strategy; /* File space strategy type */ + hbool_t new_persist; /* Persisting free-space or not */ + hsize_t new_threshold; /* Free-space section threshold */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold); + + /* Get current file space info */ + if(H5Pget_file_space_strategy(plist_id, &new_strategy, &new_persist, &new_threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy") + + /* Get value(s) */ + if(strategy) { + switch(new_strategy) { + + case H5F_FSPACE_STRATEGY_FSM_AGGR: + if(new_persist) + *strategy = H5F_FILE_SPACE_ALL_PERSIST; + else + *strategy = H5F_FILE_SPACE_ALL; + break; + + case H5F_FSPACE_STRATEGY_AGGR: + *strategy = H5F_FILE_SPACE_AGGR_VFD; + break; + + case H5F_FSPACE_STRATEGY_NONE: + *strategy = H5F_FILE_SPACE_VFD; + break; + + case H5F_FSPACE_STRATEGY_PAGE: + case H5F_FSPACE_STRATEGY_NTYPES: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy") + } + } + + if(threshold) + *threshold = new_threshold; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_file_space() */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 7ea0fe0..7865fdf 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -80,18 +80,18 @@ #define H5F_ACS_PREEMPT_READ_CHUNKS_DEC H5P__decode_double /* Definition for threshold for alignment */ #define H5F_ACS_ALIGN_THRHD_SIZE sizeof(hsize_t) -#define H5F_ACS_ALIGN_THRHD_DEF 1 +#define H5F_ACS_ALIGN_THRHD_DEF H5F_ALIGN_THRHD_DEF #define H5F_ACS_ALIGN_THRHD_ENC H5P__encode_hsize_t #define H5F_ACS_ALIGN_THRHD_DEC H5P__decode_hsize_t /* Definition for alignment */ #define H5F_ACS_ALIGN_SIZE sizeof(hsize_t) -#define H5F_ACS_ALIGN_DEF 1 +#define H5F_ACS_ALIGN_DEF H5F_ALIGN_DEF #define H5F_ACS_ALIGN_ENC H5P__encode_hsize_t #define H5F_ACS_ALIGN_DEC H5P__decode_hsize_t /* Definition for minimum metadata allocation block size (when aggregating metadata allocations. */ #define H5F_ACS_META_BLOCK_SIZE_SIZE sizeof(hsize_t) -#define H5F_ACS_META_BLOCK_SIZE_DEF 2048 +#define H5F_ACS_META_BLOCK_SIZE_DEF H5F_META_BLOCK_SIZE_DEF #define H5F_ACS_META_BLOCK_SIZE_ENC H5P__encode_hsize_t #define H5F_ACS_META_BLOCK_SIZE_DEC H5P__decode_hsize_t /* Definition for maximum sieve buffer size (when data sieving @@ -103,7 +103,7 @@ /* Definition for minimum "small data" allocation block size (when aggregating "small" raw data allocations. */ #define H5F_ACS_SDATA_BLOCK_SIZE_SIZE sizeof(hsize_t) -#define H5F_ACS_SDATA_BLOCK_SIZE_DEF 2048 +#define H5F_ACS_SDATA_BLOCK_SIZE_DEF H5F_SDATA_BLOCK_SIZE_DEF #define H5F_ACS_SDATA_BLOCK_SIZE_ENC H5P__encode_hsize_t #define H5F_ACS_SDATA_BLOCK_SIZE_DEC H5P__decode_hsize_t /* Definition for garbage-collect references */ @@ -231,6 +231,21 @@ #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_ENC H5P__facc_cache_image_config_enc #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEC H5P__facc_cache_image_config_dec #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP H5P__facc_cache_image_config_cmp +/* Definition for total size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_SIZE_SIZE sizeof(size_t) +#define H5F_ACS_PAGE_BUFFER_SIZE_DEF 0 +#define H5F_ACS_PAGE_BUFFER_SIZE_ENC H5P__encode_size_t +#define H5F_ACS_PAGE_BUFFER_SIZE_DEC H5P__decode_size_t +/* Definition for minimum metadata size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE sizeof(unsigned) +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF 0 +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC H5P__encode_unsigned +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC H5P__decode_unsigned +/* Definition for minimum raw data size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE sizeof(unsigned) +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF 0 +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC H5P__encode_unsigned +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC H5P__decode_unsigned /******************/ @@ -359,6 +374,9 @@ static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ #endif /* H5_HAVE_PARALLEL */ static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEF; /* Default metadata cache image settings */ +static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */ +static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ +static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer minumum raw data size */ /*------------------------------------------------------------------------- @@ -574,6 +592,22 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_SIZE_NAME, H5F_ACS_PAGE_BUFFER_SIZE_SIZE, &H5F_def_page_buf_size_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_SIZE_ENC, H5F_ACS_PAGE_BUFFER_SIZE_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum metadata size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE, &H5F_def_page_buf_min_meta_perc_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum raw data size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE, &H5F_def_page_buf_min_raw_perc_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4681,3 +4715,93 @@ done: } /* end H5Pget_coll_metadata_write() */ #endif /* H5_HAVE_PARALLEL */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_page_buffer_size + * + * Purpose: Set the maximum page buffering size. This has to be a + * multiple of the page allocation size which must be enabled; + * otherwise file create/open will fail. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * June 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_perc, unsigned min_raw_perc) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "izIuIu", plist_id, buf_size, min_meta_perc, min_raw_perc); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(min_meta_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum metadata fractions must be between 0 and 100 inclusive") + if(min_raw_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum rawdata fractions must be between 0 and 100 inclusive") + + if(min_meta_perc + min_raw_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Sum of minimum metadata and raw data fractions can't be bigger than 100"); + + /* Set size */ + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &buf_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set page buffer size") + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &min_meta_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min metadata entries") + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &min_raw_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min rawdata entries") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_page_buffer_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_page_buffer_size + * + * Purpose: Retrieves the maximum page buffer size. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * June 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_perc, unsigned *min_raw_perc) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "i*z*Iu*Iu", plist_id, buf_size, min_meta_perc, min_raw_perc); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get size */ + + if(buf_size) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, buf_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer size") + if(min_meta_perc) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, min_meta_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum metadata percent") + if(min_raw_perc) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, min_raw_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum raw data percent") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_page_buffer_size() */ + diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c index d451982..c778801 100644 --- a/src/H5Pfcpl.c +++ b/src/H5Pfcpl.c @@ -97,14 +97,23 @@ #define H5F_CRT_SHMSG_BTREE_MIN_ENC H5P__encode_unsigned #define H5F_CRT_SHMSG_BTREE_MIN_DEC H5P__decode_unsigned /* Definitions for file space handling strategy */ -#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(unsigned) -#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF -#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__encode_unsigned -#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__decode_unsigned -#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t) -#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF -#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t -#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t +#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(H5F_fspace_strategy_t) +#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF +#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__fcrt_fspace_strategy_enc +#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__fcrt_fspace_strategy_dec +#define H5F_CRT_FREE_SPACE_PERSIST_SIZE sizeof(hbool_t) +#define H5F_CRT_FREE_SPACE_PERSIST_DEF H5F_FREE_SPACE_PERSIST_DEF +#define H5F_CRT_FREE_SPACE_PERSIST_ENC H5P__encode_hbool_t +#define H5F_CRT_FREE_SPACE_PERSIST_DEC H5P__decode_hbool_t +#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t) +#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF +#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t +#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t +/* Definitions for file space page size in support of level-2 page caching */ +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE sizeof(hsize_t) +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF H5F_FILE_SPACE_PAGE_SIZE_DEF +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC H5P__encode_hsize_t +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC H5P__decode_hsize_t /******************/ @@ -131,6 +140,8 @@ static herr_t H5P__fcrt_shmsg_index_types_enc(const void *value, void **_pp, siz static herr_t H5P__fcrt_shmsg_index_types_dec(const void **_pp, void *value); static herr_t H5P__fcrt_shmsg_index_minsize_enc(const void *value, void **_pp, size_t *size); static herr_t H5P__fcrt_shmsg_index_minsize_dec(const void **_pp, void *value); +static herr_t H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size); +static herr_t H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value); /*********************/ @@ -178,8 +189,10 @@ static const unsigned H5F_def_sohm_index_flags_g[H5O_SHMESG_MAX_NINDEXES] = H static const unsigned H5F_def_sohm_index_minsizes_g[H5O_SHMESG_MAX_NINDEXES] = H5F_CRT_SHMSG_INDEX_MINSIZE_DEF; static const unsigned H5F_def_sohm_list_max_g = H5F_CRT_SHMSG_LIST_MAX_DEF; static const unsigned H5F_def_sohm_btree_min_g = H5F_CRT_SHMSG_BTREE_MIN_DEF; -static const unsigned H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF; +static const H5F_fspace_strategy_t H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF; +static const hbool_t H5F_def_free_space_persist_g = H5F_CRT_FREE_SPACE_PERSIST_DEF; static const hsize_t H5F_def_free_space_threshold_g = H5F_CRT_FREE_SPACE_THRESHOLD_DEF; +static const hsize_t H5F_def_file_space_page_size_g = H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF; @@ -267,12 +280,24 @@ H5P_fcrt_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the free-space persist flag */ + if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_PERSIST_NAME, H5F_CRT_FREE_SPACE_PERSIST_SIZE, &H5F_def_free_space_persist_g, + NULL, NULL, NULL, H5F_CRT_FREE_SPACE_PERSIST_ENC, H5F_CRT_FREE_SPACE_PERSIST_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the free space section threshold */ if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, H5F_CRT_FREE_SPACE_THRESHOLD_SIZE, &H5F_def_free_space_threshold_g, NULL, NULL, NULL, H5F_CRT_FREE_SPACE_THRESHOLD_ENC, H5F_CRT_FREE_SPACE_THRESHOLD_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the file space page size */ + if(H5P_register_real(pclass, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE, &H5F_def_file_space_page_size_g, + NULL, NULL, NULL, H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC, H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_fcrt_reg_prop() */ @@ -1246,15 +1271,13 @@ H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *mi if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); - /* Get value */ - if (max_list) { + /* Get value(s) */ + if(max_list) if(H5P_get(plist, H5F_CRT_SHMSG_LIST_MAX_NAME, max_list) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get list maximum"); - } - if (min_btree) { + if(min_btree) if(H5P_get(plist, H5F_CRT_SHMSG_BTREE_MIN_NAME, min_btree) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get SOHM information"); - } done: FUNC_LEAVE_API(ret_value) @@ -1262,15 +1285,12 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pset_file_space + * Function: H5Pset_file_space_strategy * - * Purpose: Sets the strategy that the library employs in managing file space. - * If strategy is zero, the property is not changed; the existing - * strategy is retained. - * Sets the threshold value that the file's free space - * manager(s) will use to track free space sections. - * If threshold is zero, the property is not changed; the existing - * threshold is retained. + * Purpose: Sets the "strategy" that the library employs in managing file space + * Sets the "persist" value as to persist free-space or not + * Sets the "threshold" value that the free space manager(s) will use to track free space sections. + * Ignore "persist" and "threshold" for strategies that do not use free-space managers * * Return: Non-negative on success/Negative on failure * @@ -1279,15 +1299,16 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold) +H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("e", "iFfh", plist_id, strategy, threshold); + H5TRACE4("e", "iFfbh", plist_id, strategy, persist, threshold); - if((unsigned)strategy >= H5F_FILE_SPACE_NTYPES) + /* Check arguments */ + if(strategy >= H5F_FSPACE_STRATEGY_NTYPES) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy") /* Get the plist structure */ @@ -1295,24 +1316,28 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") /* Set value(s), if non-zero */ - if(strategy) - if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space strategy") - if(threshold) - if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set free-space threshold") + if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy") + + /* Ignore persist and threshold settings for strategies that do not use FSM */ + if(strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || strategy == H5F_FSPACE_STRATEGY_PAGE) { + if(H5P_set(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space persisting status") + + if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space threshold") + } /* end if */ done: FUNC_LEAVE_API(ret_value) -} /* H5Pset_file_space() */ +} /* H5Pset_file_space_strategy() */ /*------------------------------------------------------------------------- - * Function: H5Pget_file_space + * Function: H5Pget_file_space_strategy * - * Purpose: Retrieves the strategy that the library uses in managing file space. - * Retrieves the threshold value that the file's free space - * managers use to track free space sections. + * Purpose: Retrieves the strategy, persist, and threshold that the library + * uses in managing file space. * * Return: Non-negative on success/Negative on failure * @@ -1321,13 +1346,13 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold) +H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold); + H5TRACE4("e", "i*Ff*b*h", plist_id, strategy, persist, threshold); /* Get the plist structure */ if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) @@ -1337,11 +1362,158 @@ H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *thre if(strategy) if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, strategy) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy") + if(persist) + if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space persisting status") if(threshold) if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, threshold) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space threshold") done: FUNC_LEAVE_API(ret_value) -} /* H5Pget_file_space() */ +} /* H5Pget_file_space_strategy() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__fcrt_fspace_strategy_enc + * + * Purpose: Callback routine which is called whenever the free-space + * strategy property in the file creation property list + * is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Friday, December 27, 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size) +{ + const H5F_fspace_strategy_t *strategy = (const H5F_fspace_strategy_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(strategy); + HDassert(size); + + if(NULL != *pp) + /* Encode free-space strategy */ + *(*pp)++ = (uint8_t)*strategy; + + /* Size of free-space strategy */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__fcrt_fspace_strategy_enc() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__fcrt_fspace_strategy_dec + * + * Purpose: Callback routine which is called whenever the free-space + * strategy property in the file creation property list + * is decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Friday, December 27, 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value) +{ + H5F_fspace_strategy_t *strategy = (H5F_fspace_strategy_t *)_value; /* Free-space strategy */ + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(strategy); + + /* Decode free-space strategy */ + *strategy = (H5F_fspace_strategy_t)*(*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__fcrt_fspace_strategy_dec() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_file_space_page_size + * + * Purpose: Sets the file space page size for paged aggregation. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; August 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ih", plist_id, fsp_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512") + + /* Set the value*/ + if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_file_space_page_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_space_page_size + * + * Purpose: Retrieves the file space page size for aggregating small metadata + * or raw data in the parameter "fsp_size". + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; August 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*h", plist_id, fsp_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get value */ + if(fsp_size) + if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, fsp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space page size") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_file_space_page_size() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 5aa8301..50b9eb0 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -300,8 +300,10 @@ H5_DLL herr_t H5Pset_shared_mesg_index(hid_t plist_id, unsigned index_num, unsig H5_DLL herr_t H5Pget_shared_mesg_index(hid_t plist_id, unsigned index_num, unsigned *mesg_type_flags, unsigned *min_mesg_size); H5_DLL herr_t H5Pset_shared_mesg_phase_change(hid_t plist_id, unsigned max_list, unsigned min_btree); H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *min_btree); -H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); -H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); +H5_DLL herr_t H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold); +H5_DLL herr_t H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold); +H5_DLL herr_t H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size); +H5_DLL herr_t H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size); /* File access property list (FAPL) routines */ H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold, @@ -367,6 +369,8 @@ H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective) #endif /* H5_HAVE_PARALLEL */ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr); H5_DLL herr_t H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr /*out*/); +H5_DLL herr_t H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_per, unsigned min_raw_per); +H5_DLL herr_t H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_per, unsigned *min_raw_per); /* Dataset creation property list (DCPL) routines */ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); @@ -534,6 +538,8 @@ H5_DLL herr_t H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, H5_DLL herr_t H5Pget_version(hid_t plist_id, unsigned *boot/*out*/, unsigned *freelist/*out*/, unsigned *stab/*out*/, unsigned *shhdr/*out*/); +H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); +H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); #endif /* H5_NO_DEPRECATED_SYMBOLS */ #ifdef __cplusplus diff --git a/src/H5err.txt b/src/H5err.txt index 44c5a93..ffa9315 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -77,6 +77,7 @@ MAJOR, H5E_SOHM, Shared Object Header Messages MAJOR, H5E_EARRAY, Extensible Array MAJOR, H5E_FARRAY, Fixed Array MAJOR, H5E_PLUGIN, Plugin for dynamically loaded library +MAJOR, H5E_PAGEBUF, Page Buffering MAJOR, H5E_NONE_MAJOR, No error # Sections (for grouping minor errors) diff --git a/src/H5trace.c b/src/H5trace.c index 44b2ed5..48c357a 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -895,32 +895,28 @@ H5_trace(const double *returning, const char *func, const char *type, ...) fprintf(out, "NULL"); } /* end if */ else { - H5F_file_space_type_t fs_type = (H5F_file_space_type_t)va_arg(ap, int); + H5F_fspace_strategy_t fs_strategy = (H5F_fspace_strategy_t)va_arg(ap, int); - switch(fs_type) { - case H5F_FILE_SPACE_DEFAULT: - fprintf(out, "H5F_FILE_SPACE_DEFAULT"); + switch(fs_strategy) { + case H5F_FSPACE_STRATEGY_FSM_AGGR: + fprintf(out, "H5F_FSPACE_STRATEGY_FSM_AGGR"); break; - case H5F_FILE_SPACE_ALL_PERSIST: - fprintf(out, "H5F_FILE_SPACE_ALL_PERSIST"); + case H5F_FSPACE_STRATEGY_PAGE: + fprintf(out, "H5F_FSPACE_STRATEGY_PAGE"); break; - case H5F_FILE_SPACE_ALL: - fprintf(out, "H5F_FILE_SPACE_ALL"); + case H5F_FSPACE_STRATEGY_AGGR: + fprintf(out, "H5F_FSPACE_STRATEGY_AGGR"); break; - case H5F_FILE_SPACE_AGGR_VFD: - fprintf(out, "H5F_FILE_SPACE_AGGR_VFD"); + case H5F_FSPACE_STRATEGY_NONE: + fprintf(out, "H5F_FSPACE_STRATEGY_NONE"); break; - case H5F_FILE_SPACE_VFD: - fprintf(out, "H5F_FILE_SPACE_VFD"); - break; - - case H5F_FILE_SPACE_NTYPES: + case H5F_FSPACE_STRATEGY_NTYPES: default: - fprintf(out, "%ld", (long)fs_type); + fprintf(out, "%ld", (long)fs_strategy); break; } /* end switch */ } /* end else */ @@ -1003,6 +999,15 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end else */ break; + case 't': + if(ptr) { + if(vp) + fprintf(out, "0x%lx", (unsigned long)vp); + else + fprintf(out, "NULL"); + } /* end if */ + break; + case 'v': if(ptr) { if(vp) diff --git a/src/Makefile.am b/src/Makefile.am index 69b54b4..59df4c2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -60,7 +60,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5F.c H5Fint.c H5Faccum.c H5Fcwfs.c \ H5Fdbg.c H5Fdeprec.c H5Fefc.c H5Ffake.c H5Fio.c \ H5Fmount.c H5Fquery.c \ - H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ + H5Fsfile.c H5Fspace.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c \ @@ -98,6 +98,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Pfapl.c H5Pfcpl.c H5Pfmpl.c \ H5Pgcpl.c H5Pint.c \ H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \ + H5PB.c \ H5PL.c \ H5R.c H5Rdeprec.c \ H5UC.c \ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9657971..d7965cb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -246,6 +246,7 @@ set (H5_TESTS cross_read freespace mf + page_buffer vds file_image unregister diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index fc8587b..f372acf 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -170,6 +170,7 @@ endforeach () #-- Copy all the HDF5 files from the test directory into the source directory # -------------------------------------------------------------------- set (HDF5_REFERENCE_TEST_FILES + aggr.h5 bad_compound.h5 be_data.h5 be_extlink1.h5 @@ -187,7 +188,15 @@ set (HDF5_REFERENCE_TEST_FILES filespace_1_8.h5 fill_old.h5 filter_error.h5 + fsm_aggr_nopersist.h5 + fsm_aggr_persist.h5 group_old.h5 + h5fc_ext1_i.h5 + h5fc_ext1_f.h5 + h5fc_ext2_if.h5 + h5fc_ext2_sf.h5 + h5fc_ext3_isf.h5 + h5fc_ext_none.h5 le_data.h5 le_extlink1.h5 le_extlink2.h5 @@ -195,6 +204,9 @@ set (HDF5_REFERENCE_TEST_FILES multi_file_v16-r.h5 multi_file_v16-s.h5 noencoder.h5 + none.h5 + paged_nopersist.h5 + paged_persist.h5 specmetaread.h5 tarrold.h5 tbad_msg_count.h5 @@ -618,6 +630,7 @@ set (H5TEST_TESTS cross_read freespace mf + page_buffer vds file_image unregister @@ -1288,7 +1301,6 @@ if (HDF5_BUILD_GENERATORS) gen_cross gen_deflate gen_filters - gen_idx gen_new_array gen_new_fill gen_new_group diff --git a/test/Makefile.am b/test/Makefile.am index 98cfc42..a64cd0e 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -54,7 +54,7 @@ check_SCRIPTS = $(TEST_SCRIPT) TEST_PROG= testhdf5 \ cache cache_api cache_image cache_tagging lheap ohdr stab gheap \ evict_on_close farray earray btree2 fheap \ - pool accum hyperslab istore bittests dt_arith \ + pool accum hyperslab istore bittests dt_arith page_buffer \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ twriteorder big mtime fillval mount flush1 flush2 app_ref enum \ set_extent ttsafe enc_dec_plist enc_dec_plist_cross_platform\ @@ -91,7 +91,7 @@ endif # --enable-build-all at configure time. # The gen_old_* files can only be compiled with older versions of the library # so do not appear in this list. -BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_idx gen_new_array \ +BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \ gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \ gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread \ gen_sizes_lheap gen_file_image gen_plist diff --git a/tools/test/h5diff/testfiles/h5diff_dset_idx2.h5 b/test/aggr.h5 Binary files differindex db7584d..b24365d 100644 --- a/tools/test/h5diff/testfiles/h5diff_dset_idx2.h5 +++ b/test/aggr.h5 diff --git a/test/cache.c b/test/cache.c index c381776..ba6fb0e 100644 --- a/test/cache.c +++ b/test/cache.c @@ -97,19 +97,19 @@ struct move_entry_test_spec /* private function declarations: */ -static unsigned smoke_check_1(int express_test); -static unsigned smoke_check_2(int express_test); -static unsigned smoke_check_3(int express_test); -static unsigned smoke_check_4(int express_test); -static unsigned smoke_check_5(int express_test); -static unsigned smoke_check_6(int express_test); -static unsigned smoke_check_7(int express_test); -static unsigned smoke_check_8(int express_test); -static unsigned smoke_check_9(int express_test); -static unsigned smoke_check_10(int express_test); -static unsigned write_permitted_check(int express_test); -static unsigned check_insert_entry(void); -static unsigned check_flush_cache(void); +static unsigned smoke_check_1(int express_test, unsigned paged); +static unsigned smoke_check_2(int express_test, unsigned paged); +static unsigned smoke_check_3(int express_test, unsigned paged); +static unsigned smoke_check_4(int express_test, unsigned paged); +static unsigned smoke_check_5(int express_test, unsigned paged); +static unsigned smoke_check_6(int express_test, unsigned paged); +static unsigned smoke_check_7(int express_test, unsigned paged); +static unsigned smoke_check_8(int express_test, unsigned paged); +static unsigned smoke_check_9(int express_test, unsigned paged); +static unsigned smoke_check_10(int express_test, unsigned paged); +static unsigned write_permitted_check(int express_test, unsigned paged); +static unsigned check_insert_entry(unsigned paged); +static unsigned check_flush_cache(unsigned paged); static void check_flush_cache__empty_cache(H5F_t * file_ptr); static void check_flush_cache__multi_entry(H5F_t * file_ptr); static void check_flush_cache__multi_entry_test(H5F_t * file_ptr, @@ -159,48 +159,49 @@ static void check_flush_cache__flush_op_test(H5F_t * file_ptr, int check_size, struct fo_flush_entry_check check[]); static void check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr); -static unsigned check_get_entry_status(void); -static unsigned check_expunge_entry(void); -static unsigned check_multiple_read_protect(void); -static unsigned check_move_entry(void); +static unsigned check_get_entry_status(unsigned paged); +static unsigned check_expunge_entry(unsigned paged); +static unsigned check_multiple_read_protect(unsigned paged); +static unsigned check_move_entry(unsigned paged); static void check_move_entry__run_test(H5F_t * file_ptr, unsigned test_num, struct move_entry_test_spec * spec_ptr); -static unsigned check_pin_protected_entry(void); -static unsigned check_resize_entry(void); -static unsigned check_evictions_enabled(void); -static unsigned check_flush_protected_err(void); -static unsigned check_destroy_pinned_err(void); -static unsigned check_destroy_protected_err(void); -static unsigned check_duplicate_insert_err(void); -static unsigned check_double_pin_err(void); -static unsigned check_double_unpin_err(void); -static unsigned check_pin_entry_errs(void); -static unsigned check_double_protect_err(void); -static unsigned check_double_unprotect_err(void); -static unsigned check_mark_entry_dirty_errs(void); -static unsigned check_expunge_entry_errs(void); -static unsigned check_move_entry_errs(void); -static unsigned check_resize_entry_errs(void); -static unsigned check_unprotect_ro_dirty_err(void); -static unsigned check_protect_ro_rw_err(void); -static unsigned check_check_evictions_enabled_err(void); -static unsigned check_auto_cache_resize(hbool_t cork_ageout); -static unsigned check_auto_cache_resize_disable(void); -static unsigned check_auto_cache_resize_epoch_markers(void); -static unsigned check_auto_cache_resize_input_errs(void); -static unsigned check_auto_cache_resize_aux_fcns(void); -static unsigned check_metadata_blizzard_absence(hbool_t fill_via_insertion); -static unsigned check_flush_deps(void); -static unsigned check_flush_deps_err(void); -static unsigned check_flush_deps_order(void); -static unsigned check_notify_cb(void); -static unsigned check_metadata_cork(hbool_t fill_via_insertion); -static unsigned check_entry_deletions_during_scans(void); +static unsigned check_pin_protected_entry(unsigned paged); +static unsigned check_resize_entry(unsigned paged); +static unsigned check_evictions_enabled(unsigned paged); +static unsigned check_flush_protected_err(unsigned paged); +static unsigned check_destroy_pinned_err(unsigned paged); +static unsigned check_destroy_protected_err(unsigned paged); +static unsigned check_duplicate_insert_err(unsigned paged); +static unsigned check_double_pin_err(unsigned paged); +static unsigned check_double_unpin_err(unsigned paged); +static unsigned check_pin_entry_errs(unsigned paged); +static unsigned check_double_protect_err(unsigned paged); +static unsigned check_double_unprotect_err(unsigned paged); +static unsigned check_mark_entry_dirty_errs(unsigned paged); +static unsigned check_expunge_entry_errs(unsigned paged); +static unsigned check_move_entry_errs(unsigned paged); +static unsigned check_resize_entry_errs(unsigned paged); +static unsigned check_unprotect_ro_dirty_err(unsigned paged); +static unsigned check_protect_ro_rw_err(unsigned paged); +static unsigned check_protect_retries(unsigned paged); +static unsigned check_check_evictions_enabled_err(unsigned paged); +static unsigned check_auto_cache_resize(hbool_t cork_ageout, unsigned paged); +static unsigned check_auto_cache_resize_disable(unsigned paged); +static unsigned check_auto_cache_resize_epoch_markers(unsigned paged); +static unsigned check_auto_cache_resize_input_errs(unsigned paged); +static unsigned check_auto_cache_resize_aux_fcns(unsigned paged); +static unsigned check_metadata_blizzard_absence(hbool_t fill_via_insertion, unsigned paged); +static unsigned check_flush_deps(unsigned paged); +static unsigned check_flush_deps_err(unsigned paged); +static unsigned check_flush_deps_order(unsigned paged); +static unsigned check_notify_cb(unsigned paged); +static unsigned check_metadata_cork(hbool_t fill_via_insertion, unsigned paged); +static unsigned check_entry_deletions_during_scans(unsigned paged); static void cedds__expunge_dirty_entry_in_flush_test(H5F_t * file_ptr); static void cedds__H5C_make_space_in_cache(H5F_t * file_ptr); static void cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * file_ptr); static void cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr); -static unsigned check_stats(void); +static unsigned check_stats(unsigned paged); #if H5C_COLLECT_CACHE_STATS static void check_stats__smoke_check_1(H5F_t * file_ptr); #endif /* H5C_COLLECT_CACHE_STATS */ @@ -228,7 +229,7 @@ static void check_stats__smoke_check_1(H5F_t * file_ptr); */ static unsigned -smoke_check_1(int express_test) +smoke_check_1(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = FALSE; @@ -239,7 +240,10 @@ smoke_check_1(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #1 -- all clean, ins, dest, ren, 4/2 MB cache"); + if(paged) + TESTING("smoke check #1 -- all clean, ins, dest, ren, 4/2 MB cache, paged aggregation") + else + TESTING("smoke check #1 -- all clean, ins, dest, ren, 4/2 MB cache") switch (express_test) { @@ -274,8 +278,7 @@ smoke_check_1(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -427,7 +430,7 @@ smoke_check_1(int express_test) */ static unsigned -smoke_check_2(int express_test) +smoke_check_2(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = TRUE; @@ -438,7 +441,10 @@ smoke_check_2(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #2 -- ~1/2 dirty, ins, dest, ren, 4/2 MB cache"); + if(paged) + TESTING("smoke check #2 -- ~1/2 dirty, ins, dest, ren, 4/2 MB cache, paged aggregation") + else + TESTING("smoke check #2 -- ~1/2 dirty, ins, dest, ren, 4/2 MB cache") switch (express_test) { @@ -473,8 +479,7 @@ smoke_check_2(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -625,7 +630,7 @@ smoke_check_2(int express_test) */ static unsigned -smoke_check_3(int express_test) +smoke_check_3(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = FALSE; @@ -636,7 +641,10 @@ smoke_check_3(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #3 -- all clean, ins, dest, ren, 2/1 KB cache"); + if(paged) + TESTING("smoke check #3 -- all clean, ins, dest, ren, 2/1 KB cache, paged aggregation") + else + TESTING("smoke check #3 -- all clean, ins, dest, ren, 2/1 KB cache") switch (express_test) { @@ -671,8 +679,7 @@ smoke_check_3(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -824,7 +831,7 @@ smoke_check_3(int express_test) */ static unsigned -smoke_check_4(int express_test) +smoke_check_4(int express_test, unsigned paged) { hbool_t show_progress = FALSE; int dirty_unprotects = TRUE; @@ -835,7 +842,10 @@ smoke_check_4(int express_test) int mile_stone = 1; H5F_t * file_ptr = NULL; - TESTING("smoke check #4 -- ~1/2 dirty, ins, dest, ren, 2/1 KB cache"); + if(paged) + TESTING("smoke check #4 -- ~1/2 dirty, ins, dest, ren, 2/1 KB cache, paged aggregation") + else + TESTING("smoke check #4 -- ~1/2 dirty, ins, dest, ren, 2/1 KB cache") switch (express_test) { @@ -870,8 +880,7 @@ smoke_check_4(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -1023,7 +1032,7 @@ smoke_check_4(int express_test) */ static unsigned -smoke_check_5(int express_test) +smoke_check_5(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1082,7 +1091,10 @@ smoke_check_5(int express_test) /* double empty_reserve = */ 0.5f }; - TESTING("smoke check #5 -- all clean, ins, prot, unprot, AR cache 1"); + if(paged) + TESTING("smoke check #5 -- all clean, ins, prot, unprot, AR cache 1, paged aggregation") + else + TESTING("smoke check #5 -- all clean, ins, prot, unprot, AR cache 1") switch (express_test) { @@ -1117,8 +1129,7 @@ smoke_check_5(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1259,7 +1270,7 @@ smoke_check_5(int express_test) */ static unsigned -smoke_check_6(int express_test) +smoke_check_6(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1318,7 +1329,10 @@ smoke_check_6(int express_test) /* double empty_reserve = */ 0.05f }; - TESTING("smoke check #6 -- ~1/2 dirty, ins, prot, unprot, AR cache 1"); + if(paged) + TESTING("smoke check #6 -- ~1/2 dirty, ins, prot, unprot, AR cache 1, paged aggregation") + else + TESTING("smoke check #6 -- ~1/2 dirty, ins, prot, unprot, AR cache 1") pass = TRUE; @@ -1353,8 +1367,7 @@ smoke_check_6(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1495,7 +1508,7 @@ smoke_check_6(int express_test) */ static unsigned -smoke_check_7(int express_test) +smoke_check_7(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1555,7 +1568,10 @@ smoke_check_7(int express_test) /* double empty_reserve = */ 0.1f }; - TESTING("smoke check #7 -- all clean, ins, prot, unprot, AR cache 2"); + if(paged) + TESTING("smoke check #7 -- all clean, ins, prot, unprot, AR cache 2, paged aggregation") + else + TESTING("smoke check #7 -- all clean, ins, prot, unprot, AR cache 2") switch (express_test) { @@ -1590,8 +1606,7 @@ smoke_check_7(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1732,7 +1747,7 @@ smoke_check_7(int express_test) */ static unsigned -smoke_check_8(int express_test) +smoke_check_8(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1792,7 +1807,10 @@ smoke_check_8(int express_test) /* double empty_reserve = */ 0.1f }; - TESTING("smoke check #8 -- ~1/2 dirty, ins, prot, unprot, AR cache 2"); + if(paged) + TESTING("smoke check #8 -- ~1/2 dirty, ins, prot, unprot, AR cache 2, paged aggregation") + else + TESTING("smoke check #8 -- ~1/2 dirty, ins, prot, unprot, AR cache 2") switch (express_test) { @@ -1827,7 +1845,7 @@ smoke_check_8(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(pass) { @@ -1975,7 +1993,7 @@ smoke_check_8(int express_test) */ static unsigned -smoke_check_9(int express_test) +smoke_check_9(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -1989,7 +2007,10 @@ smoke_check_9(int express_test) H5F_t * file_ptr = NULL; H5C_t * cache_ptr = NULL; - TESTING("smoke check #9 -- all clean, ins, dest, ren, 4/2 MB, corked"); + if(paged) + TESTING("smoke check #9 -- all clean, ins, dest, ren, 4/2 MB, corked, paged aggregation") + else + TESTING("smoke check #9 -- all clean, ins, dest, ren, 4/2 MB, corked") switch (express_test) { @@ -2024,8 +2045,7 @@ smoke_check_9(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); cache_ptr = file_ptr->shared->cache; /* disable evictions */ @@ -2295,7 +2315,7 @@ smoke_check_9(int express_test) */ static unsigned -smoke_check_10(int express_test) +smoke_check_10(int express_test, unsigned paged) { herr_t result; hbool_t show_progress = FALSE; @@ -2309,7 +2329,10 @@ smoke_check_10(int express_test) H5F_t * file_ptr = NULL; H5C_t * cache_ptr = NULL; - TESTING("smoke check #10 -- ~1/2 dirty, ins, dest, ren, 4/2 MB, corked"); + if(paged) + TESTING("smoke check #10 -- ~1/2 dirty, ins, dest, ren, 4/2 MB, corked, paged aggregation") + else + TESTING("smoke check #10 -- ~1/2 dirty, ins, dest, ren, 4/2 MB, corked") switch (express_test) { @@ -2344,8 +2367,7 @@ smoke_check_10(int express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(4 * 1024 * 1024), - (size_t)(2 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(4 * 1024 * 1024), (size_t)(2 * 1024 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(show_progress) /* 3 */ @@ -2609,7 +2631,7 @@ write_permitted_check(int #if !H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS H5_ATTR_UNUSED #endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ -express_test) +express_test, unsigned paged) { #if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS @@ -2623,7 +2645,10 @@ express_test) #endif /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ - TESTING("write permitted check -- 1/0 MB cache"); + if(paged) + TESTING("write permitted check -- 1/0 MB cache, paged aggregation") + else + TESTING("write permitted check -- 1/0 MB cache") #if H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS @@ -2660,8 +2685,7 @@ express_test) HDfprintf(stdout, "%s() - %0d -- pass = %d\n", FUNC, mile_stone++, (int)pass); - file_ptr = setup_cache((size_t)(1 * 1024 * 1024), - (size_t)(0)); + file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)0, paged); if(show_progress) /* 3 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -2823,7 +2847,7 @@ express_test) */ static unsigned -check_insert_entry(void) +check_insert_entry(unsigned paged) { int entry_type = PICO_ENTRY_TYPE; int i; @@ -2839,8 +2863,10 @@ check_insert_entry(void) test_entry_t * entry_ptr; struct H5C_cache_entry_t * search_ptr; - - TESTING("H5C_insert_entry() functionality"); + if(paged) + TESTING("H5C_insert_entry() functionality (paged aggregation)") + else + TESTING("H5C_insert_entry() functionality") pass = TRUE; @@ -2858,8 +2884,7 @@ check_insert_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -3115,11 +3140,14 @@ check_insert_entry(void) */ static unsigned -check_flush_cache(void) +check_flush_cache(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("H5C_flush_cache() functionality"); + if(paged) + TESTING("H5C_flush_cache() functionality (paged aggregation)") + else + TESTING("H5C_flush_cache() functionality") pass = TRUE; @@ -3132,8 +3160,7 @@ check_flush_cache(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); } /* first test behaviour on an empty cache. Can't do much sanity @@ -12751,7 +12778,7 @@ check_flush_cache__pinned_single_entry_test(H5F_t * file_ptr, */ static unsigned -check_get_entry_status(void) +check_get_entry_status(unsigned paged) { static char msg[128]; herr_t result; @@ -12764,7 +12791,10 @@ check_get_entry_status(void) test_entry_t * base_addr = NULL; test_entry_t * entry_ptr = NULL; - TESTING("H5C_get_entry_status() functionality"); + if(paged) + TESTING("H5C_get_entry_status() functionality (paged aggregation)") + else + TESTING("H5C_get_entry_status() functionality") pass = TRUE; @@ -12772,8 +12802,7 @@ check_get_entry_status(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); if(file_ptr == NULL) { @@ -12980,7 +13009,7 @@ check_get_entry_status(void) */ static unsigned -check_expunge_entry(void) +check_expunge_entry(unsigned paged) { static char msg[128]; herr_t result; @@ -12993,7 +13022,10 @@ check_expunge_entry(void) test_entry_t * base_addr; test_entry_t * entry_ptr; - TESTING("H5C_expunge_entry() functionality"); + if(paged) + TESTING("H5C_expunge_entry() functionality (paged aggregation)") + else + TESTING("H5C_expunge_entry() functionality") pass = TRUE; @@ -13001,8 +13033,7 @@ check_expunge_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); base_addr = entries[0]; entry_ptr = &(base_addr[0]); @@ -13289,7 +13320,7 @@ check_expunge_entry(void) *------------------------------------------------------------------------- */ static unsigned -check_multiple_read_protect(void) +check_multiple_read_protect(unsigned paged) { H5F_t * file_ptr = NULL; #if H5C_COLLECT_CACHE_STATS @@ -13297,7 +13328,10 @@ check_multiple_read_protect(void) #endif /* H5C_COLLECT_CACHE_STATS */ test_entry_t * entry_ptr; - TESTING("multiple read only protects on a single entry"); + if(paged) + TESTING("multiple read only protects on a single entry (paged aggregation)") + else + TESTING("multiple read only protects on a single entry") pass = TRUE; @@ -13326,8 +13360,7 @@ check_multiple_read_protect(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); #if H5C_COLLECT_CACHE_STATS cache_ptr = file_ptr->shared->cache; #endif /* H5C_COLLECT_CACHE_STATS */ @@ -13711,7 +13744,7 @@ check_multiple_read_protect(void) */ static unsigned -check_move_entry(void) +check_move_entry(unsigned paged) { unsigned u; H5F_t * file_ptr = NULL; @@ -13743,7 +13776,10 @@ check_move_entry(void) }, }; - TESTING("H5C_move_entry() functionality"); + if(paged) + TESTING("H5C_move_entry() functionality (paged aggregation)") + else + TESTING("H5C_move_entry() functionality") pass = TRUE; @@ -13776,8 +13812,7 @@ check_move_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); } u = 0; @@ -14003,7 +14038,7 @@ check_move_entry__run_test(H5F_t * file_ptr, */ static unsigned -check_pin_protected_entry(void) +check_pin_protected_entry(unsigned paged) { static char msg[128]; herr_t result; @@ -14011,7 +14046,10 @@ check_pin_protected_entry(void) test_entry_t * base_addr; test_entry_t * entry_ptr; - TESTING("H5C_pin_protected_entry() functionality"); + if(paged) + TESTING("H5C_pin_protected_entry() functionality (paged aggregation)") + else + TESTING("H5C_pin_protected_entry() functionality") pass = TRUE; @@ -14024,8 +14062,7 @@ check_pin_protected_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); if(file_ptr == NULL) { @@ -14102,7 +14139,7 @@ check_pin_protected_entry(void) */ static unsigned -check_resize_entry(void) +check_resize_entry(unsigned paged) { static char msg[128]; herr_t result; @@ -14117,7 +14154,10 @@ check_resize_entry(void) test_entry_t * base_addr; test_entry_t * entry_ptr = NULL; - TESTING("entry resize functionality"); + if(paged) + TESTING("entry resize functionality (paged aggregation)") + else + TESTING("entry resize functionality") /* Setup a cache and verify that it is empty. * @@ -14156,8 +14196,8 @@ check_resize_entry(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); + if(file_ptr == NULL) { pass = FALSE; @@ -15097,7 +15137,7 @@ check_resize_entry(void) */ static unsigned -check_evictions_enabled(void) +check_evictions_enabled(unsigned paged) { static char msg[128]; herr_t result; @@ -15111,7 +15151,10 @@ check_evictions_enabled(void) test_entry_t * base_addr = NULL; test_entry_t * entry_ptr; - TESTING("evictions enabled/disabled functionality"); + if(paged) + TESTING("evictions enabled/disabled functionality (paged aggregation)") + else + TESTING("evictions enabled/disabled functionality") /* Setup a cache and verify that it is empty. * @@ -15159,8 +15202,8 @@ check_evictions_enabled(void) reset_entries(); - file_ptr = setup_cache((size_t)(1 * 1024 * 1024), - (size_t)( 512 * 1024)); + file_ptr = setup_cache((size_t)(1 * 1024 * 1024), (size_t)(512 * 1024), paged); + if(file_ptr == NULL) { pass = FALSE; @@ -15830,11 +15873,14 @@ check_evictions_enabled(void) */ static unsigned -check_flush_protected_err(void) +check_flush_protected_err(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("flush cache with protected entry error"); + if(paged) + TESTING("flush cache with protected entry error (paged aggregation)") + else + TESTING("flush cache with protected entry error") pass = TRUE; @@ -15847,8 +15893,7 @@ check_flush_protected_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -15904,11 +15949,14 @@ check_flush_protected_err(void) *------------------------------------------------------------------------- */ static unsigned -check_destroy_pinned_err(void) +check_destroy_pinned_err(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("destroy cache with permanently pinned entry error"); + if(paged) + TESTING("destroy cache with permanently pinned entry error (paged aggregation)") + else + TESTING("destroy cache with permanently pinned entry error") pass = TRUE; @@ -15920,8 +15968,7 @@ check_destroy_pinned_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); unprotect_entry(file_ptr, 0, 0, H5C__PIN_ENTRY_FLAG); @@ -15985,11 +16032,14 @@ check_destroy_pinned_err(void) */ static unsigned -check_destroy_protected_err(void) +check_destroy_protected_err(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("destroy cache with protected entry error"); + if(paged) + TESTING("destroy cache with protected entry error (paged aggregation)") + else + TESTING("destroy cache with protected entry error") pass = TRUE; @@ -16002,8 +16052,7 @@ check_destroy_protected_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); /* Note: normally this call would go just before the series of * flushes prior to file close -- in particular, all entries @@ -16075,14 +16124,17 @@ check_destroy_protected_err(void) */ static unsigned -check_duplicate_insert_err(void) +check_duplicate_insert_err(unsigned paged) { herr_t result = -1; H5F_t * file_ptr = NULL; test_entry_t * base_addr; test_entry_t * entry_ptr; - TESTING("duplicate entry insertion error"); + if(paged) + TESTING("duplicate entry insertion error (paged aggregation)") + else + TESTING("duplicate entry insertion error") pass = TRUE; @@ -16095,8 +16147,7 @@ check_duplicate_insert_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16155,13 +16206,16 @@ check_duplicate_insert_err(void) */ static unsigned -check_double_pin_err(void) +check_double_pin_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("pin a pinned entry error"); + if(paged) + TESTING("pin a pinned entry error (paged aggregation)") + else + TESTING("pin a pinned entry error") pass = TRUE; @@ -16175,8 +16229,7 @@ check_double_pin_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16241,13 +16294,16 @@ check_double_pin_err(void) */ static unsigned -check_double_unpin_err(void) +check_double_unpin_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("unpin an unpinned entry error"); + if(paged) + TESTING("unpin an unpinned entry error (paged aggregation)") + else + TESTING("unpin an unpinned entry error") pass = TRUE; @@ -16263,8 +16319,7 @@ check_double_unpin_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16338,13 +16393,16 @@ check_double_unpin_err(void) */ static unsigned -check_pin_entry_errs(void) +check_pin_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("pin entry related errors"); + if(paged) + TESTING("pin entry related errors (paged aggregation)") + else + TESTING("pin entry related errors") pass = TRUE; @@ -16364,8 +16422,7 @@ check_pin_entry_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16444,13 +16501,16 @@ check_pin_entry_errs(void) */ static unsigned -check_double_protect_err(void) +check_double_protect_err(unsigned paged) { H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; H5C_cache_entry_t * cache_entry_ptr; - TESTING("protect a protected entry error"); + if(paged) + TESTING("protect a protected entry error (paged aggregation)") + else + TESTING("protect a protected entry error") pass = TRUE; @@ -16463,8 +16523,7 @@ check_double_protect_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16522,13 +16581,16 @@ check_double_protect_err(void) */ static unsigned -check_double_unprotect_err(void) +check_double_unprotect_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("unprotect an unprotected entry error"); + if(paged) + TESTING("unprotect an unprotected entry error (paged aggregation)") + else + TESTING("unprotect an unprotected entry error") pass = TRUE; @@ -16541,8 +16603,7 @@ check_double_unprotect_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16600,13 +16661,16 @@ check_double_unprotect_err(void) */ static unsigned -check_mark_entry_dirty_errs(void) +check_mark_entry_dirty_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("mark entry dirty related errors"); + if(paged) + TESTING("mark entry dirty related errors (paged aggregation)") + else + TESTING("mark entry dirty related errors") pass = TRUE; @@ -16620,8 +16684,7 @@ check_mark_entry_dirty_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry(file_ptr, 0, 0); @@ -16680,13 +16743,16 @@ check_mark_entry_dirty_errs(void) */ static unsigned -check_expunge_entry_errs(void) +check_expunge_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("expunge entry related errors"); + if(paged) + TESTING("expunge entry related errors (paged aggregation)") + else + TESTING("expunge entry related errors") pass = TRUE; @@ -16706,8 +16772,7 @@ check_expunge_entry_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); entry_ptr = &((entries[0])[0]); @@ -16798,7 +16863,7 @@ check_expunge_entry_errs(void) *------------------------------------------------------------------------- */ static unsigned -check_move_entry_errs(void) +check_move_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; @@ -16808,7 +16873,10 @@ check_move_entry_errs(void) test_entry_t * entry_0_1_ptr; test_entry_t * entry_1_0_ptr; - TESTING("move entry related errors"); + if(paged) + TESTING("move entry related errors (paged aggregation)") + else + TESTING("move entry related errors") pass = TRUE; @@ -16820,7 +16888,7 @@ check_move_entry_errs(void) if(pass) { reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; insert_entry(file_ptr, 0, 0, H5C__NO_FLAGS_SET); @@ -16863,8 +16931,7 @@ check_move_entry_errs(void) if(pass) { reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); - + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; insert_entry(file_ptr, 0, 0, H5C__NO_FLAGS_SET); @@ -16914,13 +16981,16 @@ check_move_entry_errs(void) */ static unsigned -check_resize_entry_errs(void) +check_resize_entry_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("resize entry related errors"); + if(paged) + TESTING("resize entry related errors (paged aggregation)") + else + TESTING("resize entry related errors") pass = TRUE; @@ -16939,8 +17009,7 @@ check_resize_entry_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); entry_ptr = &((entries[0])[0]); @@ -17015,13 +17084,16 @@ check_resize_entry_errs(void) */ static unsigned -check_unprotect_ro_dirty_err(void) +check_unprotect_ro_dirty_err(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; - TESTING("unprotect a read only entry dirty error"); + if(paged) + TESTING("unprotect a read only entry dirty error (paged aggregation)") + else + TESTING("unprotect a read only entry dirty error") pass = TRUE; @@ -17034,8 +17106,7 @@ check_unprotect_ro_dirty_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry_ro(file_ptr, 0, 0); @@ -17076,8 +17147,7 @@ check_unprotect_ro_dirty_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry_ro(file_ptr, 0, 0); protect_entry_ro(file_ptr, 0, 0); @@ -17142,13 +17212,16 @@ check_unprotect_ro_dirty_err(void) */ static unsigned -check_protect_ro_rw_err(void) +check_protect_ro_rw_err(unsigned paged) { H5F_t * file_ptr = NULL; test_entry_t * entry_ptr; void * thing_ptr = NULL; - TESTING("protect a read only entry rw error"); + if(paged) + TESTING("protect a read only entry rw error (paged aggregation)") + else + TESTING("protect a read only entry rw error") pass = TRUE; @@ -17162,8 +17235,7 @@ check_protect_ro_rw_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); protect_entry_ro(file_ptr, 0, 0); @@ -17219,7 +17291,7 @@ check_protect_ro_rw_err(void) *------------------------------------------------------------------------- */ static unsigned -check_protect_retries(void) +check_protect_retries(unsigned paged) { H5F_t * file_ptr = NULL; H5C_t *cache_ptr = NULL; @@ -17229,7 +17301,10 @@ check_protect_retries(void) int32_t type; int32_t idx; - TESTING("protect an entry to verify retries"); + if(paged) + TESTING("protect an entry to verify retries (paged aggregation)") + else + TESTING("protect an entry to verify retries") pass = TRUE; @@ -17238,8 +17313,7 @@ check_protect_retries(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); /* Set up read attempts for verifying checksum */ file_ptr->shared->read_attempts = 10; @@ -17354,14 +17428,17 @@ check_protect_retries(void) */ static unsigned -check_check_evictions_enabled_err(void) +check_check_evictions_enabled_err(unsigned paged) { herr_t result; hbool_t evictions_enabled; H5F_t * file_ptr = NULL; H5C_t * cache_ptr = NULL; - TESTING("get/set evictions enabled errors"); + if(paged) + TESTING("get/set evictions enabled errors (paged aggregation)") + else + TESTING("get/set evictions enabled errors") pass = TRUE; @@ -17382,8 +17459,7 @@ check_check_evictions_enabled_err(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -17506,7 +17582,7 @@ static void test_rpt_fcn(H5_ATTR_UNUSED H5C_t * cache_ptr, } static unsigned -check_auto_cache_resize(hbool_t cork_ageout) +check_auto_cache_resize(hbool_t cork_ageout, unsigned paged) { hbool_t show_progress = FALSE; herr_t result; @@ -17560,7 +17636,10 @@ check_auto_cache_resize(hbool_t cork_ageout) /* double empty_reserve = */ 0.05f }; - TESTING("automatic cache resizing"); + if(paged) + TESTING("automatic cache resizing (paged aggregation)") + else + TESTING("automatic cache resizing") pass = TRUE; @@ -17574,7 +17653,7 @@ check_auto_cache_resize(hbool_t cork_ageout) if(pass) { reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -21576,7 +21655,7 @@ check_auto_cache_resize(hbool_t cork_ageout) */ static unsigned -check_auto_cache_resize_disable(void) +check_auto_cache_resize_disable(unsigned paged) { hbool_t show_progress = FALSE; herr_t result; @@ -21630,7 +21709,10 @@ check_auto_cache_resize_disable(void) /* double empty_reserve = */ 0.05f }; - TESTING("automatic cache resize disable"); + if(paged) + TESTING("automatic cache resize disable (paged aggregation)") + else + TESTING("automatic cache resize disable") pass = TRUE; @@ -21645,8 +21727,7 @@ check_auto_cache_resize_disable(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(file_ptr == NULL) { @@ -24301,7 +24382,7 @@ check_auto_cache_resize_disable(void) */ static unsigned -check_auto_cache_resize_epoch_markers(void) +check_auto_cache_resize_epoch_markers(unsigned paged) { hbool_t show_progress = FALSE; herr_t result; @@ -24356,7 +24437,10 @@ check_auto_cache_resize_epoch_markers(void) /* double empty_reserve = */ 0.05f }; - TESTING("automatic cache resize epoch marker management"); + if(paged) + TESTING("automatic cache resize epoch marker management (paged aggregation)") + else + TESTING("automatic cache resize epoch marker management") pass = TRUE; @@ -24366,8 +24450,7 @@ check_auto_cache_resize_epoch_markers(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -25010,7 +25093,7 @@ check_auto_cache_resize_epoch_markers(void) */ static unsigned -check_auto_cache_resize_input_errs(void) +check_auto_cache_resize_input_errs(unsigned paged) { herr_t result; H5F_t * file_ptr = NULL; @@ -25064,7 +25147,10 @@ check_auto_cache_resize_input_errs(void) H5C_auto_size_ctl_t invalid_auto_size_ctl; H5C_auto_size_ctl_t test_auto_size_ctl; - TESTING("automatic cache resize input errors"); + if(paged) + TESTING("automatic cache resize input errors (paged aggregation)") + else + TESTING("automatic cache resize input errors") pass = TRUE; @@ -25078,8 +25164,7 @@ check_auto_cache_resize_input_errs(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -27417,7 +27502,7 @@ check_auto_cache_resize_input_errs(void) */ static unsigned -check_auto_cache_resize_aux_fcns(void) +check_auto_cache_resize_aux_fcns(unsigned paged) { herr_t result; int32_t i; @@ -27478,7 +27563,10 @@ check_auto_cache_resize_aux_fcns(void) }; - TESTING("automatic cache resize auxilary functions"); + if(paged) + TESTING("automatic cache resize auxilary functions (paged aggregation)") + else + TESTING("automatic cache resize auxilary functions") pass = TRUE; @@ -27489,8 +27577,7 @@ check_auto_cache_resize_aux_fcns(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), - (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; } @@ -27962,7 +28049,7 @@ check_auto_cache_resize_aux_fcns(void) */ static unsigned -check_metadata_blizzard_absence(hbool_t fill_via_insertion) +check_metadata_blizzard_absence(hbool_t fill_via_insertion, unsigned paged) { int entry_type = HUGE_ENTRY_TYPE; size_t entry_size = HUGE_ENTRY_SIZE; /* 16 KB */ @@ -28142,13 +28229,17 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion) reset_entries(); if(fill_via_insertion) { - - TESTING("to ensure metadata blizzard absence when inserting"); - - } else { - - TESTING("to ensure metadata blizzard absence on protect/unprotect"); - } + if(paged) + TESTING("to ensure metadata blizzard absence when inserting (paged aggregation)") + else + TESTING("to ensure metadata blizzard absence when inserting") + } /* end if */ + else { + if(paged) + TESTING("to ensure metadata blizzard absence on protect/unprotect (paged aggregation)") + else + TESTING("to ensure metadata blizzard absence on protect/unprotect") + } /* end else */ if(show_progress) /* 0 */ HDfprintf(stdout, "\n%s: check point %d -- pass %d\n", FUNC, checkpoint++, pass); @@ -28160,8 +28251,7 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion) * The max_cache_size should have room for 50 entries. * The min_clean_size is half of that, or 25 entries. */ - file_ptr = setup_cache((size_t)(50 * entry_size), /* max_cache_size */ - (size_t)(25 * entry_size)); /* min_clean_size */ + file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged); if(file_ptr == NULL) { @@ -28805,7 +28895,7 @@ check_metadata_blizzard_absence(hbool_t fill_via_insertion) */ static unsigned -check_flush_deps(void) +check_flush_deps(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ H5C_t * cache_ptr = NULL; /* Metadata cache for this test */ @@ -28823,7 +28913,10 @@ check_flush_deps(void) { PICO_ENTRY_TYPE, 4, PICO_ENTRY_SIZE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, {0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0}, 0, 0, 0, -1, FALSE} }; - TESTING("flush dependencies"); + if(paged) + TESTING("flush dependencies (paged aggregation)") + else + TESTING("flush dependencies") pass = TRUE; @@ -28832,7 +28925,7 @@ check_flush_deps(void) */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; base_addr = entries[entry_type]; @@ -30541,13 +30634,16 @@ done: */ static unsigned -check_flush_deps_err(void) +check_flush_deps_err(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ int entry_type = PICO_ENTRY_TYPE; /* Use very small entry size (size of entries doesn't matter) */ unsigned test_count; /* Test iteration variable */ - TESTING("flush dependency errors"); + if(paged) + TESTING("flush dependency errors (paged aggregation)") + else + TESTING("flush dependency errors") pass = TRUE; @@ -30560,7 +30656,7 @@ check_flush_deps_err(void) /* Allocate a cache */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(!pass) CACHE_ERROR("setup_cache failed") /* Insert entries to work with into the cache */ @@ -30762,7 +30858,7 @@ done: */ static unsigned -check_flush_deps_order(void) +check_flush_deps_order(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ H5C_t * cache_ptr = NULL; /* Metadata cache for this test */ @@ -30780,7 +30876,10 @@ check_flush_deps_order(void) }; unsigned flush_order; /* Index for tracking flush order */ - TESTING("flush dependencies flush order"); + if(paged) + TESTING("flush dependencies flush order (paged aggregation)") + else + TESTING("flush dependencies flush order") pass = TRUE; @@ -30789,7 +30888,7 @@ check_flush_deps_order(void) */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); cache_ptr = file_ptr->shared->cache; if(!pass) CACHE_ERROR("setup_cache failed") @@ -33330,7 +33429,7 @@ done: */ static unsigned -check_notify_cb(void) +check_notify_cb(unsigned paged) { H5F_t * file_ptr = NULL; /* File for this test */ H5C_t * cache_ptr = NULL; /* Metadata cache for this test */ @@ -33349,7 +33448,10 @@ check_notify_cb(void) { NOTIFY_ENTRY_TYPE, 4, NOTIFY_ENTRY_SIZE, FALSE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, {0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0}, 0, 0, 0, -1, FALSE} }; - TESTING("'notify' callback"); + if(paged) + TESTING("'notify' callback (paged)") + else + TESTING("'notify' callback") pass = TRUE; @@ -33358,7 +33460,7 @@ check_notify_cb(void) */ reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); if(!file_ptr) CACHE_ERROR("setup_cache returned NULL") cache_ptr = file_ptr->shared->cache; base_addr = entries[entry_type]; @@ -33531,7 +33633,7 @@ done: *------------------------------------------------------------------------- */ static unsigned -check_metadata_cork(hbool_t fill_via_insertion) +check_metadata_cork(hbool_t fill_via_insertion, unsigned paged) { const char * fcn_name = "check_metadata_cork"; int entry_type = HUGE_ENTRY_TYPE; @@ -33727,8 +33829,7 @@ check_metadata_cork(hbool_t fill_via_insertion) * The max_cache_size should have room for 50 entries. * The min_clean_size is half of that, or 25 entries. */ - file_ptr = setup_cache((size_t)(50 * entry_size), /* max_cache_size */ - (size_t)(25 * entry_size)); /* min_clean_size */ + file_ptr = setup_cache((size_t)(50 * entry_size), (size_t)(25 * entry_size), paged); if(file_ptr == NULL) { @@ -34277,11 +34378,14 @@ check_metadata_cork(hbool_t fill_via_insertion) *------------------------------------------------------------------------- */ static unsigned -check_entry_deletions_during_scans(void) +check_entry_deletions_during_scans(unsigned paged) { H5F_t * file_ptr = NULL; - TESTING("entry deletion during list scan detection and adaption"); + if(paged) + TESTING("entry deletion during list scan detection and adaption (paged aggregation)") + else + TESTING("entry deletion during list scan detection and adaption") pass = TRUE; @@ -34294,8 +34398,7 @@ check_entry_deletions_during_scans(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); } /* run the tests. This set of tests is somewhat eclectic, as @@ -35769,7 +35872,7 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t * file_ptr) */ static unsigned -check_stats(void) +check_stats(unsigned paged) { #if H5C_COLLECT_CACHE_STATS @@ -35778,7 +35881,10 @@ check_stats(void) #endif /* H5C_COLLECT_CACHE_STATS */ - TESTING("metadata cache statistics collection"); + if(paged) + TESTING("metadata cache statistics collection (paged aggregation)") + else + TESTING("metadata cache statistics collection") #if H5C_COLLECT_CACHE_STATS @@ -35786,8 +35892,7 @@ check_stats(void) reset_entries(); - file_ptr = setup_cache((size_t)(2 * 1024 * 1024), - (size_t)(1 * 1024 * 1024)); + file_ptr = setup_cache((size_t)(2 * 1024 * 1024), (size_t)(1 * 1024 * 1024), paged); if(pass) { @@ -36268,6 +36373,7 @@ int main(void) { unsigned nerrs = 0; + unsigned paged; int express_test; H5open(); @@ -36285,61 +36391,63 @@ main(void) return EXIT_FAILURE; } /* end if */ - nerrs += smoke_check_1(express_test); - nerrs += smoke_check_2(express_test); - nerrs += smoke_check_3(express_test); - nerrs += smoke_check_4(express_test); - nerrs += smoke_check_5(express_test); - nerrs += smoke_check_6(express_test); - nerrs += smoke_check_7(express_test); - nerrs += smoke_check_8(express_test); - nerrs += smoke_check_9(express_test); - nerrs += smoke_check_10(express_test); - - nerrs += write_permitted_check(express_test); - - nerrs += check_insert_entry(); - nerrs += check_flush_cache(); - nerrs += check_get_entry_status(); - nerrs += check_expunge_entry(); - nerrs += check_multiple_read_protect(); - nerrs += check_move_entry(); - nerrs += check_pin_protected_entry(); - nerrs += check_resize_entry(); - nerrs += check_evictions_enabled(); - nerrs += check_flush_protected_err(); - nerrs += check_destroy_pinned_err(); - nerrs += check_destroy_protected_err(); - nerrs += check_duplicate_insert_err(); - nerrs += check_double_pin_err(); - nerrs += check_double_unpin_err(); - nerrs += check_pin_entry_errs(); - nerrs += check_double_protect_err(); - nerrs += check_double_unprotect_err(); - nerrs += check_mark_entry_dirty_errs(); - nerrs += check_expunge_entry_errs(); - nerrs += check_move_entry_errs(); - nerrs += check_resize_entry_errs(); - nerrs += check_unprotect_ro_dirty_err(); - nerrs += check_protect_ro_rw_err(); - nerrs += check_protect_retries(); - nerrs += check_check_evictions_enabled_err(); - nerrs += check_auto_cache_resize(FALSE); - nerrs += check_auto_cache_resize(TRUE); - nerrs += check_auto_cache_resize_disable(); - nerrs += check_auto_cache_resize_epoch_markers(); - nerrs += check_auto_cache_resize_input_errs(); - nerrs += check_auto_cache_resize_aux_fcns(); - nerrs += check_metadata_blizzard_absence(TRUE); - nerrs += check_metadata_blizzard_absence(FALSE); - nerrs += check_flush_deps(); - nerrs += check_flush_deps_err(); - nerrs += check_flush_deps_order(); - nerrs += check_notify_cb(); - nerrs += check_metadata_cork(TRUE); - nerrs += check_metadata_cork(FALSE); - nerrs += check_entry_deletions_during_scans(); - nerrs += check_stats(); + /* Test with paged aggregation enabled or not */ + /* Each test will call setup_cache() which set up the file space strategy according to "paged" */ + for(paged = FALSE; paged <= TRUE; paged++) { + nerrs += smoke_check_1(express_test, paged); + nerrs += smoke_check_2(express_test, paged); + nerrs += smoke_check_3(express_test, paged); + nerrs += smoke_check_4(express_test, paged); + nerrs += smoke_check_5(express_test, paged); + nerrs += smoke_check_6(express_test, paged); + nerrs += smoke_check_7(express_test, paged); + nerrs += smoke_check_8(express_test, paged); + nerrs += smoke_check_9(express_test, paged); + nerrs += smoke_check_10(express_test, paged); + nerrs += write_permitted_check(express_test, paged); + nerrs += check_insert_entry(paged); + nerrs += check_flush_cache(paged); + nerrs += check_get_entry_status(paged); + nerrs += check_expunge_entry(paged); + nerrs += check_multiple_read_protect(paged); + nerrs += check_move_entry(paged); + nerrs += check_pin_protected_entry(paged); + nerrs += check_resize_entry(paged); + nerrs += check_evictions_enabled(paged); + nerrs += check_flush_protected_err(paged); + nerrs += check_destroy_pinned_err(paged); + nerrs += check_destroy_protected_err(paged); + nerrs += check_duplicate_insert_err(paged); + nerrs += check_double_pin_err(paged); + nerrs += check_double_unpin_err(paged); + nerrs += check_pin_entry_errs(paged); + nerrs += check_double_protect_err(paged); + nerrs += check_double_unprotect_err(paged); + nerrs += check_mark_entry_dirty_errs(paged); + nerrs += check_expunge_entry_errs(paged); + nerrs += check_move_entry_errs(paged); + nerrs += check_resize_entry_errs(paged); + nerrs += check_unprotect_ro_dirty_err(paged); + nerrs += check_protect_ro_rw_err(paged); + nerrs += check_protect_retries(paged); + nerrs += check_check_evictions_enabled_err(paged); + nerrs += check_auto_cache_resize(FALSE, paged); + nerrs += check_auto_cache_resize(TRUE, paged); + nerrs += check_auto_cache_resize_disable(paged); + nerrs += check_auto_cache_resize_epoch_markers(paged); + nerrs += check_auto_cache_resize_input_errs(paged); + nerrs += check_auto_cache_resize_aux_fcns(paged); + nerrs += check_metadata_blizzard_absence(TRUE, paged); + nerrs += check_metadata_blizzard_absence(FALSE, paged); + nerrs += check_flush_deps(paged); + nerrs += check_flush_deps_err(paged); + nerrs += check_flush_deps_order(paged); + nerrs += check_notify_cb(paged); + nerrs += check_metadata_cork(TRUE, paged); + nerrs += check_metadata_cork(FALSE, paged); + nerrs += check_entry_deletions_during_scans(paged); + nerrs += check_stats(paged); + } /* end for */ /* can't fail, returns void */ free_entry_arrays(); diff --git a/test/cache_api.c b/test/cache_api.c index 710e38f..867f055 100644 --- a/test/cache_api.c +++ b/test/cache_api.c @@ -30,12 +30,12 @@ /* private function declarations: */ -static hbool_t check_fapl_mdc_api_calls(void); -static hbool_t check_file_mdc_api_calls(void); -static hbool_t mdc_api_call_smoke_check(int express_test); +static hbool_t check_fapl_mdc_api_calls(unsigned paged, hid_t fcpl_id); +static hbool_t check_file_mdc_api_calls(unsigned paged, hid_t fcpl_id); +static hbool_t mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id); static H5AC_cache_config_t * init_invalid_configs(void); static hbool_t check_fapl_mdc_api_errs(void); -static hbool_t check_file_mdc_api_errs(void); +static hbool_t check_file_mdc_api_errs(unsigned paged, hid_t fcpl_id); @@ -64,7 +64,7 @@ static hbool_t check_file_mdc_api_errs(void); *------------------------------------------------------------------------- */ static hbool_t -check_fapl_mdc_api_calls(void) +check_fapl_mdc_api_calls(unsigned paged, hid_t fcpl_id) { char filename[512]; herr_t result; @@ -113,7 +113,10 @@ check_fapl_mdc_api_calls(void) H5C_auto_size_ctl_t default_auto_size_ctl; H5C_auto_size_ctl_t mod_auto_size_ctl; - TESTING("MDC/FAPL related API calls"); + if(paged) + TESTING("MDC/FAPL related API calls for paged aggregation strategy") + else + TESTING("MDC/FAPL related API calls") pass = TRUE; @@ -217,7 +220,7 @@ check_fapl_mdc_api_calls(void) /* create the file using the default FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -365,7 +368,7 @@ check_fapl_mdc_api_calls(void) /* create the file using the modified FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); if ( file_id < 0 ) { @@ -517,7 +520,7 @@ check_fapl_mdc_api_calls(void) *------------------------------------------------------------------------- */ static hbool_t -check_file_mdc_api_calls(void) +check_file_mdc_api_calls(unsigned paged, hid_t fcpl_id) { char filename[512]; hid_t file_id = -1; @@ -669,7 +672,10 @@ check_file_mdc_api_calls(void) H5AC__DEFAULT_METADATA_WRITE_STRATEGY }; - TESTING("MDC/FILE related API calls"); + if(paged) + TESTING("MDC/FILE related API calls for paged aggregation strategy") + else + TESTING("MDC/FILE related API calls") pass = TRUE; @@ -695,7 +701,7 @@ check_file_mdc_api_calls(void) /* create the file using the default FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -874,7 +880,7 @@ check_file_mdc_api_calls(void) #define NUM_RANDOM_ACCESSES 200000 static hbool_t -mdc_api_call_smoke_check(int express_test) +mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id) { char filename[512]; hbool_t valid_chunk; @@ -1003,7 +1009,10 @@ mdc_api_call_smoke_check(int express_test) H5AC__DEFAULT_METADATA_WRITE_STRATEGY }; - TESTING("MDC API smoke check"); + if(paged) + TESTING("MDC API smoke check for paged aggregation strategy") + else + TESTING("MDC API smoke check") pass = TRUE; @@ -1038,7 +1047,7 @@ mdc_api_call_smoke_check(int express_test) /* create the file using the default FAPL */ if ( pass ) { - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -1921,7 +1930,7 @@ check_fapl_mdc_api_errs(void) *------------------------------------------------------------------------- */ static hbool_t -check_file_mdc_api_errs(void) +check_file_mdc_api_errs(unsigned paged, hid_t fcpl_id) { char filename[512]; static char msg[128]; @@ -1937,7 +1946,10 @@ check_file_mdc_api_errs(void) H5AC_cache_config_t default_config = H5AC__DEFAULT_CACHE_CONFIG; H5AC_cache_config_t scratch; - TESTING("MDC/FILE related API input errors"); + if(paged) + TESTING("MDC/FILE related API input errors for paged aggregation strategy") + else + TESTING("MDC/FILE related API input errors") pass = TRUE; @@ -1968,7 +1980,7 @@ check_file_mdc_api_errs(void) HDfprintf(stdout, "%s: calling H5Fcreate().\n", FUNC); } - file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); if ( file_id < 0 ) { @@ -2267,6 +2279,9 @@ main(void) { unsigned nerrs = 0; int express_test; + hid_t fcpl_id = -1; + hid_t fcpl2_id = -1; + unsigned paged; H5open(); @@ -2281,42 +2296,64 @@ main(void) /* Initialize invalid configurations. */ invalid_configs = init_invalid_configs(); - if ( NULL == invalid_configs ) { - failure_mssg = "Unable to allocate memory for invalid configs."; HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); return EXIT_FAILURE; - } + } /* end if */ - if ( !check_fapl_mdc_api_calls() ) { + if((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0) { + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ - nerrs += 1; - } + /* Set file space strategy to default or paged aggregation strategy */ + if((fcpl2_id = H5Pcopy(fcpl_id)) < 0) { + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ - if ( !check_file_mdc_api_calls() ) { + if(H5Pset_file_space_strategy(fcpl2_id, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)1) < 0) { + failure_mssg = "H5Pset_file_space_strategy() failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ - nerrs += 1; - } + /* Test with paged aggregation enabled or not */ + /* The "my_fcpl" passed to each test has the paged or non-paged strategy set up accordinly */ + for(paged = FALSE; paged <= TRUE; paged++) { + hid_t my_fcpl = fcpl_id; - if ( !mdc_api_call_smoke_check(express_test) ) { + if(paged) + my_fcpl = fcpl2_id; - nerrs += 1; - } + if(!check_fapl_mdc_api_calls(paged, my_fcpl)) + nerrs += 1; - if ( !check_fapl_mdc_api_errs() ) { + if(!check_file_mdc_api_calls(paged, my_fcpl)) + nerrs += 1; - nerrs += 1; - } + if(!mdc_api_call_smoke_check(express_test, paged, my_fcpl)) + nerrs += 1; - if ( !check_file_mdc_api_errs() ) { + if(!check_file_mdc_api_errs(paged, my_fcpl)) + nerrs += 1; + } /* end for paged */ + if(!check_fapl_mdc_api_errs()) nerrs += 1; - } if(invalid_configs) HDfree(invalid_configs); + if(H5Pclose(fcpl_id) < 0 ) { + failure_mssg = "H5Pclose() failed.\n"; + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", FUNC, failure_mssg); + return EXIT_FAILURE; + } /* end if */ + if(nerrs > 0) return EXIT_FAILURE; else diff --git a/test/cache_common.c b/test/cache_common.c index d1bbf10..52dbb52 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -39,6 +39,12 @@ hid_t saved_fapl_id = H5P_DEFAULT; /* store the fapl id here between * close. */ +hid_t saved_fcpl_id = H5P_DEFAULT; /* store the fcpl id here between + * cache setup and takedown. Note + * that if saved_fcpl_id == H5P_DEFAULT, + * we assume that there is no fcpl to + * close. + */ hid_t saved_fid = -1; /* store the file id here between cache setup * and takedown. */ @@ -1632,9 +1638,12 @@ free_icr(test_entry_t *entry, int32_t entry_type) HDassert(entry->cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert((entry->header.destroy_in_progress) || (entry->header.addr == entry->addr)); + HDassert(entry->header.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(entry->header.size == entry->size); HDassert((entry->type == VARIABLE_ENTRY_TYPE) || (entry->size == entry_sizes[entry->type])); + HDassert(entry->header.tl_next == NULL); + HDassert(entry->header.tl_prev == NULL); if(entry->num_pins > 0) { int i; @@ -3165,7 +3174,8 @@ verify_unprotected(void) H5F_t * setup_cache(size_t max_cache_size, - size_t min_clean_size) + size_t min_clean_size, + unsigned paged) { char filename[512]; hbool_t show_progress = FALSE; @@ -3177,6 +3187,7 @@ setup_cache(size_t max_cache_size, H5F_t * ret_val = NULL; haddr_t actual_base_addr; hid_t fapl_id = H5P_DEFAULT; + hid_t fcpl_id = H5P_DEFAULT; if(show_progress) /* 1 */ HDfprintf(stdout, "%s() - %0d -- pass = %d\n", @@ -3184,6 +3195,36 @@ setup_cache(size_t max_cache_size, saved_fid = -1; + if(pass) { + if((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) == FAIL) { + pass = FALSE; + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed.\n"; + } + } + + if(pass && paged) { + /* Set up paged aggregation strategy */ + if(H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)1) == FAIL) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_strategy() failed.\n"; + H5Pclose(fcpl_id); + fcpl_id = H5P_DEFAULT; + } + } + + if(pass && paged) { + /* Set up file space page size to BASE_ADDR */ + if(H5Pset_file_space_page_size(fcpl_id, (hsize_t)BASE_ADDR) == FAIL) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_page_size() failed.\n"; + H5Pclose(fcpl_id); + fcpl_id = H5P_DEFAULT; + } + } + + if(pass) + saved_fcpl_id = fcpl_id; + /* setup the file name */ if(pass) { if(NULL == h5_fixname(FILENAME[0], H5P_DEFAULT, filename, sizeof(filename))) { @@ -3207,7 +3248,7 @@ setup_cache(size_t max_cache_size, pass = FALSE; failure_mssg = "H5P_set_fapl_core() failed.\n"; } - else if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) { + else if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id)) < 0) { core_file_driver_failed = TRUE; if(verbose) @@ -3226,8 +3267,8 @@ setup_cache(size_t max_cache_size, * If this fails, we are cooked. */ if(pass && fid < 0) { - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); - saved_fid = fid; + fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); + saved_fid = fid; if(fid < 0) { pass = FALSE; @@ -3438,6 +3479,11 @@ takedown_cache(H5F_t * file_ptr, saved_fapl_id = H5P_DEFAULT; } + if ( saved_fcpl_id != H5P_DEFAULT ) { + H5Pclose(saved_fcpl_id); + saved_fcpl_id = H5P_DEFAULT; + } + if ( saved_fid != -1 ) { if ( H5F_addr_defined(saved_actual_base_addr) ) { diff --git a/test/cache_common.h b/test/cache_common.h index f003189..f17c16b 100644 --- a/test/cache_common.h +++ b/test/cache_common.h @@ -643,7 +643,7 @@ H5TEST_DLL void resize_entry(H5F_t * file_ptr, size_t new_size, hbool_t in_cache); -H5TEST_DLL H5F_t *setup_cache(size_t max_cache_size, size_t min_clean_size); +H5TEST_DLL H5F_t *setup_cache(size_t max_cache_size, size_t min_clean_size, unsigned paged); H5TEST_DLL void row_major_scan_forward(H5F_t * file_ptr, int32_t max_index, diff --git a/test/cache_image.c b/test/cache_image.c index 0e7928a..f49f7b3 100644 --- a/test/cache_image.c +++ b/test/cache_image.c @@ -59,6 +59,9 @@ static unsigned cache_image_smoke_check_6(void); static unsigned cache_image_api_error_check_1(void); static unsigned cache_image_api_error_check_2(void); static unsigned cache_image_api_error_check_3(void); +static unsigned cache_image_api_error_check_4(void); + +static unsigned get_free_sections_test(void); /****************************************************************************/ @@ -664,12 +667,10 @@ open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, } if ( ( pass ) && ( config_fsm ) ) { - - if ( H5Pset_file_space(fcpl_id, H5F_FILE_SPACE_ALL_PERSIST, 1) < 0 ) { - - pass = FALSE; - failure_mssg = "H5Pset_file_space() failed."; - } + if(H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_strategy() failed."; + } } if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); @@ -5091,11 +5092,14 @@ cache_image_smoke_check_5(void) * 13) Close the file. * * 14) Get the size of the file. Verify that it is less - * than 1 KB. Without deletions and persistant free + * than 20 KB. Without deletions and persistant free * space managers, size size is about 167 MB, so this * is sufficient to verify that the persistant free * space managers are more or less doing their job. * + * Note that in the absence of paged allocation, file + * size gets below 1 KB. + * * 15) Delete the file. * * Return: void @@ -5408,20 +5412,24 @@ cache_image_smoke_check_6(void) /* 14) Get the size of the file. Verify that it is less - * than 1 KB. Without deletions and persistant free + * than 20 KB. Without deletions and persistant free * space managers, size size is about 167 MB, so this * is sufficient to verify that the persistant free * space managers are more or less doing their job. + * + * Note that in the absence of paged allocation, file + * size gets below 1 KB, but since this test is run both + * with and without paged allocation, we must leave some + * extra space for the paged allocation case. */ - if((file_size = h5_get_file_size(filename, H5P_DEFAULT)) < 0) { - - pass = FALSE; - failure_mssg = "h5_get_file_size() failed.\n"; - - } else if ( file_size > 1024 ) { - - pass = FALSE; - failure_mssg = "unexpectedly large file size.\n"; + if(pass) { + if((file_size = h5_get_file_size(filename, H5P_DEFAULT)) < 0) { + pass = FALSE; + failure_mssg = "h5_get_file_size() failed.\n"; + } else if(file_size > 20 * 1024) { + pass = FALSE; + failure_mssg = "unexpectedly large file size.\n"; + } } if ( show_progress ) @@ -7069,6 +7077,494 @@ cache_image_api_error_check_4(void) return !pass; } /* cache_image_api_error_check_4() */ + + +/*------------------------------------------------------------------------- + * Function: get_free_sections_test() + * + * Purpose: It is possible that H5Fget_free_sections() to be + * called before any activity on the metadata cache. + * This is a potential problem, as satisfying the + * H5Fget_free_sections() call requires access to all + * free space managers. When persistant free space + * managers are enabled, this will require calling + * H5MF_tidy_self_referential_fsm_hack(). This is a + * non issue in the absence of a cache image. However, + * this is a problem if a cache image exists, as + * the call to H5MF_tidy_self_referential_fsm_hack() + * will free the file space allocated to the cache + * image. + * + * The objective of this test is to create a test file + * with both non-empty self referential presistant + * free space managers, and a cache image, and then + * verify that this situation is handled correctly if + * H5Fget_free_sections() is called before the metadata + * cache image is loaded. + * + * The test is set up as follows: + * + * 1) Create a HDF5 file with a cache image requested + * and persistant free space managers enabled. + * + * 2) Create some data sets, and then delete some of + * of those near the beginning of the file. + * + * 3) Close the file. + * + * 4) Open the file read only. + * + * 5) Verify that a cache image exists, and has not + * been loaded. + * + * 6) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + * + * 7) Call H5Fget_free_sections(). + * + * 8) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + * + * 9) Verify that the remaining data sets contain the + * expected data. + * + * 10) Close the file. + * + * 11) Open the file R/W. + * + * 12) Verify that a cache image exists, and has not + * been loaded. + * + * 13) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + * + * 14) Call H5Fget_free_sections(). + * + * 15) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + * + * 16) Verify that the remaining data sets contain the + * expected data. + * + * 17) Delete the remaining data sets. + * + * 18) Close the file. + * + * 19) Verify that file space has been reclaimed. + * + * 20) Discard the file. + * + * Return: void + * + * Programmer: John Mainzer + * 1/10/17 + * + *------------------------------------------------------------------------- + */ +static unsigned +get_free_sections_test(void) +{ + const char * fcn_name = "get_free_sections_test()"; + char filename[512]; + hbool_t show_progress = FALSE; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + h5_stat_size_t file_size; + int cp = 0; + + TESTING("Cache image / H5Fget_free_sections() interaction"); + + pass = TRUE; + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + if ( h5_fixname(FILENAMES[0], H5P_DEFAULT, filename, sizeof(filename)) + == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 1) Create a HDF5 file with a cache image requested + * and persistant free space managers enabled. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ TRUE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ TRUE, + /* config_fsm */ TRUE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Create some data sets, and then delete some of + * of those near the beginning of the file. + */ + + if ( pass ) { + + create_datasets(file_id, 1, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + verify_datasets(file_id, 1, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + delete_datasets(file_id, 1, 5); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 3) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (1).\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 4) Open the file read only. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ TRUE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 5) Verify that a cache image exists, and has not been loaded. */ + + if ( pass ) { + + if ( ( ! file_ptr->shared->cache->load_image ) || + ( file_ptr->shared->cache->image_loaded ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 6) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + */ + + if ( pass ) { + + /* file_ptr->shared->first_alloc_dealloc is set to FALSE if the + * file is opened R/O. + */ + if ( ( file_ptr->shared->first_alloc_dealloc ) || + ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) || + ( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) || + ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc, + file_ptr->shared->cache->image_addr) ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status (1).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 7) Call H5Fget_free_sections(). */ + + if ( pass ) { + + if ( H5Fget_free_sections(file_id, H5FD_MEM_DEFAULT, (size_t)0, NULL) + < 0 ){ + + pass = FALSE; + failure_mssg = "H5Fget_free_sections() failed (1).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 8) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + */ + if ( pass ) { + + if ( ! file_ptr->shared->cache->image_loaded ) { + + pass = FALSE; + failure_mssg = "cache image not loaded (1).\n"; + + } else if ( file_ptr->shared->first_alloc_dealloc ) { + + pass = FALSE; + failure_mssg = "self referential FSMs not floated (1).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 9) Verify that the remaining data sets contain the expected data. */ + + if ( pass ) { + + verify_datasets(file_id, 6, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 10) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 11) Open the file R/W. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 12) Verify that a cache image exists, and has not been loaded. */ + + if ( pass ) { + + if ( ( ! file_ptr->shared->cache->load_image ) || + ( file_ptr->shared->cache->image_loaded ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 13) Verify that one or more self referential FSMs + * have been stored at the end of the file just + * before the cache image. + */ + if ( pass ) { + + if ( ( ! file_ptr->shared->first_alloc_dealloc ) || + ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) || + ( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) || + ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc, + file_ptr->shared->cache->image_addr) ) ) { + + pass = FALSE; + failure_mssg = "unexpected cache image status (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 14) Call H5Fget_free_sections(). */ + + if ( pass ) { + + if ( H5Fget_free_sections(file_id, H5FD_MEM_DEFAULT, (size_t)0, NULL) + < 0 ){ + + pass = FALSE; + failure_mssg = "H5Fget_free_sections() failed (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 15) Verify that the cache image has been loaded and + * that the self referential FSMs have been floated. + */ + if ( pass ) { + + if ( ! file_ptr->shared->cache->image_loaded ) { + + pass = FALSE; + failure_mssg = "cache image not loaded (2).\n"; + + } else if ( file_ptr->shared->first_alloc_dealloc ) { + + pass = FALSE; + failure_mssg = "self referential FSMs not floated (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 16) Verify that the remaining data sets contain the expected data. */ + + if ( pass ) { + + verify_datasets(file_id, 6, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 17) Delete the remaining data sets. */ + + if ( pass ) { + + delete_datasets(file_id, 6, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 18) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (3).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 19) Verify that file space has been reclaimed. */ + + if ( pass ) { + + if((file_size = h5_get_file_size(filename, H5P_DEFAULT)) < 0) { + + pass = FALSE; + failure_mssg = "h5_get_file_size() failed.\n"; + + } else if ( file_size > 20 * 1024 ) { + + pass = FALSE; + failure_mssg = "unexpectedly large file size.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 20) Discard the file. */ + + if ( pass ) { + + if ( HDremove(filename) < 0 ) { + + pass = FALSE; + failure_mssg = "HDremove() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { PASSED(); } else { H5_FAILED(); } + + if ( ! pass ) + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", + FUNC, failure_mssg); + + return !pass; + +} /* get_free_sections_test() */ + + /*------------------------------------------------------------------------- * Function: main * @@ -7117,6 +7613,8 @@ main(void) nerrs += cache_image_api_error_check_3(); nerrs += cache_image_api_error_check_4(); + nerrs += get_free_sections_test(); + return(nerrs > 0); } /* main() */ diff --git a/test/cache_tagging.c b/test/cache_tagging.c index 8901468..957187b 100644 --- a/test/cache_tagging.c +++ b/test/cache_tagging.c @@ -1514,6 +1514,7 @@ check_attribute_rename_tags(hid_t fcpl, int type) haddr_t g_tag = 0; hsize_t dims1[2] = {DIMS, DIMS}; /* dimensions */ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* dimensions */ + hbool_t persistant_fsms = FALSE; /* Testing Macro */ TESTING("tag application during attribute renaming"); @@ -1522,6 +1523,10 @@ check_attribute_rename_tags(hid_t fcpl, int type) /* Setup */ /* ===== */ + /* check to see if the FCPL specified persistant free space managers */ + if(H5Pget_file_space_strategy(fcpl, NULL, &persistant_fsms, NULL) < 0) + TEST_ERROR; + /* Allocate array */ if ( (NULL == (data = (int *)HDcalloc(DIMS * DIMS, sizeof(int)))) ) TEST_ERROR; @@ -1612,9 +1617,18 @@ check_attribute_rename_tags(hid_t fcpl, int type) * 3 calls to verify_tag() for verifying free space: * one freespace header tag for H5FD_MEM_DRAW manager, * one freespace header tag for H5FD_MEM_SUPER manager + * one freespace section info tag for H5FD_MEM_SUPER manager */ if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; - if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; + + /* If the free space managers are persistant, the + * H5MF_tidy_self_referential_fsm_hack() must have been run. + * Since this function floats all self referential free space + * managers, the H5FD_MEM_SUPER FSM will not be in the metadata + * cache. + */ + if(!persistant_fsms && verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR; + if(!persistant_fsms && verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR; /* verify btree header and leaf node belonging to group */ if ( verify_tag(fid, H5AC_BT2_HDR_ID, g_tag) < 0 ) TEST_ERROR; @@ -1679,6 +1693,7 @@ check_attribute_delete_tags(hid_t fcpl, int type) haddr_t g_tag = 0; hsize_t dims1[2] = {DIMS, DIMS}; /* dimensions */ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* dimensions */ + hbool_t persistant_fsms = FALSE; /* Testing Macro */ TESTING("tag application during attribute delete"); @@ -1687,6 +1702,10 @@ check_attribute_delete_tags(hid_t fcpl, int type) /* Setup */ /* ===== */ + /* check to see if the FCPL specified persistant free space managers */ + if ( H5Pget_file_space_strategy(fcpl, NULL, &persistant_fsms, NULL) < 0 ) + TEST_ERROR; + /* Allocate array */ if ( (NULL == (data = (int *)HDcalloc(DIMS * DIMS, sizeof(int)))) ) TEST_ERROR; @@ -1752,12 +1771,26 @@ check_attribute_delete_tags(hid_t fcpl, int type) if ( verify_tag(fid, H5AC_SOHM_TABLE_ID, H5AC__SOHM_TAG) < 0 ) TEST_ERROR; /* - * 2 calls to verify_tag() for verifying free space: - * one freespace header tag for H5FD_MEM_DRAW manager, - * one freespace header tag for H5FD_MEM_SUPER manager - */ - if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; - if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; + * 2 calls to verify_tag() for verifying free space: + * one freespace header tag for free-space header, + * one freespace header tag for free-space section info + */ + if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) + TEST_ERROR; + if ( verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0 ) + TEST_ERROR; + +#if 0 + /* If the free space managers are persistant, the + * H5MF_tidy_self_referential_fsm_hack() must have been run. + * Since this function floats all self referential free space + * managers, the H5FD_MEM_SUPER FSM will not be in the metadata + * cache. + */ + if ( ( ! persistant_fsms ) && + ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) ) + TEST_ERROR; +#endif } /* end if */ @@ -3699,7 +3732,7 @@ main(void) fcpl_shmesg_all = H5Pcreate(H5P_FILE_CREATE); H5Pset_shared_mesg_nindexes(fcpl_shmesg_all, 1); H5Pset_shared_mesg_index(fcpl_shmesg_all, 0, H5O_SHMESG_ALL_FLAG, 20); - H5Pset_file_space(fcpl_shmesg_all, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); + H5Pset_file_space_strategy(fcpl_shmesg_all, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)0); /* ========= */ /* Run Tests */ diff --git a/test/dsets.c b/test/dsets.c index 39b7c22..c090174 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -633,7 +633,7 @@ error: *------------------------------------------------------------------------- */ static herr_t -test_userblock_offset(const char *env_h5_drvr, hid_t fapl) +test_userblock_offset(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { char filename[FILENAME_BUF_SIZE]; hid_t file = -1, fcpl = -1, dataset = -1, space = -1; @@ -651,6 +651,9 @@ test_userblock_offset(const char *env_h5_drvr, hid_t fapl) if((fcpl=H5Pcreate(H5P_FILE_CREATE)) < 0) goto error; if(H5Pset_userblock(fcpl, (hsize_t)USER_BLOCK) < 0) goto error; + if(new_format) + if(H5Pset_file_space_page_size(fcpl, (hsize_t)USER_BLOCK) < 0) + goto error; if((file=H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) goto error; @@ -12658,18 +12661,24 @@ main(void) { char filename[FILENAME_BUF_SIZE]; hid_t file, grp, fapl, fapl2; + hid_t fcpl = -1, fcpl2 = -1; unsigned new_format; + unsigned paged; int mdc_nelmts; size_t rdcc_nelmts; size_t rdcc_nbytes; double rdcc_w0; int nerrors = 0; const char *envval; + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ /* Don't run this test using certain file drivers */ envval = HDgetenv("HDF5_DRIVER"); if(envval == NULL) - envval = "sec2"; + envval = "nomatch"; + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") && HDstrcmp(envval, "multi")); /* Set the random # seed */ HDsrandom((unsigned)HDtime(NULL)); @@ -12691,101 +12700,136 @@ main(void) /* Set the "use the latest version of the format" bounds for creating objects in the file */ if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR + /* create a file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + if((fcpl2 = H5Pcopy(fcpl)) < 0) TEST_ERROR + + /* Set file space strategy to paged aggregation and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + TEST_ERROR + h5_fixname(FILENAME[0], fapl, filename, sizeof filename); - /* Test with old & new format groups */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { - hid_t my_fapl; + /* Test with paged aggregation enabled or not */ + for(paged = FALSE; paged <= TRUE; paged++) { - /* Set the FAPL for the type of format */ - if(new_format) { - puts("\nTesting with new file format:"); - my_fapl = fapl2; - } /* end if */ - else { - puts("Testing with old file format:"); - my_fapl = fapl; - } /* end else */ + /* Temporary: skip testing for multi/split drivers: + fail file create when persisting free-space or using paged aggregation strategy */ + if(!contig_addr_vfd && paged) + continue; - /* Create the file for this test */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) - goto error; + /* Test with old & new format groups */ + for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl, my_fcpl; + + /* Set the FAPL for the type of format */ + if(new_format) { + my_fapl = fapl2; + if(paged) { + my_fcpl = fcpl2; + puts("\nTesting with new file format and paged aggregation"); + } else { + my_fcpl = fcpl; + puts("\nTesting with new file format and non-paged aggregation"); + } + } /* end if */ + else { + my_fapl = fapl; + if(paged) { + my_fcpl = fcpl2; + puts("Testing with old file format and paged aggregation:"); + } else { + my_fcpl = fcpl; + puts("Testing with old file format and non-paged aggregation:"); + } + } /* end else */ - /* Cause the library to emit initial messages */ - if((grp = H5Gcreate2(file, "emit diagnostics", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; - if(H5Oset_comment(grp, "Causes diagnostic messages to be emitted") < 0) - goto error; - if(H5Gclose(grp) < 0) - goto error; + /* Create the file for this test */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, my_fcpl, my_fapl)) < 0) + goto error; + + /* Cause the library to emit initial messages */ + if((grp = H5Gcreate2(file, "emit diagnostics", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if(H5Oset_comment(grp, "Causes diagnostic messages to be emitted") < 0) + goto error; + if(H5Gclose(grp) < 0) + goto error; + + nerrors += (test_create(file) < 0 ? 1 : 0); + nerrors += (test_simple_io(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_compact_io(my_fapl) < 0 ? 1 : 0); + nerrors += (test_max_compact(my_fapl) < 0 ? 1 : 0); + nerrors += (test_conv_buffer(file) < 0 ? 1 : 0); + nerrors += (test_tconv(file) < 0 ? 1 : 0); + nerrors += (test_filters(file, my_fapl) < 0 ? 1 : 0); + nerrors += (test_onebyte_shuffle(file) < 0 ? 1 : 0); + nerrors += (test_nbit_int(file) < 0 ? 1 : 0); + nerrors += (test_nbit_float(file) < 0 ? 1 : 0); + nerrors += (test_nbit_double(file) < 0 ? 1 : 0); + nerrors += (test_nbit_array(file) < 0 ? 1 : 0); + nerrors += (test_nbit_compound(file) < 0 ? 1 : 0); + nerrors += (test_nbit_compound_2(file) < 0 ? 1 : 0); + nerrors += (test_nbit_compound_3(file) < 0 ? 1 : 0); + nerrors += (test_nbit_int_size(file) < 0 ? 1 : 0); + nerrors += (test_nbit_flt_size(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_int(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_int_2(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_float(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_float_2(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_double(file) < 0 ? 1 : 0); + nerrors += (test_scaleoffset_double_2(file) < 0 ? 1 : 0); + nerrors += (test_multiopen (file) < 0 ? 1 : 0); + nerrors += (test_types(file) < 0 ? 1 : 0); + nerrors += (test_userblock_offset(envval, my_fapl, new_format) < 0 ? 1 : 0); + nerrors += (test_missing_filter(file) < 0 ? 1 : 0); + nerrors += (test_can_apply(file) < 0 ? 1 : 0); + nerrors += (test_can_apply2(file) < 0 ? 1 : 0); + nerrors += (test_set_local(my_fapl) < 0 ? 1 : 0); + nerrors += (test_can_apply_szip(file) < 0 ? 1 : 0); + nerrors += (test_compare_dcpl(file) < 0 ? 1 : 0); + nerrors += (test_copy_dcpl(file, my_fapl) < 0 ? 1 : 0); + nerrors += (test_filter_delete(file) < 0 ? 1 : 0); + nerrors += (test_filters_endianess() < 0 ? 1 : 0); + nerrors += (test_zero_dims(file) < 0 ? 1 : 0); + nerrors += (test_missing_chunk(file) < 0 ? 1 : 0); + nerrors += (test_random_chunks(my_fapl) < 0 ? 1 : 0); - nerrors += (test_create(file) < 0 ? 1 : 0); - nerrors += (test_simple_io(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_compact_io(my_fapl) < 0 ? 1 : 0); - nerrors += (test_max_compact(my_fapl) < 0 ? 1 : 0); - nerrors += (test_conv_buffer(file) < 0 ? 1 : 0); - nerrors += (test_tconv(file) < 0 ? 1 : 0); - nerrors += (test_filters(file, my_fapl) < 0 ? 1 : 0); - nerrors += (test_onebyte_shuffle(file) < 0 ? 1 : 0); - nerrors += (test_nbit_int(file) < 0 ? 1 : 0); - nerrors += (test_nbit_float(file) < 0 ? 1 : 0); - nerrors += (test_nbit_double(file) < 0 ? 1 : 0); - nerrors += (test_nbit_array(file) < 0 ? 1 : 0); - nerrors += (test_nbit_compound(file) < 0 ? 1 : 0); - nerrors += (test_nbit_compound_2(file) < 0 ? 1 : 0); - nerrors += (test_nbit_compound_3(file) < 0 ? 1 : 0); - nerrors += (test_nbit_int_size(file) < 0 ? 1 : 0); - nerrors += (test_nbit_flt_size(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_int(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_int_2(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_float(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_float_2(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_double(file) < 0 ? 1 : 0); - nerrors += (test_scaleoffset_double_2(file) < 0 ? 1 : 0); - nerrors += (test_multiopen (file) < 0 ? 1 : 0); - nerrors += (test_types(file) < 0 ? 1 : 0); - nerrors += (test_userblock_offset(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_missing_filter(file) < 0 ? 1 : 0); - nerrors += (test_can_apply(file) < 0 ? 1 : 0); - nerrors += (test_can_apply2(file) < 0 ? 1 : 0); - nerrors += (test_set_local(my_fapl) < 0 ? 1 : 0); - nerrors += (test_can_apply_szip(file) < 0 ? 1 : 0); - nerrors += (test_compare_dcpl(file) < 0 ? 1 : 0); - nerrors += (test_copy_dcpl(file, my_fapl) < 0 ? 1 : 0); - nerrors += (test_filter_delete(file) < 0 ? 1 : 0); - nerrors += (test_filters_endianess() < 0 ? 1 : 0); - nerrors += (test_zero_dims(file) < 0 ? 1 : 0); - nerrors += (test_missing_chunk(file) < 0 ? 1 : 0); - nerrors += (test_random_chunks(my_fapl) < 0 ? 1 : 0); #ifndef H5_NO_DEPRECATED_SYMBOLS - nerrors += (test_deprec(file) < 0 ? 1 : 0); + nerrors += (test_deprec(file) < 0 ? 1 : 0); #endif /* H5_NO_DEPRECATED_SYMBOLS */ - nerrors += (test_huge_chunks(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_cache(my_fapl) < 0 ? 1 : 0); - nerrors += (test_big_chunks_bypass_cache(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_fast(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_reopen_chunk_fast(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_fast_bug1(my_fapl) < 0 ? 1 : 0); - nerrors += (test_chunk_expand(my_fapl) < 0 ? 1 : 0); - nerrors += (test_layout_extend(my_fapl) < 0 ? 1 : 0); - nerrors += (test_fixed_array(my_fapl) < 0 ? 1 : 0); - nerrors += (test_idx_compatible() < 0 ? 1 : 0); - nerrors += (test_unfiltered_edge_chunks(my_fapl) < 0 ? 1 : 0); - nerrors += (test_single_chunk(my_fapl) < 0 ? 1 : 0); - nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); - nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0); - nerrors += (test_swmr_non_latest(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_earray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_farray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_bt2_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); - nerrors += (test_storage_size(my_fapl) < 0 ? 1 : 0); - - if(H5Fclose(file) < 0) - goto error; - } /* end for */ - /* Close 2nd FAPL */ + nerrors += (test_huge_chunks(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_cache(my_fapl) < 0 ? 1 : 0); + nerrors += (test_big_chunks_bypass_cache(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_fast(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_reopen_chunk_fast(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_fast_bug1(my_fapl) < 0 ? 1 : 0); + nerrors += (test_chunk_expand(my_fapl) < 0 ? 1 : 0); + nerrors += (test_layout_extend(my_fapl) < 0 ? 1 : 0); + nerrors += (test_fixed_array(my_fapl) < 0 ? 1 : 0); + nerrors += (test_idx_compatible() < 0 ? 1 : 0); + nerrors += (test_unfiltered_edge_chunks(my_fapl) < 0 ? 1 : 0); + nerrors += (test_single_chunk(my_fapl) < 0 ? 1 : 0); + nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); + nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0); + nerrors += (test_storage_size(my_fapl) < 0 ? 1 : 0); + + nerrors += (test_swmr_non_latest(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_earray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_farray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); + nerrors += (test_bt2_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); + + if(H5Fclose(file) < 0) + goto error; + } /* end for new_format */ + } /* end for paged */ + + /* Close property lists */ if(H5Pclose(fapl2) < 0) TEST_ERROR + if(H5Pclose(fcpl) < 0) TEST_ERROR + if(H5Pclose(fcpl2) < 0) TEST_ERROR /* Tests that do not use files */ nerrors += (test_scatter() < 0 ? 1 : 0); diff --git a/test/fheap.c b/test/fheap.c index 82859d2..f08c0f8 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -26,6 +26,10 @@ #define H5HF_TESTING #include "H5HFpkg.h" /* Fractal heaps */ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" + /* Other private headers that this test requires */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ @@ -126,6 +130,7 @@ typedef struct fheap_test_param_t { fheap_test_fill_t fill; /* How to "bulk" fill heap blocks */ size_t actual_id_len; /* The actual length of heap IDs for a test */ fheap_test_comp_t comp; /* Whether to compress the blocks or not */ + hid_t my_fcpl; /* File creation property list with file space strategy setting */ } fheap_test_param_t; /* Heap state information */ @@ -642,7 +647,7 @@ open_heap(char *filename, hid_t fapl, hid_t dxpl, const H5HF_create_t *cparam, h5_fixname(FILENAME[0], fapl, filename, (size_t)FHEAP_FILENAME_LEN); /* Create the file to work on */ - if((*file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((*file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Check for deleting the entire heap */ @@ -1827,7 +1832,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ char filename[FHEAP_FILENAME_LEN]; /* Filename to use */ @@ -1844,7 +1849,7 @@ test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Close file */ @@ -1944,7 +1949,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ char filename[FHEAP_FILENAME_LEN]; /* Filename to use */ @@ -1956,12 +1961,13 @@ test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED h5_stat_size_t file_size; /* File size, after deleting heap */ size_t id_len; /* Size of fractal heap IDs */ fheap_heap_state_t state; /* State of fractal heap */ + hbool_t page = FALSE; /* Paged aggregation strategy or not */ /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Close file */ @@ -1980,6 +1986,9 @@ test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED if(NULL == (f = (H5F_t *)H5I_object(file))) STACK_ERROR + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) + page = TRUE; + /* Ignore metadata tags in the file's cache */ if (H5AC_ignore_tags(f) < 0) FAIL_STACK_ERROR @@ -2058,8 +2067,9 @@ test_reopen(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED TEST_ERROR /* Verify the file is correct size */ - if(file_size != empty_size) - TEST_ERROR + if(!page || (page && !tparam->reopen_heap)) + if(file_size != empty_size) + TEST_ERROR /* All tests passed */ PASSED() @@ -2090,7 +2100,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ hid_t file2 = -1; /* File ID */ @@ -2105,12 +2115,13 @@ test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UN h5_stat_size_t file_size; /* File size, after deleting heap */ size_t id_len; /* Size of fractal heap IDs */ fheap_heap_state_t state; /* State of fractal heap */ + hbool_t page = FALSE; /* Paged aggregation strategy or not */ /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Close file */ @@ -2129,6 +2140,9 @@ test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UN if(NULL == (f = (H5F_t *)H5I_object(file))) STACK_ERROR + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) + page = TRUE; + /* Ignore metadata tags in the file's cache */ if (H5AC_ignore_tags(f) < 0) FAIL_STACK_ERROR @@ -2226,8 +2240,9 @@ test_open_twice(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UN TEST_ERROR /* Verify the file is correct size */ - if(file_size != empty_size) - TEST_ERROR + if(!page || (page && !tparam->reopen_heap)) + if(file_size != empty_size) + TEST_ERROR /* All tests passed */ PASSED() @@ -2262,7 +2277,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_delete_open(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_UNUSED *tparam) +test_delete_open(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) { hid_t file = -1; /* File ID */ char filename[FHEAP_FILENAME_LEN]; /* Filename to use */ @@ -2280,7 +2295,7 @@ test_delete_open(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t H5_ATTR_U h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Close file */ @@ -2433,7 +2448,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_id_limits(hid_t fapl, H5HF_create_t *cparam) +test_id_limits(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file = -1; /* File ID */ hid_t dxpl = H5AC_ind_read_dxpl_id; /* DXPL to use */ @@ -2451,7 +2466,7 @@ test_id_limits(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -2778,7 +2793,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_filtered_create(hid_t fapl, H5HF_create_t *cparam) +test_filtered_create(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file = -1; /* File ID */ hid_t dxpl = H5AC_ind_read_dxpl_id; /* DXPL to use */ @@ -2794,7 +2809,7 @@ test_filtered_create(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -2901,7 +2916,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_size(hid_t fapl, H5HF_create_t *cparam) +test_size(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file = -1; /* File ID */ hid_t dxpl = H5AC_ind_read_dxpl_id; /* DXPL to use */ @@ -2917,7 +2932,7 @@ test_size(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3045,7 +3060,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_reopen_hdr(hid_t fapl, H5HF_create_t *cparam) +test_reopen_hdr(hid_t fapl, H5HF_create_t *cparam, hid_t fcpl) { hid_t file1 = -1; /* File ID */ hid_t file2 = -2; /* File ID */ @@ -3060,7 +3075,7 @@ test_reopen_hdr(hid_t fapl, H5HF_create_t *cparam) h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file1 = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3199,7 +3214,7 @@ test_man_insert_weird(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3309,7 +3324,7 @@ test_man_insert_first(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -3410,7 +3425,7 @@ test_man_insert_second(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tp h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3507,7 +3522,7 @@ test_man_insert_root_mult(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3606,7 +3621,7 @@ test_man_insert_force_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test_par h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3712,7 +3727,7 @@ test_man_insert_fill_second(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3819,7 +3834,7 @@ test_man_insert_third_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_param h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -3930,7 +3945,7 @@ test_man_fill_first_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4026,7 +4041,7 @@ test_man_start_second_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4129,7 +4144,7 @@ test_man_fill_second_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4230,7 +4245,7 @@ test_man_start_third_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4341,7 +4356,7 @@ test_man_fill_fourth_row(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t * h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4438,7 +4453,7 @@ test_man_fill_all_root_direct(hid_t fapl, H5HF_create_t *cparam, fheap_test_para h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4534,7 +4549,7 @@ test_man_first_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4636,7 +4651,7 @@ test_man_second_direct_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fhe h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4746,7 +4761,7 @@ test_man_fill_first_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4849,7 +4864,7 @@ test_man_second_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_test h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -4960,7 +4975,7 @@ test_man_fill_second_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5068,7 +5083,7 @@ test_man_fill_recursive_indirect_row(hid_t fapl, H5HF_create_t *cparam, fheap_te h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5166,7 +5181,7 @@ test_man_start_2nd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5275,7 +5290,7 @@ test_man_recursive_indirect_two_deep(hid_t fapl, H5HF_create_t *cparam, fheap_te h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5378,7 +5393,7 @@ test_man_start_3rd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5488,7 +5503,7 @@ test_man_fill_first_3rd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fh h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5599,7 +5614,7 @@ test_man_fill_3rd_recursive_indirect_row(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5706,7 +5721,7 @@ test_man_fill_all_3rd_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5814,7 +5829,7 @@ test_man_start_4th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -5929,7 +5944,7 @@ test_man_fill_first_4th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fh h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6049,7 +6064,7 @@ test_man_fill_4th_recursive_indirect_row(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6161,7 +6176,7 @@ test_man_fill_all_4th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fhea h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6276,7 +6291,7 @@ test_man_start_5th_recursive_indirect(hid_t fapl, H5HF_create_t *cparam, fheap_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6410,7 +6425,7 @@ test_man_remove_bogus(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6563,7 +6578,7 @@ test_man_remove_one(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6727,7 +6742,7 @@ test_man_remove_two(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -6920,7 +6935,7 @@ test_man_remove_one_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -7089,7 +7104,7 @@ test_man_remove_two_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -7333,7 +7348,7 @@ test_man_remove_three_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) TEST_ERROR /* Get a pointer to the internal file object */ @@ -7632,7 +7647,7 @@ test_man_incr_insert_remove(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, tparam->my_fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -16353,6 +16368,8 @@ main(void) H5HF_create_t small_cparam; /* Creation parameters for "small" heap */ H5HF_create_t large_cparam; /* Creation parameters for "large" heap */ hid_t fapl = -1; /* File access property list for data files */ + hid_t fcpl = -1; /* File creation property list for data files */ + hid_t fcpl2 = -1; /* File creation property list for data files */ fheap_test_type_t curr_test; /* Current test being worked on */ unsigned u; /* Local index variable */ unsigned nerrors = 0; /* Cumulative error count */ @@ -16375,6 +16392,16 @@ main(void) shared_wobj_g = (unsigned char *)H5MM_malloc(shared_obj_size_g); shared_robj_g = (unsigned char *)H5MM_malloc(shared_obj_size_g); + /* create a file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + if((fcpl2 = H5Pcopy(fcpl)) < 0) TEST_ERROR + + /* Set file space strategy and persisting free-space */ + /* This will be modified later on to run the test with different file space strategy setting */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_FSM_AGGR, FALSE, (hsize_t)1) < 0) + TEST_ERROR + /* Initialize the shared write buffer for objects */ for(u = 0; u < shared_obj_size_g; u++) shared_wobj_g[u] = (unsigned char)u; @@ -16385,6 +16412,9 @@ main(void) HDmemset(&tparam, 0, sizeof(fheap_test_param_t)); tparam.actual_id_len = HEAP_ID_LEN; + /* This will be modified later on to run the test with different file space strategy setting */ + tparam.my_fcpl = fcpl2; + /* Set appropriate testing parameters for each test */ switch(curr_test) { /* "Normal" testing parameters */ @@ -16409,10 +16439,11 @@ main(void) nerrors += test_reopen(fapl, &small_cparam, &tparam); nerrors += test_open_twice(fapl, &small_cparam, &tparam); nerrors += test_delete_open(fapl, &small_cparam, &tparam); - nerrors += test_id_limits(fapl, &small_cparam); - nerrors += test_filtered_create(fapl, &small_cparam); - nerrors += test_size(fapl, &small_cparam); - nerrors += test_reopen_hdr(fapl, &small_cparam); + + nerrors += test_id_limits(fapl, &small_cparam, tparam.my_fcpl); + nerrors += test_filtered_create(fapl, &small_cparam, tparam.my_fcpl); + nerrors += test_size(fapl, &small_cparam, tparam.my_fcpl); + nerrors += test_reopen_hdr(fapl, &small_cparam, tparam.my_fcpl); { fheap_test_fill_t fill; /* Size of objects to fill heap blocks with */ @@ -16732,6 +16763,9 @@ main(void) H5MM_xfree(shared_lens_g); H5MM_xfree(shared_offs_g); + if(H5Pclose(fcpl) < 0) TEST_ERROR + if(H5Pclose(fcpl2) < 0) TEST_ERROR + /* Clean up file used */ #ifndef QAK h5_cleanup(FILENAME, fapl); diff --git a/test/filespace_1_8.h5 b/test/filespace_1_8.h5 Binary files differindex 85138b0..3fa2822 100644 --- a/test/filespace_1_8.h5 +++ b/test/filespace_1_8.h5 diff --git a/test/freespace.c b/test/freespace.c index 181e6a1..2bf9111 100644 --- a/test/freespace.c +++ b/test/freespace.c @@ -86,7 +86,7 @@ typedef struct TEST_free_section_t { static herr_t TEST_sect_init_cls(H5FS_section_class_t *, void *); static herr_t TEST_sect_free(H5FS_section_info_t *_sect); static herr_t TEST_sect_can_merge(const H5FS_section_info_t *, const H5FS_section_info_t *, void H5_ATTR_UNUSED *); -static herr_t TEST_sect_merging(H5FS_section_info_t *, H5FS_section_info_t *, void H5_ATTR_UNUSED *); +static herr_t TEST_sect_merging(H5FS_section_info_t **, H5FS_section_info_t *, void H5_ATTR_UNUSED *); static herr_t TEST_sect_can_shrink(const H5FS_section_info_t *, void *); static herr_t TEST_sect_shrinking(H5FS_section_info_t **, void *); @@ -245,26 +245,26 @@ TEST_sect_can_merge(const H5FS_section_info_t *_sect1, * Merge the two sections (second section is merged into the first section) */ static herr_t -TEST_sect_merging(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +TEST_sect_merging(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) { - TEST_free_section_t *sect1 = (TEST_free_section_t *)_sect1; + TEST_free_section_t **sect1 = (TEST_free_section_t **)_sect1; TEST_free_section_t *sect2 = (TEST_free_section_t *)_sect2; herr_t ret_value = SUCCEED; /* Return value */ /* Check arguments. */ HDassert(sect1); - HDassert((sect1->sect_info.type == TEST_FSPACE_SECT_TYPE) || - (sect1->sect_info.type == TEST_FSPACE_SECT_TYPE_NEW) || - (sect1->sect_info.type == TEST_FSPACE_SECT_TYPE_NONE)); + HDassert(((*sect1)->sect_info.type == TEST_FSPACE_SECT_TYPE) || + ((*sect1)->sect_info.type == TEST_FSPACE_SECT_TYPE_NEW) || + ((*sect1)->sect_info.type == TEST_FSPACE_SECT_TYPE_NONE)); HDassert(sect2); HDassert((sect2->sect_info.type == TEST_FSPACE_SECT_TYPE) || (sect2->sect_info.type == TEST_FSPACE_SECT_TYPE_NEW) || (sect2->sect_info.type == TEST_FSPACE_SECT_TYPE_NONE)); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ if(TEST_sect_free((H5FS_section_info_t *)sect2) < 0) @@ -2540,7 +2540,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend a block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_SIZE80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE40)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_SIZE80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE40, 0, NULL)) < 0) FAIL_STACK_ERROR if(FALSE == status) TEST_ERROR @@ -2616,7 +2616,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend the block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE50)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE50, 0, NULL)) < 0) FAIL_STACK_ERROR if(TRUE == status) TEST_ERROR @@ -2689,7 +2689,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend the block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE30)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE20, (hsize_t)TEST_SECT_SIZE30, 0, NULL)) < 0) TEST_ERROR if(FALSE == status) TEST_ERROR @@ -2763,7 +2763,7 @@ test_fs_sect_extend(hid_t fapl) TEST_ERROR /* Extend the block by requested-size */ - if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE15, (hsize_t)TEST_SECT_SIZE40)) < 0) + if((status = H5FS_sect_try_extend(f, dxpl_id, frsp, (haddr_t)TEST_SECT_ADDR80, (hsize_t)TEST_SECT_SIZE15, (hsize_t)TEST_SECT_SIZE40, 0, NULL)) < 0) TEST_ERROR if(TRUE == status) TEST_ERROR diff --git a/test/fsm_aggr_nopersist.h5 b/test/fsm_aggr_nopersist.h5 Binary files differnew file mode 100644 index 0000000..159e7f7 --- /dev/null +++ b/test/fsm_aggr_nopersist.h5 diff --git a/test/fsm_aggr_persist.h5 b/test/fsm_aggr_persist.h5 Binary files differnew file mode 100644 index 0000000..1a837dd --- /dev/null +++ b/test/fsm_aggr_persist.h5 diff --git a/test/gen_filespace.c b/test/gen_filespace.c index e0c42e8..52715ae 100644 --- a/test/gen_filespace.c +++ b/test/gen_filespace.c @@ -14,69 +14,105 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "hdf5.h" +#include <assert.h> #define NELMTS(X) (sizeof(X)/sizeof(X[0])) /* # of elements */ -#define TEST_THRESHOLD2 2 /* Free space section threshold */ const char *FILENAMES[] = { - "filespace_persist.h5", /* H5F_FILE_SPACE_ALL_PERSIST */ - "filespace_default.h5", /* H5F_FILE_SPACE_ALL */ - "filespace_aggr_vfd.h5", /* H5F_FILE_SPACE_AGGR_VFD */ - "filespace_vfd.h5", /* H5F_FILE_SPACE_VFD */ - "filespace_threshold.h5" /* H5F_FILE_SPACE_ALL, non-default threshold */ + "fsm_aggr_nopersist.h5", /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ + "fsm_aggr_persist.h5", /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ + "paged_nopersist.h5", /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ + "paged_persist.h5", /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ + "aggr.h5", /* H5F_FSPACE_STRATEGY_AGGR */ + "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ }; #define DATASET "dset" #define NUM_ELMTS 100 +#define FALSE 0 +#define TRUE 1 +#define INC_ENUM(TYPE,VAR) (VAR)=((TYPE)((VAR)+1)) /* - * Compile and run this program in file-space branch to generate - * HDF5 files with different kinds of file space strategies - * Move the HDF5 files to the 1.6 and 1.8 branch for compatibility - * testing:test_filespace_compatible() will use the files + * Compile and run this program in the trunk to generate + * HDF5 files with combinations of 4 file space strategies + * and persist/not persist free-space. + * The library creates the file space info message with "mark if unknown" + * in these files. + * + * Move these files to 1.8 branch for compatibility testing: + * test_filespace_compatible() in test/tfile.c will use these files. + * + * Copy these files from the 1.8 branch back to the trunk for + * compatibility testing via test_filespace_round_compatible() in test/tfile.c. + * */ -static void gen_file(void) -{ - hid_t fid; - hid_t fcpl; - hid_t dataset, space; - hsize_t dim[1]; - int data[NUM_ELMTS]; - size_t j; /* Local index variable */ - int i; /* Local index variable */ - H5F_file_space_type_t fs_type; /* File space handling strategy */ - - for(j = 0, fs_type = H5F_FILE_SPACE_ALL_PERSIST; j < NELMTS(FILENAMES); j++, fs_type = (H5F_file_space_type_t)(fs_type + 1)) { - /* Get a copy of the default file creation property */ - fcpl = H5Pcreate(H5P_FILE_CREATE); - - if(fs_type == H5F_FILE_SPACE_NTYPES) /* last file */ - /* Set default strategy but non-default threshold */ - H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL, (hsize_t)TEST_THRESHOLD2); - else - /* Set specified file space strategy and free space section threshold */ - H5Pset_file_space(fcpl, fs_type, (hsize_t)0); - - /* Create the file with the file space info */ - fid = H5Fcreate(FILENAMES[j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT); - - dim[0] = NUM_ELMTS; - space = H5Screate_simple(1, dim, NULL); - dataset = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - - for(i = 0; i < NUM_ELMTS; i++) - data[i] = i; - - H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); - H5Dclose(dataset); - H5Sclose(space); - H5Fclose(fid); - } -} - int main(void) { - gen_file(); + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t did = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hsize_t dim[1]; /* Dimension sizes */ + int data[NUM_ELMTS]; /* Buffer for data */ + int i, j; /* Local index variables */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ + unsigned fs_persist; /* Persisting free-space or not */ + + j = 0; + for(fs_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_strategy < H5F_FSPACE_STRATEGY_NTYPES; INC_ENUM(H5F_fspace_strategy_t, fs_strategy)) { + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + + if(fs_persist && fs_strategy >= H5F_FSPACE_STRATEGY_AGGR) + continue; + + /* Get a copy of the default file creation property */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + + if(H5Pset_file_space_strategy(fcpl, fs_strategy, fs_persist, (hsize_t)1) < 0) + goto error; + + /* Create the file with the file space info */ + if((fid = H5Fcreate(FILENAMES[j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Create the dataset */ + dim[0] = NUM_ELMTS; + if((sid = H5Screate_simple(1, dim, NULL)) < 0) + goto error; + if((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + for(i = 0; i < NUM_ELMTS; i++) + data[i] = i; + + /* Write the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + goto error; + + /* Closing */ + if(H5Dclose(did) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Fclose(fid) < 0) + goto error; + if(H5Pclose(fcpl) < 0) + goto error; + ++j; + } + } + assert(j == NELMTS(FILENAMES)); return 0; + +error: + H5E_BEGIN_TRY { + H5Sclose(sid); + H5Sclose(did); + H5Pclose(fcpl); + H5Fclose(fid); + } H5E_END_TRY; + return -1; } diff --git a/test/gen_idx.c b/test/gen_idx.c deleted file mode 100644 index 8c24198..0000000 --- a/test/gen_idx.c +++ /dev/null @@ -1,126 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the files COPYING and Copyright.html. COPYING can be found at the root * - * of the source code distribution tree; Copyright.html can be found at the * - * root level of an installed copy of the electronic HDF5 document set and * - * is linked from the top-level documents page. It can also be found at * - * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * - * access to either file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Purpose: This program is run to generate an HDF5 data file with datasets - * that use Fixed Array indexing method. - * - * To test compatibility, compile and run this program - * which will generate a file called "fixed_idx.h5". - * Move it to the test directory in the HDF5 v1.6/1.8 source tree. - * The test: test_idx_compatible() in dsets.c will read it. - */ -#include <assert.h> -#include "hdf5.h" - -const char *FILENAME[1] = { - "fixed_idx.h5" /* file with datasets that use Fixed Array indexing method */ -}; - -#define DSET "dset" -#define DSET_FILTER "dset_filter" - -/* - * Function: gen_idx_file - * - * Purpose: Create a file with datasets that use Fixed Array indexing: - * one dataset: fixed dimension, chunked layout, w/o filters - * one dataset: fixed dimension, chunked layout, w/ filters - * - */ -static void gen_idx_file(void) -{ - hid_t fapl; /* file access property id */ - hid_t fid; /* file id */ - hid_t sid; /* space id */ - hid_t dcpl; /* dataset creation property id */ - hid_t did, did2; /* dataset id */ - hsize_t dims[1] = {10}; /* dataset dimension */ - hsize_t c_dims[1] = {2}; /* chunk dimension */ - herr_t status; /* return status */ - int i; /* local index variable */ - int buf[10]; /* data buffer */ - - - /* Get a copy of the file aaccess property */ - fapl = H5Pcreate(H5P_FILE_ACCESS); - assert(fapl >= 0); - - /* Set the "use the latest format" bounds for creating objects in the file */ - status = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); - assert(status >= 0); - - /* Create dataset */ - fid = H5Fcreate(FILENAME[0], H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - assert(fid >= 0); - - /* Create data */ - for(i = 0; i < 10; i++) - buf[i] = i; - - /* Set chunk */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - assert(dcpl >= 0); - status = H5Pset_chunk(dcpl, 1, c_dims); - assert(status >= 0); - - sid = H5Screate_simple(1, dims, NULL); - assert(sid >= 0); - - /* Create a 1D dataset */ - did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - assert(did >= 0); - - /* Write to the dataset */ - status = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); - assert(status >= 0); - -#if defined (H5_HAVE_FILTER_DEFLATE) - /* set deflate data */ - status = H5Pset_deflate(dcpl, 9); - assert(status >= 0); - - /* Create and write the dataset */ - did2 = H5Dcreate2(fid, DSET_FILTER, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - assert(did2 >= 0); - - status = H5Dwrite(did2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); - assert(status >= 0); - - /* Close the dataset */ - status = H5Dclose(did2); - assert(status >= 0); -#endif - - /* closing */ - status = H5Dclose(did); - assert(status >= 0); - status = H5Sclose(sid); - assert(status >= 0); - status = H5Pclose(dcpl); - assert(status >= 0); - status = H5Pclose(fapl); - assert(status >= 0); - status = H5Fclose(fid); - assert(status >= 0); -} /* gen_idx_file() */ - -int main(void) -{ - gen_idx_file(); - - return 0; -} - diff --git a/test/gen_plist.c b/test/gen_plist.c index 8cb6c00..5c54ce8 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -403,6 +403,12 @@ main(void) if((ret = H5Pset_sizes(fcpl1, 8, 4) < 0)) assert(ret > 0); + if((ret = H5Pset_file_space_strategy(fcpl1, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1)) < 0) + assert(ret > 0); + + if((ret = H5Pset_file_space_page_size(fcpl1, (hsize_t)4096)) < 0) + assert(ret > 0); + if((ret = encode_plist(fcpl1, little_endian, word_length, "testfiles/plist_files/fcpl_")) < 0) assert(ret > 0); diff --git a/test/h5fc_ext1_f.h5 b/test/h5fc_ext1_f.h5 Binary files differnew file mode 100644 index 0000000..b5c5867 --- /dev/null +++ b/test/h5fc_ext1_f.h5 diff --git a/test/h5fc_ext1_i.h5 b/test/h5fc_ext1_i.h5 Binary files differnew file mode 100644 index 0000000..960a8d5 --- /dev/null +++ b/test/h5fc_ext1_i.h5 diff --git a/test/h5fc_ext2_if.h5 b/test/h5fc_ext2_if.h5 Binary files differnew file mode 100644 index 0000000..88e42e7 --- /dev/null +++ b/test/h5fc_ext2_if.h5 diff --git a/test/h5fc_ext2_sf.h5 b/test/h5fc_ext2_sf.h5 Binary files differnew file mode 100644 index 0000000..c59a3ca --- /dev/null +++ b/test/h5fc_ext2_sf.h5 diff --git a/test/h5fc_ext3_isf.h5 b/test/h5fc_ext3_isf.h5 Binary files differnew file mode 100644 index 0000000..d00fc55 --- /dev/null +++ b/test/h5fc_ext3_isf.h5 diff --git a/test/h5fc_ext_none.h5 b/test/h5fc_ext_none.h5 Binary files differnew file mode 100644 index 0000000..b1b1553 --- /dev/null +++ b/test/h5fc_ext_none.h5 diff --git a/test/links.c b/test/links.c index 3364c7e..e3bbdb2 100644 --- a/test/links.c +++ b/test/links.c @@ -11102,6 +11102,8 @@ corder_delete(hid_t fapl) for(reopen_file = FALSE; reopen_file <= TRUE; reopen_file++) { /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + + /* Creating file with latest format will enable paged aggregation with persistent fs */ if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR /* Close file */ @@ -20,6 +20,7 @@ * test_mf_aggr_*() tests for file memory that interact with the aggregators * test_mf_align_*() tests for file memory with alignment setting * test_filespace_*() tests for file space management + * test_page_*() tests for file space paging */ #include "h5test.h" @@ -38,41 +39,63 @@ #include "H5Iprivate.h" #include "H5VMprivate.h" -#define FILENAME_LEN 1024 - -#define TEST_BLOCK_SIZE1 1 -#define TEST_BLOCK_SIZE2 2 -#define TEST_BLOCK_SIZE3 3 -#define TEST_BLOCK_SIZE4 4 -#define TEST_BLOCK_SIZE5 5 -#define TEST_BLOCK_SIZE6 6 -#define TEST_BLOCK_SIZE7 7 -#define TEST_BLOCK_SIZE8 8 -#define TEST_BLOCK_SIZE20 20 -#define TEST_BLOCK_SIZE30 30 -#define TEST_BLOCK_SIZE40 40 -#define TEST_BLOCK_SIZE50 50 -#define TEST_BLOCK_SIZE80 80 -#define TEST_BLOCK_SIZE200 200 -#define TEST_BLOCK_SIZE600 600 -#define TEST_BLOCK_SIZE700 700 -#define TEST_BLOCK_SIZE1034 1034 -#define TEST_BLOCK_SIZE1970 1970 -#define TEST_BLOCK_SIZE2058 2058 -#define TEST_BLOCK_SIZE8000 8000 -#define TEST_BLOCK_SIZE2048 2048 - -#define TEST_BLOCK_ADDR70 70 -#define TEST_BLOCK_ADDR100 100 - -#define TEST_ALIGN1024 1024 -#define TEST_ALIGN4096 4096 - -#define TEST_THRESHOLD10 10 -#define TEST_THRESHOLD3 3 - -#define CORE_INCREMENT 1024 -#define FAMILY_SIZE 1024 +#define FILENAME_LEN 1024 + +#define TBLOCK_SIZE1 1 +#define TBLOCK_SIZE2 2 +#define TBLOCK_SIZE3 3 +#define TBLOCK_SIZE4 4 +#define TBLOCK_SIZE5 5 +#define TBLOCK_SIZE6 6 +#define TBLOCK_SIZE7 7 +#define TBLOCK_SIZE8 8 +#define TBLOCK_SIZE10 10 +#define TBLOCK_SIZE11 11 +#define TBLOCK_SIZE20 20 +#define TBLOCK_SIZE30 30 +#define TBLOCK_SIZE36 36 +#define TBLOCK_SIZE40 40 +#define TBLOCK_SIZE50 50 +#define TBLOCK_SIZE80 80 +#define TBLOCK_SIZE90 90 +#define TBLOCK_SIZE98 98 +#define TBLOCK_SIZE100 100 +#define TBLOCK_SIZE150 150 +#define TBLOCK_SIZE200 200 +#define TBLOCK_SIZE600 600 +#define TBLOCK_SIZE700 700 +#define TBLOCK_SIZE1034 1034 +#define TBLOCK_SIZE1970 1970 +#define TBLOCK_SIZE2048 2048 +#define TBLOCK_SIZE2058 2058 +#define TBLOCK_SIZE2192 2192 +#define TBLOCK_SIZE3080 3080 +#define TBLOCK_SIZE3088 3088 +#define TBLOCK_SIZE3198 3198 +#define TBLOCK_SIZE3286 3286 +#define TBLOCK_SIZE3248 3248 +#define TBLOCK_SIZE3900 3900 +#define TBLOCK_SIZE4020 4020 +#define TBLOCK_SIZE4086 4086 +#define TBLOCK_SIZE4096 4096 +#define TBLOCK_SIZE4106 4106 +#define TBLOCK_SIZE5000 5000 +#define TBLOCK_SIZE6000 6000 +#define TBLOCK_SIZE8000 8000 +#define TBLOCK_SIZE8100 8100 +#define TBLOCK_SIZE8192 8192 +#define TBLOCK_SIZE8190 8190 +#define TBLOCK_SIZE12000 12000 + +#define TBLOCK_ADDR70 70 +#define TBLOCK_ADDR100 100 + +#define TEST_ALIGN16 16 +#define TEST_ALIGN1024 1024 +#define TEST_ALIGN4096 4096 + +#define TEST_THRESHOLD10 10 +#define TEST_THRESHOLD3 3 const char *FILENAME[] = { "mf", @@ -80,24 +103,17 @@ const char *FILENAME[] = { }; typedef enum { - TEST_NORMAL, /* size of aggregator is >= alignment size */ - TEST_AGGR_SMALL, /* size of aggregator is smaller than alignment size */ - TEST_NTESTS /* The number of test types, must be last */ + TEST_NORMAL, /* size of aggregator is >= alignment size */ + TEST_AGGR_SMALL, /* size of aggregator is smaller than alignment size */ + TEST_NTESTS /* The number of test types, must be last */ } test_type_t; -typedef struct frspace_state_t { - hsize_t tot_space; /* Total amount of space tracked */ - hsize_t tot_sect_count; /* Total # of sections tracked */ - hsize_t serial_sect_count; /* # of serializable sections tracked */ - hsize_t ghost_sect_count; /* # of un-serializable sections tracked */ -} frspace_state_t; +static int check_stats(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *state); - -static int check_stats(const H5F_t *, const H5FS_t *, frspace_state_t *); static unsigned test_mf_eoa(const char *env_h5_drvr, hid_t fapl); static unsigned test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl); static unsigned test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl); -static unsigned test_mf_tmp(const char *env_h5_drvr, hid_t fapl); +static unsigned test_dichotomy(hid_t fapl); static unsigned test_mf_fs_start(hid_t fapl); static unsigned test_mf_fs_alloc_free(hid_t fapl); static unsigned test_mf_fs_extend(hid_t fapl); @@ -119,21 +135,27 @@ static unsigned test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t static unsigned test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl); static unsigned test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl); static unsigned test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl); -static unsigned test_mf_fs_persist(hid_t fapl_new, hid_t fcpl); -static unsigned test_mf_fs_gone(hid_t fapl_new, hid_t fcpl); -static unsigned test_mf_fs_split(hid_t fapl_new, hid_t fcpl); -static unsigned test_mf_fs_multi(hid_t fapl, hid_t fcpl); -static unsigned test_mf_fs_drivers(hid_t fapl); +static unsigned test_mf_tmp(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_strat_thres_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_fs_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_strat_thres_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format); +static unsigned test_mf_fs_persist_split(void); +static unsigned test_mf_fs_persist_multi(void); +static unsigned test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_small(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_large(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_large_try_extend(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_small_try_extend(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_try_shrink(const char *env_h5_drvr, hid_t fapl); +static unsigned test_page_alignment(const char *env_h5_drvr, hid_t fapl); /* * Verify statistics for the free-space manager * - * Modifications: - * Vailin Choi; July 2012 - * To ensure "f" and "frsp" are valid pointers */ static int -check_stats(const H5F_t *f, const H5FS_t *frsp, frspace_state_t *state) +check_stats(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *state) { H5FS_stat_t frspace_stats; /* Statistics about the heap */ @@ -146,22 +168,22 @@ check_stats(const H5F_t *f, const H5FS_t *frsp, frspace_state_t *state) if(frspace_stats.tot_space != state->tot_space) { HDfprintf(stdout, "frspace_stats.tot_space = %Hu, state->tot_space = %Zu\n", - frspace_stats.tot_space, state->tot_space); + frspace_stats.tot_space, state->tot_space); TEST_ERROR } /* end if */ if(frspace_stats.tot_sect_count != state->tot_sect_count) { HDfprintf(stdout, "frspace_stats.tot_sect_count = %Hu, state->tot_sect_count = %Hu\n", - frspace_stats.tot_sect_count, state->tot_sect_count); + frspace_stats.tot_sect_count, state->tot_sect_count); TEST_ERROR } /* end if */ if(frspace_stats.serial_sect_count != state->serial_sect_count) { HDfprintf(stdout, "frspace_stats.serial_sect_count = %Hu, state->serial_sect_count = %Hu\n", - frspace_stats.serial_sect_count, state->serial_sect_count); + frspace_stats.serial_sect_count, state->serial_sect_count); TEST_ERROR } /* end if */ if(frspace_stats.ghost_sect_count != state->ghost_sect_count) { HDfprintf(stdout, "frspace_stats.ghost_sect_count = %Hu, state->ghost_sect_count = %Hu\n", - frspace_stats.ghost_sect_count, state->ghost_sect_count); + frspace_stats.ghost_sect_count, state->ghost_sect_count); TEST_ERROR } /* end if */ @@ -173,13 +195,15 @@ error: } /* check_stats() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from file allocation * * Set up: - * Turn off using meta/small data aggregator - * There is nothing in free-space manager + * Turn off using meta/small data aggregator + * There is nothing in free-space manager * * Allocate two blocks which should be from file allocation + *------------------------------------------------------------------------- */ static unsigned test_mf_eoa(const char *env_h5_drvr, hid_t fapl) @@ -234,7 +258,7 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* nothing should be changed in meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &ma_size); @@ -244,7 +268,7 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) if (addr1 < (haddr_t)file_size) TEST_ERROR - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* nothing should be changed in meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &ma_size); @@ -262,7 +286,7 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)) + if (new_file_size != (file_size+TBLOCK_SIZE30+TBLOCK_SIZE50)) TEST_ERROR /* Re-open the file */ @@ -273,8 +297,8 @@ test_mf_eoa(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -308,6 +332,7 @@ error: } /* test_mf_eoa() */ /* + *------------------------------------------------------------------------- * To verify that an allocated block from file allocation is shrunk. * * Set up: @@ -323,6 +348,7 @@ error: * Test 4: Allocate a block of 30 from file allocation * H5MF_try_shrink() the block by 20 from the end: succeed * + *------------------------------------------------------------------------- */ static unsigned test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) @@ -377,7 +403,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if (addr < (haddr_t)file_size) TEST_ERROR @@ -395,7 +421,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30)) + if (new_file_size != (file_size+TBLOCK_SIZE30)) TEST_ERROR /* Re-open the file */ @@ -409,7 +435,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* should succeed */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30) <= 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -453,13 +479,13 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if (addr < (haddr_t)file_size) TEST_ERROR /* should not succeed in shrinking */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30 - 10) > 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30 - 10) > 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -475,7 +501,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30)) + if(new_file_size != (file_size + TBLOCK_SIZE30)) TEST_ERROR PASSED() @@ -503,7 +529,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* should not succeed in shrinking */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30 + 10) > 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30 + 10) > 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -519,7 +545,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30)) + if(new_file_size != (file_size + TBLOCK_SIZE30)) TEST_ERROR PASSED() @@ -546,7 +572,7 @@ test_mf_eoa_shrink(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* should succeed in shrinking */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr+10, (hsize_t)(TEST_BLOCK_SIZE30 - 10)) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr+10, (hsize_t)(TBLOCK_SIZE30 - 10)) <= 0) TEST_ERROR /* nothing should be changed in meta_aggr */ @@ -588,6 +614,7 @@ error: } /* test_mf_eoa_shrink() */ /* + *------------------------------------------------------------------------- * To verify that an allocated block from file allocation is extended. * * Set up: @@ -599,6 +626,7 @@ error: * * Test 2: Allocate a block of 30 * H5MF_try_extend() the block of size 20 by 50: fail + *------------------------------------------------------------------------- */ static unsigned test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) @@ -656,7 +684,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if (addr < (haddr_t)file_size) TEST_ERROR @@ -673,7 +701,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30)) + if(new_file_size != (file_size + TBLOCK_SIZE30)) TEST_ERROR /* Re-open the file */ @@ -685,7 +713,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) FAIL_STACK_ERROR /* should succeed */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)TEST_BLOCK_SIZE50); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)TBLOCK_SIZE30, (hsize_t)TBLOCK_SIZE50); if(was_extended <= 0) TEST_ERROR @@ -703,7 +731,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != (file_size + TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50)) + if(new_file_size != (file_size + TBLOCK_SIZE30 + TBLOCK_SIZE50)) TEST_ERROR PASSED() @@ -734,7 +762,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); if(addr < (haddr_t)file_size) TEST_ERROR @@ -744,7 +772,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) if(new_ma_addr != ma_addr) TEST_ERROR - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)(TEST_BLOCK_SIZE30-10), (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)(TBLOCK_SIZE30-10), (hsize_t)(TBLOCK_SIZE50)); /* should not succeed */ if(was_extended > 0) @@ -763,7 +791,7 @@ test_mf_eoa_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if(new_file_size != file_size + TEST_BLOCK_SIZE30) + if(new_file_size != file_size + TBLOCK_SIZE30) TEST_ERROR if(H5Pclose(fapl_new) < 0) @@ -787,6 +815,7 @@ error: } /* test_mf_eoa_extend() */ /* + *------------------------------------------------------------------------- * To verify that temporary blocks are allocated correctly * * Set up: @@ -806,13 +835,19 @@ error: * space fails * - Check that allocating another 1/2 of the file as normal address * space fails + *------------------------------------------------------------------------- */ static unsigned -test_mf_tmp(const char *env_h5_drvr, hid_t fapl) +test_mf_tmp(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t file = -1; /* File ID */ + hid_t file = -1; /* File ID */ + hid_t fapl2 = -1; /* File access property list */ + hid_t fcpl = -1; /* File creation property list */ - TESTING("'temporary' file space allocation"); + if(new_format) + TESTING("'temporary' file space allocation with new library format") + else + TESTING("'temporary' file space allocation with old library format") /* Can't run this test with multi-file VFDs */ if(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi") && HDstrcmp(env_h5_drvr, "family")) { @@ -829,8 +864,22 @@ test_mf_tmp(const char *env_h5_drvr, hid_t fapl) /* Set the filename to use for this test */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + if(new_format) { + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR + + /* Set the "use the latest version of the format" bounds for creating objects in the file */ + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + } /* end if */ + /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, new_format?fapl2:fapl)) < 0) FAIL_STACK_ERROR /* Close file */ @@ -855,13 +904,13 @@ test_mf_tmp(const char *env_h5_drvr, hid_t fapl) FAIL_STACK_ERROR /* Allocate some temporary address space */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc_tmp(f, (hsize_t)TEST_BLOCK_SIZE30))) + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc_tmp(f, (hsize_t)TBLOCK_SIZE30))) FAIL_STACK_ERROR /* Check if temporary file address is valid */ if(!H5F_IS_TMP_ADDR(f, tmp_addr)) TEST_ERROR - if(tmp_addr < (haddr_t)(maxaddr - TEST_BLOCK_SIZE30)) + if(tmp_addr < (haddr_t)(maxaddr - TBLOCK_SIZE30)) TEST_ERROR /* Reading & writing with a temporary address value should fail */ @@ -878,7 +927,7 @@ test_mf_tmp(const char *env_h5_drvr, hid_t fapl) /* Freeing a temporary address value should fail */ H5E_BEGIN_TRY { - status = H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, tmp_addr, (hsize_t)TEST_BLOCK_SIZE30); + status = H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, tmp_addr, (hsize_t)TBLOCK_SIZE30); } H5E_END_TRY; if(status >= 0) TEST_ERROR @@ -963,10 +1012,12 @@ error: } /* test_mf_tmp() */ /* + *------------------------------------------------------------------------- * To verify that the free-space manager is created or opened * * Set up: * Turn off using meta/small data aggregator + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_start(hid_t fapl) @@ -976,11 +1027,10 @@ test_mf_fs_start(hid_t fapl) char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ h5_stat_size_t file_size, new_file_size; /* file size */ - H5FD_mem_t type; - frspace_state_t state; + H5FS_stat_t state; - TESTING("H5MF_alloc_create()/H5MF__alloc_open() of free-space manager"); + TESTING("H5MF_create_fstype()/H5MF_open_fstype() of free-space manager"); /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); @@ -1011,20 +1061,18 @@ test_mf_fs_start(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - /* Start up free-space manager */ - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR - - HDmemset(&state, 0, sizeof(frspace_state_t)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -1055,6 +1103,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that a block is allocated/freed from/to the free-space manager * * Set up: @@ -1078,10 +1127,7 @@ error: * The block is allocated from file allocation * Deallocate the block which will be returned to free-space manager * (the space is shrunk and freed since it is at end of file) - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_alloc_free(hid_t fapl) @@ -1091,12 +1137,10 @@ test_mf_fs_alloc_free(hid_t fapl) char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ h5_stat_size_t file_size, new_file_size; /* file size */ - H5FD_mem_t type; H5MF_free_section_t *sect_node = NULL; - haddr_t addr; - frspace_state_t state; - H5MF_sect_ud_t udata; - H5FS_section_info_t *node; + haddr_t addr; + haddr_t tmp; + H5FS_stat_t state; TESTING("H5MF_alloc()/H5MF_xfree() of free-space manager:test 1"); @@ -1129,69 +1173,59 @@ test_mf_fs_alloc_free(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space */ - if (addr != TEST_BLOCK_ADDR70) + if(addr != TBLOCK_ADDR70) TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); - state.tot_space += TEST_BLOCK_SIZE30; + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove section A from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE30, (H5FS_section_info_t **)&node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != TBLOCK_ADDR70) + TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1216,67 +1250,57 @@ test_mf_fs_alloc_free(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); - /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add section A to free-space manager */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 20 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE20)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE20)); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) + if(addr != TBLOCK_ADDR70) TEST_ERROR /* should still have 1 section of size 10 left in free-space manager */ - state.tot_space -= (TEST_BLOCK_SIZE20); + state.tot_space -= (TBLOCK_SIZE20); - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space manager */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE20)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE20)); /* Still 1 section in free-space because of merging */ - state.tot_space += TEST_BLOCK_SIZE20; - if(check_stats(f, f->shared->fs_man[type], &state)) + state.tot_space += TBLOCK_SIZE20; + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR - /* Remove section A from free-space manager */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE30, (H5FS_section_info_t **)&node) < 0) - FAIL_STACK_ERROR + /* Remove section A from free-space */ + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != TBLOCK_ADDR70) + TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1301,36 +1325,28 @@ test_mf_fs_alloc_free(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* @@ -1338,38 +1354,37 @@ test_mf_fs_alloc_free(hid_t fapl) * Since free-space manager cannot fulfull the request, * the block is obtained from file allocation */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE40)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE40)); /* Verify that the allocated block is not section A in free-space */ - if (addr == TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr == TBLOCK_ADDR70) + TEST_ERROR /* free-space info should be the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove section A from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE30, (H5FS_section_info_t **)&node) < 0) - FAIL_STACK_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + /* Verify that the block is section A in free-space */ + if(tmp != TBLOCK_ADDR70) + TEST_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block of size 40 to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE40)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE40)); /* * Free-space info is the same. * The block is returned to free-space. * It is shrunk and freed because it is at end of file. */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -1400,6 +1415,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that a block allocated from the free-space manager can be extended * * Set up: @@ -1435,10 +1451,7 @@ error: * Try to extend the allocated block by 50 from the free-space_manager: * Fail: section A does not adjoin section B (70+20 != address of section B) even though * the requested-size (50) equal to size of section B (50) - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_extend(hid_t fapl) @@ -1448,13 +1461,11 @@ test_mf_fs_extend(hid_t fapl) char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ h5_stat_size_t file_size, new_file_size; /* file size */ - H5FD_mem_t type; H5MF_free_section_t *sect_node1 = NULL, *sect_node2=NULL; - haddr_t addr; - frspace_state_t state; /* State of free space*/ - H5MF_sect_ud_t udata; - H5FS_section_info_t *node; - htri_t was_extended; + haddr_t addr; + haddr_t tmp; + H5FS_stat_t state; /* State of free space*/ + htri_t was_extended; TESTING("H5MF_try_extend() of free-space manager:test 1"); @@ -1487,101 +1498,91 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)TEST_BLOCK_SIZE50); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30, (hsize_t)TBLOCK_SIZE50); /* should succeed */ if(was_extended <= 0) TEST_ERROR /* Section B is removed from free-space manager */ - state.tot_space -= TEST_BLOCK_SIZE50; + state.tot_space -= TBLOCK_SIZE50; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the extended block to free-space manager */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50)); /* Verify that the extended block is back into free-space */ - state.tot_space += TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50; + state.tot_space += (TBLOCK_SIZE30+TBLOCK_SIZE50); state.tot_sect_count = 1; state.serial_sect_count = 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove the extended block */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR + if(tmp != TBLOCK_ADDR70) + TEST_ERROR + if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1605,95 +1606,84 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); - /* Add section A to free-space */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add section A to free-space manager */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)(TEST_BLOCK_SIZE50+10)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30, (hsize_t)(TBLOCK_SIZE50+10)); /* Should not be able to extend the allocated block */ if(was_extended) TEST_ERROR /* free-space info should remain the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the allocated block A to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); /* the returned section A is merged with section B in free-space */ /* rest of the info remains the same */ - state.tot_space += TEST_BLOCK_SIZE30; + state.tot_space += TBLOCK_SIZE30; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove the merged sections A & B from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != addr) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1718,95 +1708,84 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30); + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30); - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; - - /* Add section A to free-space */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add section A to free-space manager */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE30; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE30; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 30 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE30; + state.tot_space -= TBLOCK_SIZE30; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE30, (hsize_t)(TEST_BLOCK_SIZE40)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE30, (hsize_t)(TBLOCK_SIZE40)); /* Should succeed in extending the allocated block */ if(was_extended <=0) TEST_ERROR /* Should have 1 section of size=10 left in free-space manager */ - state.tot_space -= (TEST_BLOCK_SIZE40); - if(check_stats(f, f->shared->fs_man[type], &state)) + state.tot_space -= (TBLOCK_SIZE40); + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the extended block */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE40)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE40)); /* rest info is same, the extended section returned is merged with the section in free-space */ - state.tot_space += (TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE40); + state.tot_space += (TBLOCK_SIZE30+TBLOCK_SIZE40); - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove the merged sections A & B from free-space */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR + + if(tmp != addr) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1831,104 +1810,88 @@ test_mf_fs_extend(hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR /* Create section A */ - sect_node1 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)(TEST_BLOCK_SIZE30-10)); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node1 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)(TBLOCK_SIZE30-10)); /* Add section A of size=20 to free-space */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node1, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node1)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += (TEST_BLOCK_SIZE30-10); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += (TBLOCK_SIZE30-10); state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of size=20 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE30-10)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30-10)); /* Verify that the allocated block is section A in free-space manager */ - if (addr != TEST_BLOCK_ADDR70) - TEST_ERROR + if(addr != TBLOCK_ADDR70) + TEST_ERROR - state.tot_space -= (TEST_BLOCK_SIZE30-10); + state.tot_space -= (TBLOCK_SIZE30-10); state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Create section B */ - sect_node2 = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR100, (hsize_t)TEST_BLOCK_SIZE50); + sect_node2 = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR100, (hsize_t)TBLOCK_SIZE50); /* Add section B to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node2, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node2)) + FAIL_STACK_ERROR - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Try to extend the allocated block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)TEST_BLOCK_ADDR70, (hsize_t)(TEST_BLOCK_SIZE30-10), (hsize_t)TEST_BLOCK_SIZE50); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)TBLOCK_ADDR70, (hsize_t)(TBLOCK_SIZE30-10), (hsize_t)TBLOCK_SIZE50); /* Should not succeed in extending the allocated block */ if(was_extended) TEST_ERROR /* Free-space info should be the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the allocated block */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE30-10)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE30-10)); - state.tot_space += (TEST_BLOCK_SIZE30-10); + state.tot_space += (TBLOCK_SIZE30-10); state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Remove section A from free-space manger */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(TEST_BLOCK_SIZE30-10), (H5FS_section_info_t **)&node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30-10), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != addr) TEST_ERROR /* Remove section B from free-space manager */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE50, (H5FS_section_info_t **)&node) < 0) - TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50, f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) + TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -1957,6 +1920,7 @@ error: } /* test_mf_fs_extend() */ /* + *------------------------------------------------------------------------- * To verify that an aggregator is absorbed into a section. * * Test 1: To aborb the aggregator onto the beginning of the section @@ -1979,25 +1943,20 @@ error: * which will absorb meta_aggr to the section: * section size + remaining size of aggregator is > aggr->alloc_size, * section is allowed to absorb an aggregator (allow_sect_absorb is true) - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) { - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type, stype; - haddr_t addr, saddr; - haddr_t ma_addr=HADDR_UNDEF; - hsize_t ma_size=0; + hid_t file = -1; /* File ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr, saddr; + haddr_t tmp; + haddr_t ma_addr=HADDR_UNDEF; + hsize_t ma_size=0; H5MF_free_section_t *sect_node=NULL; - H5MF_sect_ud_t udata; - H5FS_section_info_t *node; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("A free-space section absorbs an aggregator: test 1"); @@ -2023,53 +1982,39 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR /* Allocate a section from meta_aggr */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - /* Add a section to free-space that adjoins end of the aggregator */ - sect_node = H5MF_sect_simple_new((haddr_t)(ma_addr+ma_size), (hsize_t)TEST_BLOCK_SIZE2048); + /* Create a section */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)(ma_addr+ma_size), (hsize_t)TBLOCK_SIZE2048); - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; - - /* When adding, meta_aggr is absorbed onto the beginning of the section */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + /* Add a section to free-space that adjoins end of the aggregator */ + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR /* Verify that the section did absorb the aggregator */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE2048, (H5FS_section_info_t **)&node) < 0) + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(ma_addr+ma_size), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) TEST_ERROR - if (node->addr != ma_addr) TEST_ERROR - if (node->size != (ma_size + TEST_BLOCK_SIZE2048)) TEST_ERROR - - /* Remove the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - TEST_ERROR + if(tmp != ma_addr) TEST_ERROR /* Restore info for aggregator */ f->shared->meta_aggr.addr = ma_addr; f->shared->meta_aggr.size = ma_size; /* Remove section from meta_aggr */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2093,48 +2038,34 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR /* Allocate a section from meta_aggr */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); /* Allocate a section from sdata_aggr */ - stype = H5FD_MEM_DRAW; - saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Add a section to free-space that adjoins the beginning of meta_aggr */ - sect_node = H5MF_sect_simple_new((haddr_t)addr, (hsize_t)TEST_BLOCK_SIZE30); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)addr, (hsize_t)TBLOCK_SIZE30); /* When adding, meta_aggr is absorbed onto the end of the section */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR /* Verify that the section did absorb the aggregator */ - if(H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)(ma_size+TEST_BLOCK_SIZE30), (H5FS_section_info_t **)&node) < 0) + if(H5MF_find_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(ma_size+TBLOCK_SIZE30), f->shared->fs_man[H5FD_MEM_SUPER], &tmp) != TRUE) TEST_ERROR - if ((node->addr + TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR - if (node->size != (ma_size + TEST_BLOCK_SIZE30)) TEST_ERROR - - /* free the free-space section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) + if((tmp + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* restore info to meta_aggr */ @@ -2142,9 +2073,9 @@ test_mf_fs_absorb(const char *env_h5_drvr, hid_t fapl) f->shared->meta_aggr.size = ma_size; /* Remove section from meta_aggr */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); /* Remove section from sdata_aggr */ - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2166,6 +2097,7 @@ error: } /* test_mf_fs_absorb() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2180,6 +2112,7 @@ error: * Result: * The second block of 50 is allocated from meta_aggr * There is space of 1968 left in meta_aggr + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) @@ -2187,6 +2120,7 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) hid_t file = -1; /* File ID */ char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ + hid_t fcpl; /* File creation property list */ h5_stat_size_t file_size, new_file_size; /* file size */ H5FD_mem_t type; haddr_t addr1, addr2; @@ -2202,13 +2136,21 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set to H5F_FSPACE_STRATEGY_AGGR strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_AGGR, FALSE, (hsize_t)1) < 0) + TEST_ERROR + /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Close file */ if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get the size of a file */ if((file_size = h5_get_file_size(filename, fapl)) < 0) @@ -2216,54 +2158,58 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) /* Re-open the file */ if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get a pointer to the internal file object */ if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR + TEST_ERROR /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + if((addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30)) == HADDR_UNDEF) + TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + if((addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50)) == HADDR_UNDEF) + TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get the size of the file */ if((new_file_size = h5_get_file_size(filename, fapl)) < 0) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)) + if(new_file_size != (file_size+TBLOCK_SIZE30+TBLOCK_SIZE50)) TEST_ERROR /* Re-open the file */ if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get a pointer to the internal file object */ if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR + TEST_ERROR - /* Free the two blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); + /* Free the two blocks: order matters because of H5F_FSPACE_STRATEGY_AGGR strategy */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50) < 0) + TEST_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30) < 0) + TEST_ERROR if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + TEST_ERROR /* Get the size of the file */ if((new_file_size = h5_get_file_size(filename, fapl)) < 0) @@ -2273,6 +2219,9 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) if (new_file_size != file_size) TEST_ERROR + if(H5Pclose(fcpl) < 0) + TEST_ERROR + PASSED() } /* end if */ else { @@ -2285,11 +2234,13 @@ test_mf_aggr_alloc1(const char *env_h5_drvr, hid_t fapl) error: H5E_BEGIN_TRY { H5Fclose(file); + H5Pclose(fcpl); } H5E_END_TRY; return(1); } /* test_mf_aggr_alloc1() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2311,6 +2262,7 @@ error: * A block of request-size is extended via file allocation and is merged with meta_aggr * The block of 2058 is allocated out of meta_aggr * There is space of 1968 left in meta_aggr + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) @@ -2354,29 +2306,29 @@ test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) FAIL_STACK_ERROR type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if (ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3+TEST_BLOCK_SIZE2058) != ma_addr) + if((addr3+TBLOCK_SIZE2058) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR if(H5Fclose(file) < 0) @@ -2388,7 +2340,7 @@ test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) /* Verify the file is the correct size */ /* Unused space is freed from the end of the file */ - if (new_file_size != (file_size+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50+TEST_BLOCK_SIZE2058)) + if(new_file_size != (file_size+TBLOCK_SIZE30+TBLOCK_SIZE50+TBLOCK_SIZE2058)) TEST_ERROR /* Re-open the file */ @@ -2399,7 +2351,7 @@ test_mf_aggr_alloc2(const char *env_h5_drvr, hid_t fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50+TEST_BLOCK_SIZE2058); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30+TBLOCK_SIZE50+TBLOCK_SIZE2058); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2429,6 +2381,7 @@ error: } /* test_mf_aggr_alloc2() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr : (nothing in the aggregator) @@ -2463,6 +2416,7 @@ error: * Result: * The fourth block of 50 is allocated from what is left in meta_aggr * There is space of 1968 left in meta_aggr + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc3(const char *env_h5_drvr, hid_t fapl) @@ -2509,55 +2463,55 @@ test_mf_aggr_alloc3(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR - if (sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) TEST_ERROR + if(sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate third block, which is from file allocation not from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE2058)); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE2058)); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); /* info for meta_aggr shouldn't be changed */ - if (addr3 != (sdata_addr+sdata_size)) TEST_ERROR - if ((addr3+TEST_BLOCK_SIZE2058) == new_ma_addr) TEST_ERROR - if ((new_ma_addr != ma_addr) || (new_ma_size != ma_size)) TEST_ERROR + if(addr3 != (sdata_addr+sdata_size)) TEST_ERROR + if((addr3+TBLOCK_SIZE2058) == new_ma_addr) TEST_ERROR + if((new_ma_addr != ma_addr) || (new_ma_size != ma_size)) TEST_ERROR /* Allocate fourth block, which should be from meta_aggr */ - addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr4+TEST_BLOCK_SIZE50) != ma_addr) + if((addr4+TBLOCK_SIZE50) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50 + TBLOCK_SIZE50))) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE2058); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE2058); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2567,7 +2521,7 @@ test_mf_aggr_alloc3(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -2588,6 +2542,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2626,6 +2581,7 @@ error: * The new block's address is returned * The block does not adjoin meta_aggr * meta_aggr's info is unchanged + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc4(const char *env_h5_drvr, hid_t fapl) @@ -2670,59 +2626,59 @@ test_mf_aggr_alloc4(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) + if(ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* Allocate second block from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE2048 - TBLOCK_SIZE30)); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (saddr2+(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30) != sdata_addr) + if(saddr2+(TBLOCK_SIZE2048 - TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* Allocate third block from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr3+TEST_BLOCK_SIZE50) != sdata_addr) + if((saddr3+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR - if(sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE50)) + if(sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE50)) TEST_ERROR /* Allocate second block of 2058, which is from file allocation, not from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); - if (addr2 != sdata_addr) + if(addr2 != sdata_addr) TEST_ERROR /* sdata_aggr is reset 0 */ H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((sdata_addr != 0) && (sdata_size != 0)) + if((sdata_addr != 0) && (sdata_size != 0)) TEST_ERROR /* info is unchanged in meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if ((new_ma_addr != ma_addr) && (new_ma_size != ma_size)) + if((new_ma_addr != ma_addr) && (new_ma_size != ma_size)) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE2058); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE2058); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)TBLOCK_SIZE2048 - TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2732,7 +2688,7 @@ test_mf_aggr_alloc4(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -2752,6 +2708,7 @@ error: } /* test_mf_aggr_alloc4() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2772,6 +2729,7 @@ error: * The block of 1970 is allocated from there * There is space of 2046 left in meta_aggr * + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc5(const char *env_h5_drvr, hid_t fapl) @@ -2816,34 +2774,34 @@ test_mf_aggr_alloc5(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (addr2+TEST_BLOCK_SIZE50 != ma_addr) + if(addr2+TBLOCK_SIZE50 != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30+TBLOCK_SIZE50))) TEST_ERROR /* Allocate third block from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (addr3 != ma_addr) TEST_ERROR - if ((addr3+TEST_BLOCK_SIZE1970) != new_ma_addr) TEST_ERROR - if (new_ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE1970 - ma_size))) + if(addr3 != ma_addr) TEST_ERROR + if((addr3+TBLOCK_SIZE1970) != new_ma_addr) TEST_ERROR + if(new_ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE1970 - ma_size))) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -2853,7 +2811,7 @@ test_mf_aggr_alloc5(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -2873,6 +2831,7 @@ error: } /* test_mf_aggr_alloc5() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * * Allocate first block (30) from meta_aggr: (nothing in the aggregator) @@ -2903,6 +2862,7 @@ error: * sdata_aggr is untouched * meta_aggr's unused space of [880, 1968] is freed to free-space * meta_aggr is updated to point to the new block + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) @@ -2915,7 +2875,7 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2, addr3, saddr1; haddr_t ma_addr=HADDR_UNDEF, new_ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, new_ma_size=0, sdata_size=0; - frspace_state_t state; + H5FS_stat_t state; hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 6"); @@ -2949,44 +2909,44 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) type = H5FD_MEM_SUPER; /* Allocate first block from meta_aggr */ - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (addr2+TEST_BLOCK_SIZE50 != ma_addr) + if(addr2+TBLOCK_SIZE50 != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if(ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) TEST_ERROR - if (sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) TEST_ERROR + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR + if(sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate third block from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if ((addr3+TEST_BLOCK_SIZE1970) != new_ma_addr) TEST_ERROR - if (addr3 != (sdata_addr+sdata_size)) TEST_ERROR + if((addr3+TBLOCK_SIZE1970) != new_ma_addr) TEST_ERROR + if(addr3 != (sdata_addr+sdata_size)) TEST_ERROR - if ((ma_addr+TEST_BLOCK_SIZE1970) == new_ma_addr) TEST_ERROR - if (new_ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE1970)) + if((ma_addr+TBLOCK_SIZE1970) == new_ma_addr) TEST_ERROR + if(new_ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE1970)) TEST_ERROR /* Verify that meta_aggr's unused space of 1968 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += (TBLOCK_SIZE2048 - (TBLOCK_SIZE30+TBLOCK_SIZE50)); state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -2994,10 +2954,10 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3007,7 +2967,7 @@ test_mf_aggr_alloc6(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (new_file_size != file_size) + if(new_file_size != file_size) TEST_ERROR PASSED() @@ -3027,13 +2987,14 @@ error: } /* test_mf_aggr_alloc6() */ /* + *------------------------------------------------------------------------- * To verify that blocks are allocated from the aggregator * - * Allocate first block (30) from meta_aggr: (nothing in meta_aggr) - * request-size is > what is left in meta_aggr and < meta_aggr->alloc_size - * Result: - * A block of meta_aggr->alloc_size is obtained via file allocation - * The first block of 30 is allocated from there + * Allocate first block (30) from meta_aggr: (nothing in meta_aggr) + * request-size is > what is left in meta_aggr and < meta_aggr->alloc_size + * Result: + * A block of meta_aggr->alloc_size is obtained via file allocation + * The first block of 30 is allocated from there * * Allocate second block (50) from meta_aggr: * request-size is <= what is left in meta_aggr @@ -3068,6 +3029,7 @@ error: * The block does not adjoin meta_aggr * meta_aggr's unused space of [880, 1968] is freed to free-space * meta_aggr is updated to point to the new block + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) @@ -3080,7 +3042,7 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) haddr_t addr1, addr2, addr3, saddr1, saddr2, saddr3; haddr_t ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, sdata_size=0; - frspace_state_t state; + H5FS_stat_t state; hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("H5MF_alloc() of meta/sdata aggregator:test 7"); @@ -3113,51 +3075,51 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) /* Allocate the first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate the second block from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (addr2+TEST_BLOCK_SIZE50 != ma_addr) + if (addr2+TBLOCK_SIZE50 != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50))) + if (ma_size != (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50))) TEST_ERROR /* Allocate the first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) + if ((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* Allocate the second block from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2048 - TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr2+(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) != sdata_addr) + if ((saddr2+(TBLOCK_SIZE2048 - TBLOCK_SIZE30)) != sdata_addr) TEST_ERROR if (sdata_size != 0) TEST_ERROR /* Allocate the third block from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr3+TEST_BLOCK_SIZE50) != sdata_addr) + if ((saddr3+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR - if (sdata_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE50)) + if (sdata_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE50)) TEST_ERROR /* Allocate the third block from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); if (addr3 != sdata_addr) TEST_ERROR - if ((addr3 + TEST_BLOCK_SIZE1970) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE1970)) TEST_ERROR + if ((addr3 + TBLOCK_SIZE1970) != ma_addr) TEST_ERROR + if (ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE1970)) TEST_ERROR /* sdata_aggr info is reset to 0 */ H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); @@ -3165,8 +3127,8 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) if (sdata_size != 0) TEST_ERROR /* Verify that meta_aggr's unused space of 1968 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += (TEST_BLOCK_SIZE2048 - (TEST_BLOCK_SIZE30 + TEST_BLOCK_SIZE50)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += (TBLOCK_SIZE2048 - (TBLOCK_SIZE30 + TBLOCK_SIZE50)); state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -3174,12 +3136,12 @@ test_mf_aggr_alloc7(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Free all the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)(TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)(TBLOCK_SIZE2048 - TBLOCK_SIZE30)); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3209,6 +3171,7 @@ error: } /* test_mf_aggr_alloc7() */ /* + *------------------------------------------------------------------------- * To verify that a block can be extended from the aggregator * * Test 1: Allocate block A from meta_aggr which is at end of file @@ -3229,6 +3192,7 @@ error: * Allocate block B from sdata_aggr so that meta_aggr is not at end of file * Try to extend a block which adjoins meta_aggr but meta_aggr cannot fulfill the extended-request * H5MF_try_extend() fails + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) @@ -3274,9 +3238,9 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) /* Allocate the first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr+TEST_BLOCK_SIZE30) != ma_addr) + if((addr+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Adjust meta_aggr's info info for testing */ @@ -3286,7 +3250,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) new_addr = addr - 10; /* Try to extend the block by an amount < (% * aggr->alloc_size) */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE50)); /* should succeed */ if(!was_extended) @@ -3294,15 +3258,15 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr+TEST_BLOCK_SIZE50)) + if(new_ma_addr != (addr+TBLOCK_SIZE50)) TEST_ERROR - if (new_ma_size != (f->shared->meta_aggr.alloc_size - TEST_BLOCK_SIZE50)) TEST_ERROR + if(new_ma_size != (f->shared->meta_aggr.alloc_size - TBLOCK_SIZE50)) TEST_ERROR /* Free the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE50); /* Try to extend the block by an amount > (% * aggr->alloc_size) but amount < aggr->alloc_size */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE700)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE700)); /* should succeed */ if(!was_extended) @@ -3310,15 +3274,15 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr + TEST_BLOCK_SIZE700)) + if(new_ma_addr != (addr + TBLOCK_SIZE700)) TEST_ERROR - if (new_ma_size != (f->shared->meta_aggr.alloc_size * 2 - TEST_BLOCK_SIZE700)) TEST_ERROR + if(new_ma_size != (f->shared->meta_aggr.alloc_size * 2 - TBLOCK_SIZE700)) TEST_ERROR /* Free the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE700); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE700); /* Try to extend the block by an amount > (% * aggr->alloc_size) but amount > aggr->alloc_size */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE2058)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE2058)); /* should succeed */ if(!was_extended) @@ -3326,12 +3290,12 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr + TEST_BLOCK_SIZE2058)) + if (new_ma_addr != (addr + TBLOCK_SIZE2058)) TEST_ERROR if (new_ma_size != f->shared->meta_aggr.size) TEST_ERROR /* Free the allocated blocks */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE2058); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE2058); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3365,16 +3329,16 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) /* Allocate the first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr+TEST_BLOCK_SIZE30) != ma_addr) + if((addr+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate the first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr+TEST_BLOCK_SIZE50) != sdata_addr) + if((saddr+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR /* Adjust meta_aggr's info info for testing */ @@ -3384,24 +3348,24 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) new_addr = addr - 10; /* should be able to fulfill request from the aggreqator itself */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE50)); if(!was_extended) TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != (addr+TEST_BLOCK_SIZE50)) + if(new_ma_addr != (addr+TBLOCK_SIZE50)) TEST_ERROR - if (new_ma_size != (f->shared->meta_aggr.alloc_size-TEST_BLOCK_SIZE50)) + if(new_ma_size != (f->shared->meta_aggr.alloc_size-TBLOCK_SIZE50)) TEST_ERROR /* Restore info for meta_aggr */ f->shared->meta_aggr.addr = ma_addr; f->shared->meta_aggr.size = ma_size; - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3411,7 +3375,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3435,16 +3399,16 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) /* Allocate first block from meta_aggr */ type = H5FD_MEM_SUPER; - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate first block from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr+TEST_BLOCK_SIZE50) != sdata_addr) + if((saddr+TBLOCK_SIZE50) != sdata_addr) TEST_ERROR /* Adjust meta_aggr's info info for testing */ @@ -3454,7 +3418,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) new_addr = addr - 10; /* unable to fulfill request from the aggreqator itself */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE50)); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TBLOCK_SIZE50)); if(was_extended) TEST_ERROR @@ -3468,8 +3432,8 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) f->shared->meta_aggr.addr = ma_addr; f->shared->meta_aggr.size = ma_size; - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3479,7 +3443,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3499,6 +3463,7 @@ error: } /* test_mf_aggr_extend() */ /* + *------------------------------------------------------------------------- * To verify that a block is absorbed into an aggregator * * MF_try_shrink() only allows blocks to be absorbed into an aggregator @@ -3517,10 +3482,7 @@ error: * H5MF_alloc() block C from meta_aggr * H5MF_try_shrink() block B should fail since it does not adjoin the * beginning nor the end of meta_aggr - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) @@ -3567,20 +3529,20 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) /* Allocate block A from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - ma_addr = new_ma_addr - TEST_BLOCK_SIZE30; + ma_addr = new_ma_addr - TBLOCK_SIZE30; - if((addr1 + TEST_BLOCK_SIZE30) != new_ma_addr) + if((addr1 + TBLOCK_SIZE30) != new_ma_addr) TEST_ERROR /* should succeed */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30) <= 0) TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != ma_addr) TEST_ERROR + if(new_ma_addr != ma_addr) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3590,7 +3552,7 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3614,32 +3576,32 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) /* Allocate block A from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR - if (ma_size != (TEST_BLOCK_SIZE2048 - TEST_BLOCK_SIZE30)) TEST_ERROR + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR + if(ma_size != (TBLOCK_SIZE2048 - TBLOCK_SIZE30)) TEST_ERROR /* Allocate block B from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->sdata_aggr), NULL, &sdata_size); /* should succeed */ - if(H5MF_try_shrink(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE50) <= 0) + if(H5MF_try_shrink(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE50) <= 0) TEST_ERROR H5MF_aggr_query(f, &(f->shared->sdata_aggr), &new_sdata_addr, &new_sdata_size); - if (new_sdata_addr != saddr1) TEST_ERROR - if (new_sdata_size != sdata_size + TEST_BLOCK_SIZE50) TEST_ERROR + if(new_sdata_addr != saddr1) TEST_ERROR + if(new_sdata_size != sdata_size + TBLOCK_SIZE50) TEST_ERROR /* meta_aggr info should be updated because the block is absorbed into the meta_aggr */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); if (new_ma_addr != ma_addr) TEST_ERROR if (new_ma_size != (ma_size)) TEST_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3649,7 +3611,7 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3673,35 +3635,35 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) /* Allocate block A from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* Allocate block B from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2+TEST_BLOCK_SIZE50) != ma_addr) + if((addr2+TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* Allocate block C from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50)); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3+TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50) != ma_addr) + if((addr3+TBLOCK_SIZE30+TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* should not succeed */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50) > 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50) > 0) TEST_ERROR /* aggregator info should be the same as before */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); - if (new_ma_addr != ma_addr) TEST_ERROR + if(new_ma_addr != ma_addr) TEST_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)(TEST_BLOCK_SIZE30+TEST_BLOCK_SIZE50)); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)(TBLOCK_SIZE30+TBLOCK_SIZE50)); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3711,7 +3673,7 @@ test_mf_aggr_absorb(const char *env_h5_drvr, hid_t fapl) TEST_ERROR /* Verify the file is the correct size */ - if (file_size != empty_size) + if(file_size != empty_size) TEST_ERROR PASSED() @@ -3731,6 +3693,7 @@ error: } /* test_mf_aggr_absorb() */ /* + *------------------------------------------------------------------------- * To verify that a block allocated from file allocation is aligned, can be shrunk and extended * * Alignment = 1024 or 4096 @@ -3759,6 +3722,7 @@ error: * Allocate a block which should be from file allocation * The return address should be aligned * H5MF_try_extend() the block with aligned address should succeed + *------------------------------------------------------------------------- */ static unsigned test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -3773,7 +3737,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) haddr_t ma_addr=HADDR_UNDEF; hsize_t ma_size=0; htri_t was_extended; - frspace_state_t state; + H5FS_stat_t state; hsize_t alignment=0, mis_align=0, tmp=0, accum=0; hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ @@ -3819,23 +3783,23 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; - accum = mis_align + TEST_BLOCK_SIZE30; + accum = mis_align + TBLOCK_SIZE30; /* Allocate a block of 30 from file allocation */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* there should be nothing in the aggregator */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (ma_addr || ma_size) TEST_ERROR + if(ma_addr || ma_size) TEST_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); + HDmemset(&state, 0, sizeof(H5FS_stat_t)); if (mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; @@ -3848,18 +3812,18 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = 0; if ((tmp = ((hsize_t)file_size + accum) % alignment)) mis_align = alignment - tmp; - accum += (mis_align + TEST_BLOCK_SIZE50); + accum += (mis_align + TBLOCK_SIZE50); - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ if (addr2 % alignment) TEST_ERROR /* there should be nothing in the aggregator */ H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (ma_addr || ma_size) TEST_ERROR + if(ma_addr || ma_size) TEST_ERROR - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -3867,8 +3831,8 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR } - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3902,7 +3866,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* allocate a block of 50 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* address should be aligned */ if (addr1 % alignment) TEST_ERROR @@ -3923,7 +3887,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* shrink the block */ - if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE50) <= 0) + if(H5MF_try_shrink(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE50) <= 0) TEST_ERROR if(H5Fclose(file) < 0) @@ -3932,7 +3896,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if((new_file_size = h5_get_file_size(filename, fapl1)) < 0) TEST_ERROR - if (new_file_size != (file_size-TEST_BLOCK_SIZE50)) TEST_ERROR + if (new_file_size != (file_size-TBLOCK_SIZE50)) TEST_ERROR PASSED() } /* end if */ @@ -3957,10 +3921,10 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* allocate a block of 50 */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* address should be aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* Close file */ if(H5Fclose(file) < 0) @@ -3978,9 +3942,9 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* try to extend the block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr1, (hsize_t)TEST_BLOCK_SIZE50, (hsize_t)TEST_BLOCK_SIZE30); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE50, (hsize_t)TBLOCK_SIZE30); - if (was_extended <=0) TEST_ERROR + if(was_extended <=0) TEST_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -3988,7 +3952,7 @@ test_mf_align_eoa(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if((new_file_size = h5_get_file_size(filename, fapl1)) < 0) TEST_ERROR - if (new_file_size != (file_size+TEST_BLOCK_SIZE30)) TEST_ERROR + if (new_file_size != (file_size+TBLOCK_SIZE30)) TEST_ERROR PASSED() } /* end if */ @@ -4007,6 +3971,7 @@ error: } /* test_mf_align_eoa() */ /* + *------------------------------------------------------------------------- * To verify that a block allocated from the free-space manager is aligned * * Alignment = 1024 or 4096 @@ -4035,10 +4000,7 @@ error: * Allocate a block of size=40 * The free-space manager is unable to fulfill the request * The block is allocated from file allocation and should be aligned - * - * Modifications: - * Vailin Choi; July 2012 - * Initialize the new field "allow_eoa_shrink_only" for user data. + *------------------------------------------------------------------------- */ static unsigned test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -4047,12 +4009,10 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) char filename[FILENAME_LEN]; /* Filename to use */ h5_stat_size_t file_size; H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; H5MF_free_section_t *sect_node = NULL; haddr_t addr; - frspace_state_t state; - H5MF_sect_ud_t udata; - htri_t was_extended; + H5FS_stat_t state; + htri_t was_extended; hsize_t alignment=0, tmp=0, mis_align=0; hbool_t have_alloc_vfd; /* Whether VFD used has an 'alloc' callback */ @@ -4081,58 +4041,51 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR - - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - sect_node = H5MF_sect_simple_new((haddr_t)alignment, (hsize_t)TEST_BLOCK_SIZE50); + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) + TEST_ERROR - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + /* Create section A */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)alignment, (hsize_t)TBLOCK_SIZE50); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE50; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 50 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is section A in free-space */ - if (addr != (haddr_t)alignment) TEST_ERROR - if (addr % alignment) TEST_ERROR + if(addr != (haddr_t)alignment) TEST_ERROR + if(addr % alignment) TEST_ERROR - state.tot_space -= TEST_BLOCK_SIZE50; + state.tot_space -= TBLOCK_SIZE50; state.tot_sect_count -= 1; state.serial_sect_count -= 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)TBLOCK_SIZE50); - state.tot_space += TEST_BLOCK_SIZE50; + state.tot_space += TBLOCK_SIZE50; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -4151,70 +4104,63 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) + TEST_ERROR + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) - TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) - TEST_ERROR - - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE8000); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + /* Create section A */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE8000); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE8000; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE8000; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Allocate a block of 600 */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE600); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE600); /* Verify that the allocated block is aligned */ if (addr % alignment) TEST_ERROR /* should have 1 more section in free-space */ - state.tot_space -= TEST_BLOCK_SIZE600; + state.tot_space -= TBLOCK_SIZE600; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* try to extend the block */ - was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, type, (haddr_t)addr, (hsize_t)TEST_BLOCK_SIZE600, (hsize_t)TEST_BLOCK_SIZE200); + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_SUPER, (haddr_t)addr, (hsize_t)TBLOCK_SIZE600, (hsize_t)TBLOCK_SIZE200); - if (was_extended <=0) TEST_ERROR + if(was_extended <=0) TEST_ERROR /* space should be decreased by 200, # of sections remain the same */ - state.tot_space -= TEST_BLOCK_SIZE200; + state.tot_space -= TBLOCK_SIZE200; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* Free the block to free-space manager */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TEST_BLOCK_SIZE600+TEST_BLOCK_SIZE200)); + H5MF_xfree(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, addr, (hsize_t)(TBLOCK_SIZE600+TBLOCK_SIZE200)); /* only 1 section in free-space because of merging */ - state.tot_space += (TEST_BLOCK_SIZE600+TEST_BLOCK_SIZE200); + state.tot_space += (TBLOCK_SIZE600+TBLOCK_SIZE200); state.tot_sect_count = 1; state.serial_sect_count = 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -4241,45 +4187,38 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - type = H5FD_MEM_SUPER; - - if(H5MF__alloc_start(f, H5AC_ind_read_dxpl_id, type) < 0) - TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_start_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)H5FD_MEM_SUPER) < 0) + FAIL_STACK_ERROR - if (f->shared->fs_state[type] != H5F_FS_STATE_OPEN) + if(f->shared->fs_state[H5FD_MEM_SUPER] != H5F_FS_STATE_OPEN) TEST_ERROR - if (f->shared->fs_man[type]->client != H5FS_CLIENT_FILE_ID) + if(f->shared->fs_man[H5FD_MEM_SUPER]->client != H5FS_CLIENT_FILE_ID) TEST_ERROR - sect_node = H5MF_sect_simple_new((haddr_t)TEST_BLOCK_ADDR70, (hsize_t)TEST_BLOCK_SIZE700); - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = H5AC_ind_read_dxpl_id; - udata.alloc_type = type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + /* Create section A */ + sect_node = H5MF_sect_new(H5MF_FSPACE_SECT_SIMPLE, (haddr_t)TBLOCK_ADDR70, (hsize_t)TBLOCK_SIZE700); /* Add section A to free-space manager */ - if (H5FS_sect_add(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], (H5FS_section_info_t *)sect_node, H5FS_ADD_RETURNED_SPACE, &udata)) - FAIL_STACK_ERROR + if(H5MF_add_sect(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, f->shared->fs_man[H5FD_MEM_SUPER], sect_node)) + FAIL_STACK_ERROR - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE700; + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + state.tot_space += TBLOCK_SIZE700; state.tot_sect_count += 1; state.serial_sect_count += 1; - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR /* * Allocate a block of 40 * Since free-space manager cannot fulfull the request because of alignment, * the block is obtained from file allocation */ - addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)(TEST_BLOCK_SIZE40)); + addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)(TBLOCK_SIZE40)); /* Verify that the allocated block is aligned */ - if (addr % alignment) + if(addr % alignment) TEST_ERROR /* verify that the allocated block is from file allocation, not section A in free-space */ @@ -4296,7 +4235,7 @@ test_mf_align_fs(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } /* free-space info should be the same */ - if(check_stats(f, f->shared->fs_man[type], &state)) + if(check_stats(f, f->shared->fs_man[H5FD_MEM_SUPER], &state)) TEST_ERROR if(H5Fclose(file) < 0) @@ -4319,6 +4258,7 @@ error: } /* test_mf_align_fs() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -4413,6 +4353,7 @@ error: * Fragment from alignment of aggregator allocation is freed to free-space:[12368, 4016] * There is space of 2018 left in meta_aggr * EOA is at 20372 + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -4424,7 +4365,7 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5FD_mem_t type; haddr_t addr1, addr2, addr3, addr4; - frspace_state_t state; + H5FS_stat_t state; haddr_t ma_addr=HADDR_UNDEF; hsize_t ma_size=0, mis_align=0; hsize_t alignment=0, tmp=0; @@ -4467,19 +4408,19 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4487,22 +4428,22 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 50 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 50 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4510,7 +4451,7 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE50) != ma_addr) + if((addr2 + TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 80 */ @@ -4518,13 +4459,13 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) if ((tmp = ma_addr % alignment)) mis_align = alignment - tmp; - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ - if (addr3 % alignment) TEST_ERROR + if(addr3 % alignment) TEST_ERROR /* fragment for alignment of block 80 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4532,22 +4473,22 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3 + TEST_BLOCK_SIZE80) != ma_addr) + if((addr3 + TBLOCK_SIZE80) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 1970 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 1970 from meta_aggr */ - addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1970); + addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1970); /* Verify that the allocated block is aligned */ - if (addr4 % alignment) TEST_ERROR + if(addr4 % alignment) TEST_ERROR /* fragment for alignment of block 1970 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -4555,17 +4496,17 @@ test_mf_align_alloc1(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr4 + TEST_BLOCK_SIZE1970) != ma_addr) + if((addr4 + TBLOCK_SIZE1970) != ma_addr) TEST_ERROR /* Verify total size of free space after all the allocations */ if(check_stats(f, f->shared->fs_man[type], &state)) TEST_ERROR - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE80); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE1970); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE80); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE1970); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -4587,6 +4528,7 @@ error: } /* test_mf_align_alloc1() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -4669,10 +4611,7 @@ error: * The third block of 80 is allocated from the aggregator and should be aligned * There is space of 1968 left in meta_aggr * EOA is at 18432 - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -4683,7 +4622,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) h5_stat_size_t file_size; /* File size */ H5FD_mem_t type, stype; haddr_t addr1, addr2, addr3, saddr1; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, sdata_size=0, mis_align=0; hsize_t alignment=0, tmp=0; @@ -4725,19 +4664,19 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -4745,21 +4684,21 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) + if((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -4767,7 +4706,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE50) != ma_addr) + if((addr2 + TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* @@ -4782,17 +4721,17 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) * block 30 is allocated from ma_addr */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) mis_align = alignment - tmp; else if ((alignment == TEST_ALIGN4096) && (tmp = (ma_addr % alignment))) mis_align = alignment - tmp; /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* fragment for alignment of block 30 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; @@ -4804,7 +4743,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if (sdata_addr != (saddr1 + TEST_BLOCK_SIZE30)) TEST_ERROR + if(sdata_addr != (saddr1 + TBLOCK_SIZE30)) TEST_ERROR /* * Calculate fragment for the allocation of block 80 from meta_aggr: @@ -4816,19 +4755,19 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) * block 30 is allocated from ma_addr */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) mis_align = alignment - tmp; else if ((alignment == TEST_ALIGN4096) && (tmp = ((sdata_addr + sdata_size) % alignment))) mis_align = alignment - tmp; /* Allocate a block of 80 from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ if (addr3 % alignment) TEST_ERROR /* fragment for alignment of block 80 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -4836,7 +4775,7 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3 + TEST_BLOCK_SIZE80) != ma_addr) + if((addr3 + TBLOCK_SIZE80) != ma_addr) TEST_ERROR /* Verify total size of free space after all the allocations */ @@ -4850,10 +4789,10 @@ test_mf_align_alloc2(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR } - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE50); - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE80); - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE50); + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE80); + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); if(H5Fclose(file) < 0) FAIL_STACK_ERROR @@ -4875,6 +4814,7 @@ error: } /* test_mf_align_alloc2() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -5008,11 +4948,7 @@ error: * The meta_aggr is updated to point to the new space * The block of 1034 is allocated from the new block and should be aligned * There is space of 1014 left in meta_aggr - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY - * + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5024,7 +4960,7 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5FD_mem_t type, stype; haddr_t addr1, addr2, addr3; haddr_t saddr1, saddr2, saddr3; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, sdata_size=0, mis_align=0; hsize_t alignment=0, tmp=0; @@ -5067,48 +5003,48 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) FAIL_STACK_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 50 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 50 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE50) != ma_addr) + if((addr2 + TBLOCK_SIZE50) != ma_addr) TEST_ERROR /* @@ -5123,84 +5059,84 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) * block 30 is allocated from ma_addr */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = ((ma_addr + ma_size) % alignment))) mis_align = alignment - tmp; else if ((alignment == TEST_ALIGN4096) && (tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (saddr1 % alignment) TEST_ERROR + if(saddr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr1+TEST_BLOCK_SIZE30)) TEST_ERROR + if(sdata_addr != (saddr1+TBLOCK_SIZE30)) TEST_ERROR /* calculate fragment for alignment of block 50 in sdata_aggr */ mis_align = 0; - if ((tmp = sdata_addr % alignment)) + if((tmp = sdata_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 50 from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ - if (saddr2 % alignment) TEST_ERROR + if(saddr2 % alignment) TEST_ERROR /* fragment for alignment of block 50 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr2 + TEST_BLOCK_SIZE50)) TEST_ERROR + if(sdata_addr != (saddr2 + TBLOCK_SIZE50)) TEST_ERROR /* calculate fragment for alignment of block 80 in sdata_aggr */ mis_align = 0; - if ((tmp = sdata_addr % alignment)) + if((tmp = sdata_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 80 from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ - if (saddr3 % alignment) TEST_ERROR + if(saddr3 % alignment) TEST_ERROR /* fragment for alignment of block 80 for sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr3 + TEST_BLOCK_SIZE80) != sdata_addr) + if ((saddr3 + TBLOCK_SIZE80) != sdata_addr) TEST_ERROR /* calculate fragment for alignment of block 1034 */ mis_align = 0; - if ((tmp = sdata_addr % alignment)) + if((tmp = sdata_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 1034 for meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1034); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034); /* Verify that the allocated block is aligned */ - if (addr3 % alignment) TEST_ERROR + if(addr3 % alignment) TEST_ERROR /* fragment for alignment of block 1034 for meta_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5208,11 +5144,11 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* calculate unused space in meta_aggr that is freed to free-space after block 1034 */ mis_align = 0; - if ((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) + if((alignment == TEST_ALIGN1024) && (tmp = (ma_addr % alignment))) mis_align = alignment - tmp; /* fragment for unused space in meta_aggr after block 1034 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5220,7 +5156,7 @@ test_mf_align_alloc3(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr3 + TEST_BLOCK_SIZE1034) != ma_addr) + if((addr3 + TBLOCK_SIZE1034) != ma_addr) TEST_ERROR /* Verify total size of free space after all allocations */ @@ -5255,6 +5191,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 4096 aggr->alloc_size = 2048 @@ -5317,6 +5254,7 @@ error: * Fragment from alignment of aggregator allocation is freed to free-space:[10250, 2038] * There is space of 2023 left in meta_aggr * + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5327,7 +5265,7 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) h5_stat_size_t file_size; H5FD_mem_t type; haddr_t addr1, addr2, addr3; - frspace_state_t state; + H5FS_stat_t state; haddr_t ma_addr=HADDR_UNDEF; hsize_t ma_size=0, saved_ma_size=0; hsize_t alignment=0, mis_align=0, tmp=0; @@ -5370,19 +5308,19 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t)); + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -5390,21 +5328,21 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); saved_ma_size = ma_size; - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR + if((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 2058 */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 2058 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); /* Verify that the allocated block is aligned */ - if (addr2 % alignment) TEST_ERROR + if(addr2 % alignment) TEST_ERROR /* fragment for alignment of block 2058 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; @@ -5412,28 +5350,28 @@ test_mf_align_alloc4(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr2 + TEST_BLOCK_SIZE2058) != ma_addr) TEST_ERROR + if((addr2 + TBLOCK_SIZE2058) != ma_addr) TEST_ERROR /* meta_aggr->size remains the same */ - if (ma_size != saved_ma_size) TEST_ERROR + if(ma_size != saved_ma_size) TEST_ERROR /* calculate fragment for alignment of block 5 from meta_aggr */ mis_align = 0; - if ((tmp = ma_addr % alignment)) + if((tmp = ma_addr % alignment)) mis_align = alignment - tmp; /* Allocate a block of 5 from meta_aggr */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5); + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5); /* fragment for alignment of block 5 is freed to free-space */ - if (mis_align) { + if(mis_align) { state.tot_space += mis_align; state.tot_sect_count += 1; state.serial_sect_count += 1; } /* Verify that the allocated block is aligned */ - if (addr3 % alignment) TEST_ERROR + if(addr3 % alignment) TEST_ERROR /* Verify total size of free space after all allocations */ if(f->shared->fs_man[type]) { @@ -5461,6 +5399,7 @@ error: } /* test_mf_align_alloc4() */ /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -5525,10 +5464,7 @@ error: * sdata_aggr is reset to 0 * EOA is 14346 * meta_aggr and sdata_aggr are all 0 - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5539,7 +5475,7 @@ test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) h5_stat_size_t file_size; H5FD_mem_t type, stype; haddr_t addr1, addr2, saddr1; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, new_ma_addr=HADDR_UNDEF; haddr_t sdata_addr=HADDR_UNDEF, new_sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, new_ma_size=0, sdata_size=0, new_sdata_size=0; @@ -5583,23 +5519,23 @@ test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (addr1 % alignment) TEST_ERROR + if(addr1 % alignment) TEST_ERROR H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1 + TEST_BLOCK_SIZE30) != ma_addr) TEST_ERROR + if((addr1 + TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* fragment for alignment of block 30 is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5607,39 +5543,39 @@ test_mf_align_alloc5(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* calculate fragment for alignment of block 30 from sdata_aggr */ mis_align = 0; - if ((tmp = (ma_addr + ma_size) % alignment)) + if((tmp = (ma_addr + ma_size) % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ - if (saddr1 % alignment) TEST_ERROR + if(saddr1 % alignment) TEST_ERROR /* fragment of alignment for block 30 in sdata_aggr is freed to free-space */ - if (mis_align) { + if(mis_align) { state[stype].tot_space += mis_align; state[stype].tot_sect_count += 1; state[stype].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if ((saddr1+TEST_BLOCK_SIZE30) != sdata_addr) TEST_ERROR + if((saddr1+TBLOCK_SIZE30) != sdata_addr) TEST_ERROR /* calculate fragment for alignment of block 2058 from meta_aggr */ mis_align = 0; - if ((tmp = (sdata_addr + sdata_size) % alignment)) + if((tmp = (sdata_addr + sdata_size) % alignment)) mis_align = alignment - tmp; /* Allocate a block of 2058 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); /* Verify that the allocated block is aligned */ if (addr2 % alignment) TEST_ERROR /* fragment for alignment of block 2058 is freed to free-space */ - if (mis_align) { + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; @@ -5687,6 +5623,7 @@ error: /* + *------------------------------------------------------------------------- * To verify that blocks allocated from the aggregator are aligned * * Alignment = 1024 aggr->alloc_size = 2048 @@ -5792,10 +5729,7 @@ error: * Fragment from alignment of file allocation is freed to free-space:[16464, 4016] * EOA is at 22538 * meta_aggr is unchanged - * - * Modifications: - * Vailin Choi; July 2012 - * Changes due to the switch to H5FD_FLMAP_DICHOTOMY + *------------------------------------------------------------------------- */ static unsigned test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) @@ -5807,7 +5741,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) H5FD_mem_t type, stype; haddr_t addr1, addr2; haddr_t saddr1, saddr2, saddr3; - frspace_state_t state[H5FD_MEM_NTYPES]; + H5FS_stat_t state[H5FD_MEM_NTYPES]; haddr_t ma_addr=HADDR_UNDEF, new_ma_addr=HADDR_UNDEF, sdata_addr=HADDR_UNDEF; hsize_t ma_size=0, new_ma_size=0, sdata_size=0; hsize_t alignment=0, mis_align=0, tmp=0; @@ -5849,26 +5783,26 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) TEST_ERROR /* calculate fragment for alignment of block 30 */ - if ((tmp = (hsize_t)file_size % alignment)) + if((tmp = (hsize_t)file_size % alignment)) mis_align = alignment - tmp; /* Allocate a block of 30 from meta_aggr */ type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ if (addr1 % alignment) TEST_ERROR /* fragment for alignment of block 30 in meta_aggr is freed to free-space */ - HDmemset(&state, 0, sizeof(frspace_state_t) * H5FD_MEM_NTYPES); - if (mis_align) { + HDmemset(&state, 0, sizeof(H5FS_stat_t) * H5FD_MEM_NTYPES); + if(mis_align) { state[type].tot_space += mis_align; state[type].tot_sect_count += 1; state[type].serial_sect_count += 1; } H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); - if ((addr1+TEST_BLOCK_SIZE30) != ma_addr) + if ((addr1+TBLOCK_SIZE30) != ma_addr) TEST_ERROR /* calculate fragment for alignment of block 30 in sdata_aggr */ @@ -5878,7 +5812,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) /* Allocate a block of 30 from sdata_aggr */ stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); /* Verify that the allocated block is aligned */ if (saddr1 % alignment) TEST_ERROR @@ -5891,7 +5825,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr1+TEST_BLOCK_SIZE30)) TEST_ERROR + if (sdata_addr != (saddr1+TBLOCK_SIZE30)) TEST_ERROR /* calculate fragment for alignment of block 50 in sdata_aggr */ mis_align = 0; @@ -5899,7 +5833,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = alignment - tmp; /* Allocate a block of 50 from sdata_aggr */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); /* Verify that the allocated block is aligned */ if (saddr2 % alignment) TEST_ERROR @@ -5912,7 +5846,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr2+TEST_BLOCK_SIZE50)) TEST_ERROR + if (sdata_addr != (saddr2+TBLOCK_SIZE50)) TEST_ERROR /* calculate fragment for alignment of block 80 in sdata_aggr */ mis_align = 0; @@ -5920,7 +5854,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = alignment - tmp; /* Allocate a block of 80 from sdata_aggr */ - saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE80); + saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); /* Verify that the allocated block is aligned */ if (saddr3 % alignment) TEST_ERROR @@ -5933,7 +5867,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) } H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sdata_addr, &sdata_size); - if (sdata_addr != (saddr3+TEST_BLOCK_SIZE80)) TEST_ERROR + if (sdata_addr != (saddr3+TBLOCK_SIZE80)) TEST_ERROR /* calculate fragment for alignment of block 2058 */ /* remaining space in sdata_aggr is freed and shrunk */ @@ -5942,7 +5876,7 @@ test_mf_align_alloc6(const char *env_h5_drvr, hid_t fapl, hid_t new_fapl) mis_align = alignment - tmp; /* Allocate a block of 2058 from meta_aggr */ - addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2058); + addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2058); /* Verify that the allocated block is aligned */ if (addr2 % alignment) TEST_ERROR @@ -5993,7 +5927,6 @@ error: return(1); } /* test_mf_align_alloc6() */ - /* * Test a bug that occurs when an allocator with zero size left and an unaligned * endpoint is extended to allocate an aligned object @@ -6156,334 +6089,111 @@ error: return(1); } /* test_mf_bug1() */ - /* - * Verify that the file's free-space manager persists where there are free sections in the manager + * Verify that the file's free-space manager(s) are persistent for a split-file + *------------------------------------------------------------------------- */ static unsigned -test_mf_fs_persist(hid_t fapl_new, hid_t fcpl) +test_mf_fs_persist_split(void) { - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ - haddr_t tmp_addr; /* Temporary variable for address */ - - TESTING("file's free-space manager is persistent"); - - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Allocate 6 blocks */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - - /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Verify that H5FD_MEM_SUPER free-space manager is there */ - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) - FAIL_STACK_ERROR - - /* Get info for free-space manager */ - if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0) - FAIL_STACK_ERROR - - /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3+TEST_BLOCK_SIZE5)) - TEST_ERROR - - if(fs_stat.serial_sect_count < 3) - TEST_ERROR - - /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(tmp_addr != addr3) - TEST_ERROR - - /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(tmp_addr != addr1) - TEST_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl = -1; /* File access property list ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type, stype, btype; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ + haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_DRAW */ + haddr_t baddr5, baddr6, baddr7, baddr8; /* File address for H5FD_MEM_BTREE */ + haddr_t tmp_addr; /* temporary variable for address */ + + TESTING("File's free-space managers are persistent for split-file"); + + /* for now, we don't support persistant free space managers + * with the split file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ - /* Verify that H5FD_MEM_SUPER free-space manager is there */ - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - /* Retrieve block #5 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(tmp_addr != addr5) - TEST_ERROR + /* for now, we don't support persistant free space managers + * with the split file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - PASSED() + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR - return(0); + /* Set up split driver */ + if(H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT)<0) + FAIL_STACK_ERROR -error: - H5E_BEGIN_TRY { - H5Fclose(file); - } H5E_END_TRY; - return(1); -} /* test_mf_fs_persist() */ - -/* - * Verify that the free-space manager goes away - */ -static unsigned -test_mf_fs_gone(hid_t fapl_new, hid_t fcpl) -{ - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR - TESTING("file's free-space manager is going away"); + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR - /* Allocate 4 blocks */ + /* Allocate 4 blocks of type H5FD_MEM_SUPER */ type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - - /* Put block #1, #3 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - - /* Retrieve block #1, #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Put block #3 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) FAIL_STACK_ERROR - - /* Verify that H5FD_MEM_SUPER free-space manager is there */ - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) FAIL_STACK_ERROR - /* Get info for H5FD_MEM_SUPER free-space manager */ - if(H5FS_stat_info(f, f->shared->fs_man[type], &fs_stat) < 0) + /* Put block #1, #3 into H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) FAIL_STACK_ERROR - - /* Verify free-space info */ - if(!H5F_addr_defined(fs_stat.addr) || !H5F_addr_defined(fs_stat.sect_addr)) - TEST_ERROR - if(fs_stat.tot_space < TEST_BLOCK_SIZE3) - TEST_ERROR - - /* Put block #4 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TEST_BLOCK_SIZE4) < 0) + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) FAIL_STACK_ERROR - /* The H5FD_MEM_SUPER free-space manager will go away at H5MF_close() */ - if(H5Fclose(file) < 0) + /* Allocate 4 blocks of type H5FD_MEM_DRAW */ + stype = H5FD_MEM_DRAW; + if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) FAIL_STACK_ERROR - - /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - if(H5Fclose(file) < 0) + if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) FAIL_STACK_ERROR - PASSED() - - return(0); - -error: - H5E_BEGIN_TRY { - H5Fclose(file); - } H5E_END_TRY; - return(1); -} /* test_mf_fs_gone() */ - - -/* - * Verify that the file's free-space manager(s) are persistent for a split-file - */ -static unsigned -test_mf_fs_split(hid_t fapl_new, hid_t fcpl) -{ - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type, stype, btype; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ - haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_DRAW */ - haddr_t baddr5, baddr6, baddr7, baddr8; /* File address for H5FD_MEM_BTREE */ - haddr_t tmp_addr; /* temporary variable for address */ - - TESTING("file's free-space managers are persistent for split-file"); - - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + /* Put block #1, #3 into H5FD_MEM_DRAW free-space manager */ + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE1) < 0) FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE3) < 0) FAIL_STACK_ERROR - /* Allocate 4 blocks of type H5FD_MEM_SUPER */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - - /* Put block #1, #3 into H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - - /* Allocate 4 blocks of type H5FD_MEM_DRAW */ - stype = H5FD_MEM_DRAW; - if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - - /* Put block #1, #3 into H5FD_MEM_DRAW free-space manager */ - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6495,7 +6205,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6503,23 +6213,24 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR - /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #2 still in free-space */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #3 still in free-space */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != addr1) - TEST_ERROR + TEST_ERROR /* Verify that the free-space manager for H5FD_MEM_DRAW is there */ if(!H5F_addr_defined(f->shared->fs_addr[stype])) TEST_ERROR + /* Start up H5FD_MEM_DRAW free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)stype) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6527,47 +6238,47 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #1 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != saddr1) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #3 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != saddr3) - TEST_ERROR + TEST_ERROR /* H5FD_MEM_DRAW free-space manager is going away at closing */ /* works for this one because the freeing of sect_addr is to H5FD_MEM_SUPER fs, not against itself */ /* Allocate 4 blocks of type H5FD_MEM_BTREE */ btype = H5FD_MEM_BTREE; - if(HADDR_UNDEF == (baddr5 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr6 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr7 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE7))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr8 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE8))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr5 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr6 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr7 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE7))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr8 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8))) + FAIL_STACK_ERROR /* Put block #5 & #7 into H5FD_MEM_BTREE free-space manager */ - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr7, (hsize_t)TEST_BLOCK_SIZE7) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr7, (hsize_t)TBLOCK_SIZE7) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6583,7 +6294,7 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6591,27 +6302,27 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE3+TEST_BLOCK_SIZE5+TEST_BLOCK_SIZE7)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE3+TBLOCK_SIZE5+TBLOCK_SIZE7)) + TEST_ERROR /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != addr3) - TEST_ERROR + TEST_ERROR /* Retrieve block #7 from H5FD_MEM_BTREE free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE7))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE7))) + FAIL_STACK_ERROR if(tmp_addr != baddr7) - TEST_ERROR + TEST_ERROR /* There should still be block #5 of H5FD_MEM_BTREE in H5FD_MEM_BTREE free-space manager */ if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6620,10 +6331,10 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) /* Verify that the H5FD_MEM_SUPER free-space manager is there */ if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR + TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6631,11 +6342,16 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < TEST_BLOCK_SIZE5) - TEST_ERROR + if(fs_stat.tot_space < TBLOCK_SIZE5) + TEST_ERROR + /* Closing */ if(H5Fclose(file) < 0) FAIL_STACK_ERROR + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR PASSED() @@ -6643,37 +6359,113 @@ test_mf_fs_split(hid_t fapl_new, hid_t fcpl) error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(file); + H5Pclose(fcpl); + H5Pclose(fapl); } H5E_END_TRY; return(1); -} /* test_mf_fs_split() */ +} /* test_mf_fs_persist_split() */ + +#define MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) { \ + H5FD_mem_t mt; \ + HDmemset(memb_map, 0, sizeof memb_map); \ + HDmemset(memb_fapl, 0, sizeof memb_fapl); \ + HDmemset(memb_name, 0, sizeof memb_name); \ + HDmemset(memb_addr, 0, sizeof memb_addr); \ + HDmemset(sv, 0, sizeof sv); \ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { \ + memb_map[mt] = H5FD_MEM_SUPER; \ + memb_fapl[mt] = H5P_DEFAULT; \ + } \ + memb_map[H5FD_MEM_BTREE] = H5FD_MEM_BTREE; \ + memb_map[H5FD_MEM_DRAW] = H5FD_MEM_DRAW; \ + memb_map[H5FD_MEM_GHEAP] = H5FD_MEM_GHEAP; \ + memb_map[H5FD_MEM_LHEAP] = H5FD_MEM_LHEAP; \ + sprintf(sv[H5FD_MEM_SUPER], "%%s-%c.h5", 's'); \ + memb_name[H5FD_MEM_SUPER] = sv[H5FD_MEM_SUPER]; \ + memb_addr[H5FD_MEM_SUPER] = 0; \ + sprintf(sv[H5FD_MEM_BTREE], "%%s-%c.h5", 'b'); \ + memb_name[H5FD_MEM_BTREE] = sv[H5FD_MEM_BTREE]; \ + memb_addr[H5FD_MEM_BTREE] = HADDR_MAX/6; \ + sprintf(sv[H5FD_MEM_DRAW], "%%s-%c.h5", 'r'); \ + memb_name[H5FD_MEM_DRAW] = sv[H5FD_MEM_DRAW]; \ + memb_addr[H5FD_MEM_DRAW] = HADDR_MAX/3; \ + sprintf(sv[H5FD_MEM_GHEAP], "%%s-%c.h5", 'g'); \ + memb_name[H5FD_MEM_GHEAP] = sv[H5FD_MEM_GHEAP]; \ + memb_addr[H5FD_MEM_GHEAP] = HADDR_MAX/2; \ + sprintf(sv[H5FD_MEM_LHEAP], "%%s-%c.h5", 'l'); \ + memb_name[H5FD_MEM_LHEAP] = sv[H5FD_MEM_LHEAP]; \ + memb_addr[H5FD_MEM_LHEAP] = HADDR_MAX*2/3; \ + sprintf(sv[H5FD_MEM_OHDR], "%%s-%c.h5", 'o'); \ + memb_name[H5FD_MEM_OHDR] = sv[H5FD_MEM_OHDR]; \ + memb_addr[H5FD_MEM_OHDR] = HADDR_MAX*5/6; \ +} /* + *------------------------------------------------------------------------- * Verify that the file's free-space manager(s) are persistent for a multi-file + *------------------------------------------------------------------------- */ static unsigned -test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) +test_mf_fs_persist_multi(void) { - hid_t file = -1; /* File ID */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type, stype, btype, gtype; /* File allocation type */ - H5FS_stat_t fs_stat; /* Information for free-space manager */ - haddr_t addr1, addr2, addr3, addr4; /* File allocation type */ - haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_SUPER */ - haddr_t baddr1, baddr2, baddr3, baddr4; /* File address for H5FD_MEM_DRAW */ - haddr_t gaddr1, gaddr2; /* File address for H5FD_MEM_GHEAP */ - haddr_t tmp_addr; /* Temporary variable for address */ - H5FS_section_info_t *node; /* Free space section node */ - htri_t node_found = FALSE; /* Indicate section is in free-space */ - - TESTING("file's free-space managers are persistent for multi-file"); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl = -1; /* File access property list ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type, stype, btype, gtype; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4; /* File allocation type */ + haddr_t saddr1, saddr2, saddr3, saddr4; /* File address for H5FD_MEM_SUPER */ + haddr_t baddr1, baddr2, baddr3, baddr4; /* File address for H5FD_MEM_DRAW */ + haddr_t gaddr1, gaddr2; /* File address for H5FD_MEM_GHEAP */ + haddr_t tmp_addr; /* Temporary variable for address */ + H5FS_section_info_t *node; /* Free space section node */ + htri_t node_found = FALSE; /* Indicate section is in free-space */ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; /* Memory usage map */ + hid_t memb_fapl[H5FD_MEM_NTYPES]; /* Member access properties */ + char sv[H5FD_MEM_NTYPES][64]; /* Name generators */ + const char *memb_name[H5FD_MEM_NTYPES]; /* Name generators */ + haddr_t memb_addr[H5FD_MEM_NTYPES]; /* Member starting address */ + + + TESTING("File's free-space managers are persistent for multi-file"); + + /* for now, we don't support persistant free space managers + * with the multi file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ + + /* for now, we don't support persistant free space managers + * with the multi file driver. + */ + SKIPPED(); + HDfprintf(stdout, " Persistant FSMs disabled in multi file driver.\n"); + return 0; /* <========== note return */ + + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR + + MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) + + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, TRUE) < 0) + TEST_ERROR; /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6682,43 +6474,43 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) /* Allocate 4 blocks of type H5FD_MEM_SUPER */ type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR /* Put block #1, #3 into H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR /* Allocate 4 blocks of type H5FD_MEM_DRAW */ stype = H5FD_MEM_DRAW; - if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr3 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (saddr4 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR /* Put block #1, #3 into H5FD_MEM_DRAW free-space manager */ - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6730,7 +6522,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6738,23 +6530,23 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR - /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #2 still in free-space */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager; block #3 still in free-space */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != addr1) - TEST_ERROR + TEST_ERROR /* Verify that the free-space manager for H5FD_MEM_DRAW is there */ if(!H5F_addr_defined(f->shared->fs_addr[stype])) TEST_ERROR /* Start up H5FD_MEM_DRAW free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, stype) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)stype) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6762,45 +6554,45 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #1 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR if(tmp_addr != saddr1) - TEST_ERROR + TEST_ERROR /* Retrieve blocks #3 from H5FD_MEM_DRAW free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != saddr3) - TEST_ERROR + TEST_ERROR /* Allocate 4 blocks of type H5FD_MEM_BTREE */ btype = H5FD_MEM_BTREE; - if(HADDR_UNDEF == (baddr1 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr2 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr3 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (baddr4 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr1 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr2 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr3 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (baddr4 = H5MF_alloc(f, btype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR /* Put block #1 & #3 into H5FD_MEM_BTREE free-space manager */ - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, btype, H5AC_ind_read_dxpl_id, baddr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6812,7 +6604,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6820,14 +6612,14 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < TEST_BLOCK_SIZE3) - TEST_ERROR + if(fs_stat.tot_space < TBLOCK_SIZE3) + TEST_ERROR /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR if(tmp_addr != addr3) - TEST_ERROR + TEST_ERROR /* Verify that the free-space manager for H5FD_MEM_DRAW is not there */ if(H5F_addr_defined(f->shared->fs_addr[stype])) @@ -6838,7 +6630,7 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) TEST_ERROR /* Start up H5FD_MEM_BTREE free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, btype) < 0) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)btype) < 0) FAIL_STACK_ERROR /* Get free-space info */ @@ -6846,27 +6638,27 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < (TEST_BLOCK_SIZE1+TEST_BLOCK_SIZE3)) - TEST_ERROR + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3)) + TEST_ERROR if(fs_stat.serial_sect_count < 2) - TEST_ERROR + TEST_ERROR /* Allocate 2 blocks of type H5FD_MEM_GHEAP */ gtype = H5FD_MEM_GHEAP; - if(HADDR_UNDEF == (gaddr2 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (gaddr1 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR + if(HADDR_UNDEF == (gaddr2 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (gaddr1 = H5MF_alloc(f, gtype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR /* Put block #2 into H5FD_MEM_GHEAP free-space manager */ - if(H5MF_xfree(f, gtype, H5AC_ind_read_dxpl_id, gaddr2, (hsize_t)TEST_BLOCK_SIZE2) < 0) - FAIL_STACK_ERROR + if(H5MF_xfree(f, gtype, H5AC_ind_read_dxpl_id, gaddr2, (hsize_t)TBLOCK_SIZE2) < 0) + FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ @@ -6875,39 +6667,44 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) /* If H5FD_MEM_SUPER is there, should not find block #1 & #3 */ if(H5F_addr_defined(f->shared->fs_addr[type])) { - /* Start up H5FD_MEM_SUPER free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) - FAIL_STACK_ERROR - - if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE1, (H5FS_section_info_t **)&node)) < 0) - FAIL_STACK_ERROR - if(node_found) TEST_ERROR - - if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], - (hsize_t)TEST_BLOCK_SIZE3, (H5FS_section_info_t **)&node)) < 0) - FAIL_STACK_ERROR - if(node_found) TEST_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)type) < 0) + FAIL_STACK_ERROR + + if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], + (hsize_t)TBLOCK_SIZE1, (H5FS_section_info_t **)&node)) < 0) + FAIL_STACK_ERROR + if(node_found) TEST_ERROR + + if((node_found = H5FS_sect_find(f, H5AC_ind_read_dxpl_id, f->shared->fs_man[type], + (hsize_t)TBLOCK_SIZE3, (H5FS_section_info_t **)&node)) < 0) + FAIL_STACK_ERROR + if(node_found) TEST_ERROR } /* Verify that the H5FD_MEM_GHEAP free-space manager is there */ if(!H5F_addr_defined(f->shared->fs_addr[gtype])) - TEST_ERROR + TEST_ERROR /* Start up H5FD_MEM_GHEAP free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, gtype) < 0) - FAIL_STACK_ERROR + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)gtype) < 0) + FAIL_STACK_ERROR /* Get free-space info */ if(H5FS_stat_info(f, f->shared->fs_man[gtype], &fs_stat) < 0) FAIL_STACK_ERROR /* Verify free-space info */ - if(fs_stat.tot_space < TEST_BLOCK_SIZE2) - TEST_ERROR + if(fs_stat.tot_space < TBLOCK_SIZE2) + TEST_ERROR + /* Closing */ if(H5Fclose(file) < 0) FAIL_STACK_ERROR + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR PASSED() @@ -6915,460 +6712,540 @@ test_mf_fs_multi(hid_t fapl_new, hid_t fcpl) error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(file); + H5Pclose(fcpl); + H5Pclose(fapl); } H5E_END_TRY; return(1); -} /* test_mf_fs_multi() */ - -#define MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) { \ - H5FD_mem_t mt; \ - HDmemset(memb_map, 0, sizeof memb_map); \ - HDmemset(memb_fapl, 0, sizeof memb_fapl); \ - HDmemset(memb_name, 0, sizeof memb_name); \ - HDmemset(memb_addr, 0, sizeof memb_addr); \ - HDmemset(sv, 0, sizeof sv); \ - for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { \ - memb_map[mt] = H5FD_MEM_SUPER; \ - memb_fapl[mt] = H5P_DEFAULT; \ - } \ - memb_map[H5FD_MEM_BTREE] = H5FD_MEM_BTREE; \ - memb_map[H5FD_MEM_DRAW] = H5FD_MEM_DRAW; \ - memb_map[H5FD_MEM_GHEAP] = H5FD_MEM_GHEAP; \ - memb_map[H5FD_MEM_LHEAP] = H5FD_MEM_LHEAP; \ - sprintf(sv[H5FD_MEM_SUPER], "%%s-%c.h5", 's'); \ - memb_name[H5FD_MEM_SUPER] = sv[H5FD_MEM_SUPER]; \ - memb_addr[H5FD_MEM_SUPER] = 0; \ - sprintf(sv[H5FD_MEM_BTREE], "%%s-%c.h5", 'b'); \ - memb_name[H5FD_MEM_BTREE] = sv[H5FD_MEM_BTREE]; \ - memb_addr[H5FD_MEM_BTREE] = HADDR_MAX/6; \ - sprintf(sv[H5FD_MEM_DRAW], "%%s-%c.h5", 'r'); \ - memb_name[H5FD_MEM_DRAW] = sv[H5FD_MEM_DRAW]; \ - memb_addr[H5FD_MEM_DRAW] = HADDR_MAX/3; \ - sprintf(sv[H5FD_MEM_GHEAP], "%%s-%c.h5", 'g'); \ - memb_name[H5FD_MEM_GHEAP] = sv[H5FD_MEM_GHEAP]; \ - memb_addr[H5FD_MEM_GHEAP] = HADDR_MAX/2; \ - sprintf(sv[H5FD_MEM_LHEAP], "%%s-%c.h5", 'l'); \ - memb_name[H5FD_MEM_LHEAP] = sv[H5FD_MEM_LHEAP]; \ - memb_addr[H5FD_MEM_LHEAP] = HADDR_MAX*2/3; \ - sprintf(sv[H5FD_MEM_OHDR], "%%s-%c.h5", 'o'); \ - memb_name[H5FD_MEM_OHDR] = sv[H5FD_MEM_OHDR]; \ - memb_addr[H5FD_MEM_OHDR] = HADDR_MAX*5/6; \ -} +} /* test_mf_fs_persist_multi() */ /* - * Tests to verify that file's free-space managers are persistent or going away - * for different drivers. + *------------------------------------------------------------------------- + * Verify that the file's free-space persists where there are free sections in the manager + *------------------------------------------------------------------------- */ static unsigned -test_mf_fs_drivers(hid_t fapl) +test_mf_fs_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t fcpl = -1; /* file creation property list */ - hid_t fapl_new = -1; /* copy of file access property list */ - hid_t fapl2 = -1; /* copy of file access property list */ - unsigned new_format; /* To use new library format or not */ - unsigned ret = 0; /* return value */ - - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; /* Memory usage map */ - hid_t memb_fapl[H5FD_MEM_NTYPES]; /* Member access properties */ - char sv[H5FD_MEM_NTYPES][64]; /* Name generators */ - const char *memb_name[H5FD_MEM_NTYPES]; /* Name generators */ - haddr_t memb_addr[H5FD_MEM_NTYPES]; /* Member starting address */ - - /* Create a non-standard file-creation template */ - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - if(H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0) < 0) - TEST_ERROR - - /* Copy the file access property list */ - if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR - - /* Set the "use the latest version of the format" bounds for creating objects in the file */ - if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) - TEST_ERROR - - /* Test with old and new format */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { - - if(new_format) - HDputs("Testing the following tests for free-space managers with new library format..."); - else - HDputs("Testing the following tests for free-space managers with old library format..."); - - /* SEC2 */ - HDputs("Testing free-space manager(s) with sec2 driver"); - - if((fapl_new = H5Pcopy(new_format ? fapl2 : fapl)) < 0) TEST_ERROR - - if(H5Pset_fapl_sec2(fapl_new) < 0) - FAIL_STACK_ERROR - - ret += test_mf_fs_gone(fapl_new, fcpl); - ret += test_mf_fs_persist(fapl_new, fcpl); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl2 = -1; /* File access property list ID */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FD_mem_t tt; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ + haddr_t tmp_addr; /* Temporary variable for address */ + + if(new_format) + TESTING("File's free-space is persistent with new library format") + else + TESTING("File's free-space is persistent with old library format") + + if(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")) { + + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR + + if(new_format) { + /* Latest format */ + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + /* Set to paged aggregation and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + TEST_ERROR + } else { + /* Setting: aggregation with persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + TEST_ERROR + } - h5_clean_files(FILENAME, fapl_new); + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR - /* STDIO */ - HDputs("Testing free-space managers with stdio driver"); + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_stdio(fapl_new) < 0) - FAIL_STACK_ERROR + /* Allocate 6 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR - ret += test_mf_fs_gone(fapl_new, fcpl); - ret += test_mf_fs_persist(fapl_new, fcpl); + /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - /* CORE */ - HDputs("Testing free-space managers with core driver"); + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR - /* create fapl to be a "core" file */ - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_core(fapl_new, (size_t)CORE_INCREMENT, TRUE) < 0) - FAIL_STACK_ERROR + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - ret += test_mf_fs_gone(fapl_new, fcpl); - ret += test_mf_fs_persist(fapl_new, fcpl); + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE6, (H5F_mem_page_t *)&tt); - h5_clean_files(FILENAME, fapl_new); + /* Verify that H5FD_MEM_SUPER free-space manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR - /* FAMILY */ - HDputs("Testing free-space managers with family driver"); + /* Since we are about to open a self referential free space + * manager prior to the first file space allocation / deallocation + * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid + * assertion failures on the first file space alloc / dealloc. + */ + if((f->shared->first_alloc_dealloc) && + (SUCCEED != + H5MF_tidy_self_referential_fsm_hack(f, H5AC_ind_read_dxpl_id))) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_family(fapl_new, (hsize_t)FAMILY_SIZE, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR + /* Start up H5FD_MEM_SUPER free-space manager */ + if(!(f->shared->fs_man[tt])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)tt) < 0) + FAIL_STACK_ERROR - ret += test_mf_fs_persist(fapl_new, fcpl); + /* Get info for free-space manager */ + if(H5FS_stat_info(f, f->shared->fs_man[tt], &fs_stat) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + /* Verify free-space info */ + if(fs_stat.tot_space < (TBLOCK_SIZE1+TBLOCK_SIZE3+TBLOCK_SIZE5)) + TEST_ERROR - /* SPLIT */ - HDputs("Testing free-space managers with split driver"); + if(fs_stat.serial_sect_count < 3) + TEST_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_split(fapl_new, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT)<0) - FAIL_STACK_ERROR + /* Retrieve block #3 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(tmp_addr != addr3) + TEST_ERROR - ret += test_mf_fs_persist(fapl_new, fcpl); - ret += test_mf_fs_split(fapl_new, fcpl); + /* Retrieve block #1 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(tmp_addr != addr1) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - /* MULTI */ - HDputs("Testing free-space managers with multi driver"); + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + FAIL_STACK_ERROR - MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl, memb_name, memb_addr, TRUE) < 0) - TEST_ERROR; + /* Verify that H5FD_MEM_SUPER free-space manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR - ret += test_mf_fs_multi(fapl_new, fcpl); + /* Retrieve block #5 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(tmp_addr != addr5) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR - } /* end for new_format */ + PASSED() - if(H5Pclose(fcpl) < 0) - FAIL_STACK_ERROR - if(H5Pclose(fapl2) < 0) - FAIL_STACK_ERROR + } else { + SKIPPED(); + puts(" Current VFD doesn't support persisting free-space or paged aggregation strategy"); + } - return(ret); + return(0); error: H5E_BEGIN_TRY { + H5Fclose(file); H5Pclose(fcpl); H5Pclose(fapl2); - H5Pclose(fapl_new); } H5E_END_TRY; return(1); -} /* test_mf_fs_drivers() */ - +} /* test_mf_fs_persist() */ /* - * Verify that file space management performs according to the - * file space strategy and free space threshold as specified. + *------------------------------------------------------------------------- + * Verify free-space are merged/shrunk away + *------------------------------------------------------------------------- */ static unsigned -test_filespace_strategy_threshold(hid_t fapl_new) +test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t file = -1; /* File ID */ - hid_t fcpl = -1; /* File creation property list template */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ - haddr_t tmp_addr; /* Temporary variable for address */ - H5F_file_space_type_t fs_type; /* File space handling strategy */ - hsize_t fs_threshold; /* Free space section threshold */ - hsize_t tot_space, saved_tot_space; /* Total amount of free space */ - hsize_t tot_sect_count, saved_tot_sect_count; /* # of free-space sections */ - - TESTING("file space strategy and threshold"); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t fapl2 = -1; /* File access property list */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + haddr_t addr1, addr2, addr3, addr4; /* File address for H5FD_MEM_SUPER */ + H5FD_mem_t fs_type; + hbool_t contig_addr_vfd; + hbool_t ran_H5MF_tidy_self_referential_fsm_hack = FALSE; + + if(new_format) + TESTING("File's free-space is going away with new library format") + else + TESTING("File's free-space is going away with old library format") + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - for(fs_threshold = 0; fs_threshold <= TEST_THRESHOLD10; fs_threshold++) { - - for(fs_type = H5F_FILE_SPACE_ALL_PERSIST; fs_type < H5F_FILE_SPACE_NTYPES; H5_INC_ENUM(H5F_file_space_type_t, fs_type)) { - - /* Create file-creation template */ - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set default file space information */ - if(H5Pset_file_space(fcpl, fs_type, fs_threshold) < 0) - FAIL_STACK_ERROR - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Allocate 6 blocks */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - - /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - - /* Retrieve the total amount of free space and # of free-space sections */ - if(f->shared->fs_man[type] && - H5FS_sect_stats(f->shared->fs_man[type], &saved_tot_space, &saved_tot_sect_count) < 0) - FAIL_STACK_ERROR - - /* H5F_FILE_SPACE_AGGR_VFD and H5F_FILE_SPACE_VFD: should not have free-space manager */ - if(fs_type > H5F_FILE_SPACE_ALL && f->shared->fs_man[type]) - TEST_ERROR + if(contig_addr_vfd) { - /* Close the file */ - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - switch(fs_type) { - case H5F_FILE_SPACE_ALL_PERSIST: - if(fs_threshold <= TEST_BLOCK_SIZE5) { - if(!H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - /* Open the free-space manager */ - if(H5MF__alloc_open(f, H5AC_ind_read_dxpl_id, type) < 0) - FAIL_STACK_ERROR - - /* Retrieve the total amount of free space and # of free-space sections */ - if(H5FS_sect_stats(f->shared->fs_man[type], &tot_space, &tot_sect_count) < 0) - FAIL_STACK_ERROR - - /* Verify that tot_space should be >= saved_tot_space */ - /* Verify that tot_sect_count should be >= saved_tot_sect_count */ - if(tot_space < saved_tot_space || tot_sect_count < saved_tot_sect_count) - TEST_ERROR - - /* Retrieve block #5 from H5FD_MEM_SUPER free-space manager */ - if(HADDR_UNDEF == - (tmp_addr = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - - /* Should be the same as before */ - if(tmp_addr != addr5) - TEST_ERROR - } else if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - break; - - case H5F_FILE_SPACE_ALL: - case H5F_FILE_SPACE_AGGR_VFD: - case H5F_FILE_SPACE_VFD: - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - break; - - case H5F_FILE_SPACE_DEFAULT: - case H5F_FILE_SPACE_NTYPES: - default: - TEST_ERROR - break; - } /* end switch */ + /* File creation property list template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR - /* Closing */ - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - if(H5Pclose(fcpl) < 0) - FAIL_STACK_ERROR + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR - } /* end for fs_type */ - } /* end for fs_threshold */ + if(new_format) + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR - PASSED() + /* Set to aggregation and persisting free-space */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); + + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Allocate 4 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + + /* Put block #1, #3 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + + /* Retrieve block #1, #3 from H5FD_MEM_SUPER free-space manager */ + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE4, (H5F_mem_page_t *)&fs_type); + + /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ + if(H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Put block #3 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Verify that H5FD_MEM_SUPER free-space manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Since we are about to open a self referential free space + * manager prior to the first file space allocation / deallocation + * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid + * assertion failures on the first file space alloc / dealloc. + */ + if(f->shared->first_alloc_dealloc){ + if(SUCCEED!=H5MF_tidy_self_referential_fsm_hack(f,H5AC_ind_read_dxpl_id)) + FAIL_STACK_ERROR + ran_H5MF_tidy_self_referential_fsm_hack = TRUE; + } + + /* Start up H5FD_MEM_SUPER free-space manager */ + if(!(f->shared->fs_man[fs_type])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, (H5F_mem_page_t)fs_type) < 0) + FAIL_STACK_ERROR + + /* Get info for H5FD_MEM_SUPER free-space manager */ + if(H5FS_stat_info(f, f->shared->fs_man[fs_type], &fs_stat) < 0) + FAIL_STACK_ERROR + + /* if we ran H5MF_tidy_self_referential_fsm_hack(), the + * H5FD_MEM_SUPER free space manager must be floating. + * Thus fs_stat.addr must be undefined. + */ + if((!ran_H5MF_tidy_self_referential_fsm_hack) && + (!H5F_addr_defined(fs_stat.addr))) + TEST_ERROR + + if(fs_stat.tot_space < TBLOCK_SIZE3) + TEST_ERROR + + /* Put block #4 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TBLOCK_SIZE4) < 0) + FAIL_STACK_ERROR + + /* The H5FD_MEM_SUPER free-space manager will go away at H5MF_close() */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Verify that the H5FD_MEM_SUPER free-space manager is not there */ + if(H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Closing */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support persistent free-space manager"); + } return(0); error: H5E_BEGIN_TRY { - H5Pclose(fcpl); H5Fclose(file); + H5Pclose(fcpl); + H5Pclose(fapl2); } H5E_END_TRY; return(1); -} /* test_filespace_strategy_threshold() */ +} /* test_mf_fs_gone() */ /* - * Verify section is merged/shrunk away for - * H5F_FILE_SPACE_ALL_PERSIST and H5F_FILE_SPACE_ALL strategy. + *------------------------------------------------------------------------- + * Verify that free-space persist with combinations of + * file space strategy and free space threshold as specified. + *------------------------------------------------------------------------- */ static unsigned -test_filespace_gone(hid_t fapl_new) +test_mf_strat_thres_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t file = -1; /* File ID */ - hid_t fcpl = -1; /* File creation propertly list template */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_t *f = NULL; /* Internal file object pointer */ - H5FD_mem_t type; /* File allocation type */ - haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ - H5F_file_space_type_t fs_type; /* File space handling strategy */ - hsize_t fs_threshold; /* Free space section threshold */ - frspace_state_t state; /* State of free space manager */ - - TESTING("file space merge/shrink for section size < threshold"); + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list template */ + hid_t fapl2 = -1; /* File access property list template */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FD_mem_t tt; /* File allocation type */ + haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ + H5F_fspace_strategy_t fs_type; /* File space handling strategy */ + hsize_t fs_threshold; /* Free-space section threshold */ + unsigned fs_persist; /* To persist free-space or not */ + hbool_t contig_addr_vfd; + + if(new_format) + TESTING("File space strategy/persisting/threshold with new library format") + else + TESTING("File space strategy/persisting/threshold with old library format") + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); + /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_new, filename, sizeof(filename)); - - /* Set free-space threshold */ - fs_threshold = TEST_THRESHOLD3; - - for(fs_type = H5F_FILE_SPACE_ALL_PERSIST; fs_type <= H5F_FILE_SPACE_ALL; H5_INC_ENUM(H5F_file_space_type_t, fs_type)) { - /* Create file-creation template */ - if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set default file space information */ - if(H5Pset_file_space(fcpl, fs_type, fs_threshold) < 0) - FAIL_STACK_ERROR - - /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* Allocate 6 blocks */ - type = H5FD_MEM_SUPER; - if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE1))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE2))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE3))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE4))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE5))) - FAIL_STACK_ERROR - if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE6))) - FAIL_STACK_ERROR - - /* Put block #3, #5 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TEST_BLOCK_SIZE3) < 0) - FAIL_STACK_ERROR - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TEST_BLOCK_SIZE5) < 0) - FAIL_STACK_ERROR - - HDmemset(&state, 0, sizeof(frspace_state_t)); - state.tot_space += TEST_BLOCK_SIZE3 + TEST_BLOCK_SIZE5; - state.tot_sect_count += 2; - state.serial_sect_count += 2; - - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - /* section #2 is less than threshold but is merged into section #3 */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TEST_BLOCK_SIZE2) < 0) - FAIL_STACK_ERROR - - state.tot_space += TEST_BLOCK_SIZE2; - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TEST_BLOCK_SIZE4) < 0) - FAIL_STACK_ERROR - - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr6, (hsize_t)TEST_BLOCK_SIZE6) < 0) - FAIL_STACK_ERROR - - /* all sections should be shrunk away except section #1 */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - /* section #1 is less than threshold but is shrunk away */ - if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE1) < 0) - FAIL_STACK_ERROR - - /* free-space manager should be empty */ - HDmemset(&state, 0, sizeof(frspace_state_t)); - if(check_stats(f, f->shared->fs_man[type], &state)) - TEST_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - /* Re-open the file */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) - FAIL_STACK_ERROR - - /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) - FAIL_STACK_ERROR - - /* free-space manager should be empty */ - if(H5F_addr_defined(f->shared->fs_addr[type])) - TEST_ERROR - - if(H5Fclose(file) < 0) - FAIL_STACK_ERROR - - if(H5Pclose(fcpl) < 0) - FAIL_STACK_ERROR - - } /* end for fs_type */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) + FAIL_STACK_ERROR + + if(new_format) + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + + for(fs_threshold = 0; fs_threshold <= TEST_THRESHOLD10; fs_threshold++) { + + /* Testing for H5F_FSPACE_STRATEGY_FSM_AGGR and H5F_FSPACE_STRATEGY_PAGE strategies only */ + for(fs_type = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_type < H5F_FSPACE_STRATEGY_AGGR; H5_INC_ENUM(H5F_fspace_strategy_t, fs_type)) { + + if(!contig_addr_vfd && (fs_persist || fs_type == H5F_FSPACE_STRATEGY_PAGE)) + continue; + + /* Create file-creation template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set default file space information */ + if(H5Pset_file_space_strategy(fcpl, fs_type, (hbool_t)fs_persist, fs_threshold) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Allocate 6 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR + + /* Put block #1, #3, #5 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR + + + /* Close the file */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE6, (H5F_mem_page_t *)&tt); + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + if(f->shared->fs_persist) { + hssize_t nsects; /* # of free-space sections */ + int i; /* local index variable */ + H5F_sect_info_t *sect_info; /* array to hold the free-space information */ + + /* Get the # of free-space sections in the file */ + if((nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, NULL)) < 0) + FAIL_STACK_ERROR + + /* Verify no free-space sections */ + /* paged aggregation has 1 section for last_small */ + if(fs_threshold > TBLOCK_SIZE5 && nsects && fs_type != H5F_FSPACE_STRATEGY_PAGE) + TEST_ERROR + + if(nsects) { + /* Allocate storage for the free space section information */ + sect_info = (H5F_sect_info_t *)HDcalloc((size_t)nsects, sizeof(H5F_sect_info_t)); + + H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)nsects, sect_info); + + /* Verify the size of free-space sections */ + for(i = 0; i < nsects; i++) + if(sect_info[i].size < fs_threshold) + TEST_ERROR + if(sect_info) + HDfree(sect_info); + } + } else { + if(H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR + } + + /* Closing */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + } /* end for fs_type */ + } /* end for fs_threshold */ + } /* end for fs_persist */ + + if(H5Pclose(fapl2) < 0) + FAIL_STACK_ERROR PASSED() @@ -7377,141 +7254,200 @@ test_filespace_gone(hid_t fapl_new) error: H5E_BEGIN_TRY { H5Pclose(fcpl); + H5Pclose(fapl2); H5Fclose(file); } H5E_END_TRY; return(1); -} /* test_filespace_gone() */ +} /* test_mf_strat_thres_persist() */ /* - * Tests to verify file space management for different drivers. + *------------------------------------------------------------------------- + * Verify free-space are merged/shrunk away with file space settings: + * --strategy, persist/not persist file space + *------------------------------------------------------------------------- */ static unsigned -test_filespace_drivers(hid_t fapl) +test_mf_strat_thres_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) { - hid_t fapl_new = -1; /* copy of file access property list */ - hid_t fapl2 = -1; /* copy of file access property list */ - unsigned new_format; /* Using library new format or not */ - unsigned ret = 0; /* return value */ + hid_t file = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list template */ + hid_t fapl2 = -1; /* File access property list template */ + char filename[FILENAME_LEN]; /* Filename to use */ + H5F_t *f = NULL; /* Internal file object pointer */ + H5FD_mem_t type; /* File allocation type */ + H5FD_mem_t tt; /* File allocation type */ + haddr_t addr1, addr2, addr3, addr4, addr5, addr6; /* File address for H5FD_MEM_SUPER */ + H5F_fspace_strategy_t fs_type; /* File space handling strategy */ + unsigned fs_persist; /* To persist free-space or not */ + H5FS_stat_t fs_state; /* Information for free-space manager */ + H5FS_stat_t fs_state_zero; /* Information for free-space manager */ + hbool_t contig_addr_vfd; + + if(new_format) + TESTING("File space merge/shrink for section size < threshold with new library format") + else + TESTING("File space merge/shrink for section size < threshold with old library format") + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; /* Memory usage map */ - hid_t memb_fapl[H5FD_MEM_NTYPES]; /* Member access properties */ - char sv[H5FD_MEM_NTYPES][64]; /* Name generators */ - const char *memb_name[H5FD_MEM_NTYPES]; /* Name generators */ - haddr_t memb_addr[H5FD_MEM_NTYPES]; /* Member starting address */ - /* Copy the file access property list */ - if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - /* Set the "use the latest version of the format" bounds for creating objects in the file */ - if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) - TEST_ERROR + HDmemset(&fs_state_zero, 0, sizeof(H5FS_stat_t)); - /* Test with old and new format */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { + /* Copy the file access property list */ + if((fapl2 = H5Pcopy(fapl)) < 0) + FAIL_STACK_ERROR - if(new_format) - HDputs("Testing the following tests for file space management with new library format..."); - else - HDputs("Testing the following tests for file space management with old library format..."); + if(new_format) + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR - /* SEC2 */ - HDputs("Testing file space management with sec2 driver"); + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + /* Testing for H5F_FSPACE_STRATEGY_FSM_AGGR and H5F_FSPACE_STRATEGY_PAGE strategies only */ + for(fs_type = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_type < H5F_FSPACE_STRATEGY_AGGR; H5_INC_ENUM(H5F_fspace_strategy_t, fs_type)) { - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_sec2(fapl_new) < 0) - FAIL_STACK_ERROR + /* Skip for multi/split driver: persisting free-space or paged aggregation strategy */ + if(!contig_addr_vfd && (fs_persist || fs_type == H5F_FSPACE_STRATEGY_PAGE)) + continue; - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Clear out free-space statistics */ + HDmemset(&fs_state, 0, sizeof(H5FS_stat_t)); - h5_clean_files(FILENAME, fapl_new); + /* Create file-creation template */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR - /* STDIO */ - HDputs("Testing file space management with stdio driver"); + /* Set default file space information */ + if(H5Pset_file_space_strategy(fcpl, fs_type, fs_persist, (hsize_t)TEST_THRESHOLD3) < 0) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_stdio(fapl_new) < 0) - FAIL_STACK_ERROR + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) + FAIL_STACK_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + /* Allocate 6 blocks */ + type = H5FD_MEM_SUPER; + if(HADDR_UNDEF == (addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr2 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE2))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr4 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr5 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5))) + FAIL_STACK_ERROR + if(HADDR_UNDEF == (addr6 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6))) + FAIL_STACK_ERROR - /* CORE */ - HDputs("Testing file space management with core driver"); + H5MF_alloc_to_fs_type(f, type, TBLOCK_SIZE6, (H5F_mem_page_t *)&tt); - /* create fapl to be a "core" file */ - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_core(fapl_new, (size_t)CORE_INCREMENT, TRUE) < 0) - FAIL_STACK_ERROR + /* For paged aggregation, the section in the page at EOF for small meta fs is not shrunk away */ + if(fs_type == H5F_FSPACE_STRATEGY_PAGE) { + if(H5FS_stat_info(f, f->shared->fs_man[tt], &fs_state) < 0) + FAIL_STACK_ERROR + } - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Put block #3, #5 to H5FD_MEM_SUPER free-space manager */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE3) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr5, (hsize_t)TBLOCK_SIZE5) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + fs_state.tot_space += TBLOCK_SIZE3 + TBLOCK_SIZE5; + fs_state.tot_sect_count += 2; + fs_state.serial_sect_count += 2; - /* FAMILY */ - HDputs("Testing file space managers with family driver"); + if(check_stats(f, f->shared->fs_man[tt], &fs_state)) + TEST_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_family(fapl_new, (hsize_t)FAMILY_SIZE, H5P_DEFAULT) < 0) - FAIL_STACK_ERROR + /* section #2 is less than threshold but is merged into section #3 */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE2) < 0) + FAIL_STACK_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + fs_state.tot_space += TBLOCK_SIZE2; + if(check_stats(f, f->shared->fs_man[tt], &fs_state)) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr4, (hsize_t)TBLOCK_SIZE4) < 0) + FAIL_STACK_ERROR + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr6, (hsize_t)TBLOCK_SIZE6) < 0) + FAIL_STACK_ERROR - /* SPLIT */ - HDputs("Testing file space managers with split driver"); + /* For paged aggregation, the sections in the page at EOF for small meta fs are merged but are not shrunk away */ + if(fs_type == H5F_FSPACE_STRATEGY_PAGE) { + fs_state.tot_sect_count = fs_state.serial_sect_count = 1; + fs_state.tot_space += (TBLOCK_SIZE4 + TBLOCK_SIZE6); + } - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - FAIL_STACK_ERROR - if(H5Pset_fapl_split(fapl_new, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT)<0) - FAIL_STACK_ERROR + /* For old format: the sections at EOF are shrunk away */ + if(check_stats(f, f->shared->fs_man[tt], (fs_type == H5F_FSPACE_STRATEGY_PAGE) ? &fs_state:&fs_state_zero)) + TEST_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* section #1 is less than threshold but is shrunk away */ + if(H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE1) < 0) + FAIL_STACK_ERROR - h5_clean_files(FILENAME, fapl_new); + /* For paged aggregation, the section in the page at EOF for small meta fs is not shrunk away */ + if(fs_type == H5F_FSPACE_STRATEGY_PAGE) + fs_state.tot_space += TBLOCK_SIZE1; - /* MULTI */ - HDputs("Testing file space managers with multi driver"); + /* For old format: the sections at EOF are shrunk away */ + if(check_stats(f, f->shared->fs_man[tt], (fs_type == H5F_FSPACE_STRATEGY_PAGE) ? &fs_state : &fs_state_zero)) + TEST_ERROR - MULTI_SETUP(memb_map, memb_fapl, memb_name, memb_addr, sv) + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - if((fapl_new = H5Pcopy(new_format?fapl2:fapl)) < 0) - TEST_ERROR - if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl, memb_name, memb_addr, TRUE) < 0) - TEST_ERROR; + /* Re-open the file */ + if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl2)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR - ret += test_filespace_strategy_threshold(fapl_new); - ret += test_filespace_gone(fapl_new); + /* Free-space manager should be empty */ + if(!(fs_type == H5F_FSPACE_STRATEGY_PAGE && fs_persist)) + if(H5F_addr_defined(f->shared->fs_addr[tt])) + TEST_ERROR - h5_clean_files(FILENAME, fapl_new); + /* Closing */ + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR - } /* end for new_format */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + } /* end for fs_type */ + } /* end for fs_persist */ - if (H5Pclose(fapl2) < 0) + if(H5Pclose(fapl2) < 0) FAIL_STACK_ERROR - return(ret); + PASSED() + + return(0); error: H5E_BEGIN_TRY { - H5Pclose(fapl_new); + H5Pclose(fcpl); H5Pclose(fapl2); + H5Fclose(file); } H5E_END_TRY; return(1); -} /* test_filespace_drivers() */ +} /* test_mf_strat_thres_gone() */ /* + *------------------------------------------------------------------------- * To verify that file space is allocated from the corresponding free-space manager * because H5FD_FLMAP_DICHOTOMY is used as the default free-list mapping. * @@ -7530,80 +7466,1388 @@ error: * * (9) Allocate the third block (size 30) of type H5FD_MEM_SUPER * (10) Verify that this third block is not freed block from (8) + *------------------------------------------------------------------------- */ static unsigned -test_dichotomy(const char *env_h5_drvr, hid_t fapl) +test_dichotomy(hid_t fapl) { hid_t file = -1; /* File ID */ char filename[FILENAME_LEN]; /* Filename to use */ H5F_t *f = NULL; /* Internal file object pointer */ H5FD_mem_t type, stype; haddr_t addr1, addr3, saddr1, saddr2; - hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ TESTING("Allocation from raw or metadata free-space manager"); - /* Skip test when using VFDs that don't use the metadata aggregator */ + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* Create the file to work on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + + /* Allocate the first block of type H5FD_MEM_SUPER */ + type = H5FD_MEM_SUPER; + addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Allocate the second block of type H5FD_MEM_SUPER */ + H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Allocate the first block of type H5FD_MEM_DRAW */ + stype = H5FD_MEM_DRAW; + saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Free the first block of type H5FD_MEM_SUPER */ + H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE30); + + /* Allocate the second block of type H5FD_MEM_DRAW */ + saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Verify that saddr1 is not addr1 */ + if(saddr2 == addr1) TEST_ERROR + + /* Free the first block of type H5FD_MEM_DRAW */ + H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + + /* Allocate the third block of type H5FD_MEM_SUPER */ + addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + + /* Verify that addr3 is not saddr1 */ + if(addr3 == saddr1) TEST_ERROR + + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + PASSED() + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(file); + } H5E_END_TRY; + return(1); +} /* test_dichotomy() */ + +/* + *------------------------------------------------------------------------- + * set_multi_split(): + * Internal routine to set up page-aligned address space for multi/split driver + * when testing paged aggregation. + *------------------------------------------------------------------------- + */ +static int +set_multi_split(hid_t fapl, hsize_t pagesize, hbool_t multi, hbool_t split) +{ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + HDassert(multi || split); + + HDmemset(memb_name, 0, sizeof memb_name); + + /* Get current split settings */ + if(H5Pget_fapl_multi(fapl, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + if(split) { + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + pagesize - 1) / pagesize) * pagesize; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + pagesize - 1) / pagesize) * pagesize; + } else { + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + memb_addr[mt] = ((memb_addr[mt] + pagesize - 1) / pagesize) * pagesize; + } /* end else */ + + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); + + return 0; + +error: + return(-1); +} /* set_multi_split() */ + +/*------------------------------------------------------------------------- + * Function: test_page_alloc_xfree + * + * Purpose: To verify allocations and de-allocations for large/small + * sections are done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + hid_t fapl_new = -1; /* File access property list ID */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr2, addr3; /* Addresses for small meta data blocks */ + haddr_t saddr1; /* Addresses for small raw data blocks */ + haddr_t gaddr1; /* Addresses for large data blocks */ + hbool_t split = FALSE, multi = FALSE; + char filename[FILENAME_LEN]; /* Filename to use */ + haddr_t found_addr; /* Address of the found section */ + unsigned fs_persist; /* To persist free-space or not */ + + TESTING("Paged aggregation for file space: H5MF_alloc/H5MF_xfree"); + + /* Check for split or multi driver */ + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + + if(!multi && !split) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + if((fapl_new = H5Pcopy(fapl)) < 0) TEST_ERROR + + if(multi || split) + if(set_multi_split(fapl_new, 4096, multi, split) < 0) + TEST_ERROR; + + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + H5F_mem_page_t fs_type; + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, fs_persist, (hsize_t)1) < 0) + TEST_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 3 small meta data blocks: addr1, addr2, addr3 */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + addr2 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034); + addr3 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Free the block with addr2 */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE1034); + + if(!fs_persist) { + + H5MF_alloc_to_fs_type(f, H5FD_MEM_OHDR, TBLOCK_SIZE1034, (H5F_mem_page_t *)&fs_type); + + /* Verify that the freed block with addr2 is found from the small meta data manager */ + if(H5MF_find_sect(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034, f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + if(found_addr != addr2) + TEST_ERROR + } /* end if */ + + /* Allocate 2 small raw data blocks: saddr1, saddr2 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE1034); + + /* Free the block with saddr1 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + + if(!fs_persist) { + /* Verify that the freed block with saddr1 is found from the small raw data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5F_MEM_PAGE_DRAW], &found_addr) < 0) + TEST_ERROR + if(found_addr != saddr1) + TEST_ERROR + } /* end if */ + + /* Allocate 2 large data blocks: gaddr1, gaddr2 */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + + /* Free the block with gaddr1 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE5000); + + if(!fs_persist) { + + H5MF_alloc_to_fs_type(f, H5FD_MEM_DRAW, TBLOCK_SIZE5000, (H5F_mem_page_t *)&fs_type); + + /* Verify that the freed block with gaddr1 is found from the large data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8192, f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + if(found_addr != gaddr1) + TEST_ERROR + } /* end if */ + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + TEST_ERROR + + if(fs_persist) { + /* Re-open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Verify that the large generic manager is there */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_DRAW, TBLOCK_SIZE5000, (H5F_mem_page_t *)&fs_type); + if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Verify that the small meta data manager is there */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_OHDR, f->shared->fs_page_size - 1, (H5F_mem_page_t *)&fs_type); + if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) + TEST_ERROR + + /* Since we are about to open a self referential free space + * manager prior to the first file space allocation / deallocation + * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid + * assertion failures on the first file space alloc / dealloc. + */ + if(f->shared->first_alloc_dealloc){ + if(SUCCEED!=H5MF_tidy_self_referential_fsm_hack(f,H5AC_ind_read_dxpl_id)) + FAIL_STACK_ERROR + } + + /* Set up to use the small meta data manager */ + if(!(f->shared->fs_man[fs_type])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, fs_type) < 0) + TEST_ERROR + + /* Verify that the freed block with addr2 is found from the small meta data manager */ + if(H5MF_find_sect(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)(f->shared->fs_page_size-(addr3+TBLOCK_SIZE50)), f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + + if(found_addr != (addr3+TBLOCK_SIZE50)) + TEST_ERROR + + /* Verify that the small raw data manager is there */ + if(!H5F_addr_defined(f->shared->fs_addr[H5F_MEM_PAGE_DRAW])) + TEST_ERROR + + /* Set up to use the small raw data manager */ + if(!(f->shared->fs_man[H5F_MEM_PAGE_DRAW])) + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, H5F_MEM_PAGE_DRAW) < 0) + TEST_ERROR + + /* Verify that the freed block with saddr1 is found from the small raw data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30, f->shared->fs_man[H5F_MEM_PAGE_DRAW], &found_addr) < 0) + TEST_ERROR + if(found_addr != saddr1) + TEST_ERROR + + H5MF_alloc_to_fs_type(f, H5FD_MEM_DRAW, TBLOCK_SIZE5000, (H5F_mem_page_t *)&fs_type); + + if(!(f->shared->fs_man[fs_type])) + /* Set up to use the large data manager */ + if(H5MF_open_fstype(f, H5AC_ind_read_dxpl_id, fs_type) < 0) + TEST_ERROR + + /* Verify that the freed block with gaddr1 is found from the large data manager */ + if(H5MF_find_sect(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8192, f->shared->fs_man[fs_type], &found_addr) < 0) + TEST_ERROR + if(found_addr != gaddr1) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + } /* end if fs_persist */ + } /* end for */ + + if(H5Pclose(fapl_new) < 0) + TEST_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support persisting free-space or paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + H5Pclose(fapl_new); + } H5E_END_TRY; + return(1); + +} /* test_page_alloc_xfree() */ + +/*------------------------------------------------------------------------- + * Function: test_page_try_shrink + * + * Purpose: To verify that shrinking via H5MF_try_shrink() work properly + * when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_try_shrink(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr1; /* Address for small meta data block */ + haddr_t saddr1; /* Address for small raw data block */ + haddr_t gaddr1; /* Address for large data block */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + htri_t status; /* status from shrinking */ + h5_stat_size_t file_size; /* File size */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: H5MF_try_shrink()"); + + /* Current VFD that does not support continuous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi") ); + + if(contig_addr_vfd) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate a small meta data block with addr1 */ + addr1 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Try to shrink the block with addr1 */ + if((status = H5MF_try_shrink(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE50)) < 0) + FAIL_STACK_ERROR + + /* Couldn't shrink due to the section (remaining space in the page) is in the small meta data free-space manager */ + if(status == TRUE) TEST_ERROR + + /* Allocate a small raw data block with saddr1 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Try to shrink the block with saddr1 */ + if((status = H5MF_try_shrink(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE50)) < 0) + FAIL_STACK_ERROR + + /* Couldn't shrink due to the section (remaining space in the page) is in the small raw data free-space manager */ + if(status == TRUE) TEST_ERROR + + /* Allocate a large data block with gaddr1 */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + + /* Try to shrink the block with gaddr1 */ + if((status = H5MF_try_shrink(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE5000)) < 0) + FAIL_STACK_ERROR + + /* Couldn't shrink due to the section (remaining space in the page) is in the large-sized free-space manager */ + if(status == TRUE) TEST_ERROR + + /* Free the block with saddr1--merge to become 1 page, then return to the large manager */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE50); + + /* Merge all 3 sections and shrunk */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE5000); + + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Get the size of the file */ + if((file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + + /* Should be on page boundary */ + if(file_size % TBLOCK_SIZE4096) + TEST_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + } H5E_END_TRY; + return(1); + +} /* test_page_try_shrink() */ + +/*------------------------------------------------------------------------- + * Function: test_page_small_try_extend + * + * Purpose: To verify that extending a small block via H5MF_try_extend() works + * properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_small_try_extend(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr1, addr2, addr3; /* Addresses for small meta data blocks */ + haddr_t saddr1; /* Address for small raw data block */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + htri_t was_extended; /* Whether the block can be extended or not */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: H5MF_try_extend() a small block"); + + /* Current VFD that does not support continuous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi") && HDstrcmp(env_h5_drvr, "family")); + + if(contig_addr_vfd) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate a small meta data block with addr1 */ + addr1 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE98); + + /* Try extending the block with addr1 at EOF not crossing page boundary */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE98, (hsize_t)3100); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Allocate 2 small meta data blocks with addr2 and addr3--will be on another meta data page */ + addr2 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE100); + addr3 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE150); + + /* The block with addr2 should be page aligned */ + /* The block with addr3 resides right next to the block with addr2 */ + if(addr2 % TBLOCK_SIZE4096) + TEST_ERROR + if(addr3 != (addr2 + TBLOCK_SIZE100)) + TEST_ERROR + + /* Free the block with addr2 */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr2, (hsize_t)TBLOCK_SIZE100); + + /* Try extending the block with addr1 that will cross to the next page where the freed block with addr2 resides */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3198, (hsize_t)TBLOCK_SIZE100); + /* Shouldn't succeed--should not cross page boundary */ + if(was_extended == TRUE) TEST_ERROR + + /* Try extending the block with addr1 into the free-space section that is big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3198, (hsize_t)TBLOCK_SIZE50); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Free the block with addr1 */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TBLOCK_SIZE3248); + + /* Allocate a new meta data block with addr1 */ + /* There is a page end threshold of size H5F_FILE_SPACE_PGEND_META_THRES at the end of the block */ + /* The block is right next to the threshold */ + addr1 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3286); + + /* Try extending the block into the threshold with size > H5F_FILE_SPACE_PGEND_META_THRES */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3286, (hsize_t)TBLOCK_SIZE11); + /* Shouldn't succeed */ + if(was_extended == TRUE) TEST_ERROR + + /* Try extending the block into the threshold with size < H5F_FILE_SPACE_PGEND_META_THRES */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_OHDR, (haddr_t)addr1, (hsize_t)TBLOCK_SIZE3286, (hsize_t)TBLOCK_SIZE2); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Free the block with addr3--will merge with the remaining sections to become a page and then free the page */ + H5MF_xfree(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, addr3, (hsize_t)TBLOCK_SIZE150); + + /* Allocate a small raw data block with saddr1 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4086); + + /* Try extending the block crossing the page boundary */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4086, (hsize_t)TBLOCK_SIZE11); + /* Shouldn't succeed */ + if(was_extended == TRUE) TEST_ERROR + + /* Try extending the block not crossing page boundary */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4086, (hsize_t)TBLOCK_SIZE10); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* The extended block is now "large" in size */ + /* Try extending the block */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4096, (hsize_t)TBLOCK_SIZE10); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Try extending the large-sized block */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)saddr1, (hsize_t)TBLOCK_SIZE4106, (hsize_t)TBLOCK_SIZE5000); + /* Should not succeed because the mis-aligned fragment in the page is in the large-sized free-space manager */ + if(was_extended == TRUE) TEST_ERROR + + /* Close the file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + } H5E_END_TRY; + return(1); + +} /* test_page_small_try_extend() */ + +/*------------------------------------------------------------------------- + * Function: test_page_large_try_extend + * + * Purpose: To verify that extending a large block via H5MF_try_extend() + * is done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_large_try_extend(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t gaddr1, gaddr2, gaddr3, gaddr4; /* Addresses for large data blocks */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + htri_t was_extended; /* Whether the block can be extended or not */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: H5MF_try_extend() a large block"); + + /* Current VFD that does not support continuous address space */ contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); + if(contig_addr_vfd) { + /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + /* Create the file to work on */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR /* Get a pointer to the internal file object */ - if(NULL == (f = (H5F_t *)H5I_object(file))) + if(NULL == (f = (H5F_t *)H5I_object(fid))) FAIL_STACK_ERROR - /* Allocate the first block of type H5FD_MEM_SUPER */ - type = H5FD_MEM_SUPER; - addr1 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Allocate a large data block with gaddr1 */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)6000); - /* Allocate the second block of type H5FD_MEM_SUPER */ - H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE50); + /* Should be page aligned */ + if(gaddr1 % TBLOCK_SIZE4096) + TEST_ERROR - /* Allocate the first block of type H5FD_MEM_DRAW */ - stype = H5FD_MEM_DRAW; - saddr1 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Extending the block with gaddr1 at EOF to become 2 pages */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr1, (hsize_t)TBLOCK_SIZE6000, (hsize_t)TBLOCK_SIZE2192); + /* Should succeed */ + if(was_extended != TRUE) TEST_ERROR + + /* Allocate a large data block with gaddr2 */ + gaddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + /* Should be page aligned */ + if(gaddr2 % TBLOCK_SIZE4096) + TEST_ERROR - /* Free the first block of type H5FD_MEM_SUPER */ - H5MF_xfree(f, type, H5AC_ind_read_dxpl_id, addr1, (hsize_t)TEST_BLOCK_SIZE30); + /* Try extending the block with gaddr1 */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr1, (hsize_t)TBLOCK_SIZE8192, (hsize_t)TBLOCK_SIZE50); + /* Should not succeed */ + if(was_extended == TRUE) TEST_ERROR - /* Allocate the second block of type H5FD_MEM_DRAW */ - saddr2 = H5MF_alloc(f, stype, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Allocate a large data block with gaddr3 */ + gaddr3 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + /* Should be page aligned */ + if(gaddr3 % TBLOCK_SIZE4096) + TEST_ERROR - /* Verify that saddr1 is not addr1 */ - if(saddr2 == addr1) TEST_ERROR + /* Try extending the block with gaddr2--there is a free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8000, (hsize_t)TBLOCK_SIZE100); + /* Should succeed */ + if(was_extended == FALSE) TEST_ERROR - /* Free the first block of type H5FD_MEM_DRAW */ - H5MF_xfree(f, stype, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TEST_BLOCK_SIZE30); + /* Try extending the block with gaddr2--there is no free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8100, (hsize_t)TBLOCK_SIZE100); + /* Should not succeed */ + if(was_extended == TRUE) TEST_ERROR - /* Allocate the third block of type H5FD_MEM_SUPER */ - addr3 = H5MF_alloc(f, type, H5AC_ind_read_dxpl_id, (hsize_t)TEST_BLOCK_SIZE30); + /* Try extending the block with gaddr2--there is a free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8100, (hsize_t)TBLOCK_SIZE90); + /* Should succeed */ + if(was_extended == FALSE) TEST_ERROR - /* Verify that addr3 is not saddr1 */ - if(addr3 == saddr1) TEST_ERROR + /* Try extending the block with gaddr2 */ + /* There is no free-space section big enough to fulfill the request (request is < H5F_FILE_SPACE_PGEND_META_THRES) */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8190, (hsize_t)TBLOCK_SIZE5); + /* Should not succeed */ + if(was_extended == TRUE) TEST_ERROR - if(H5Fclose(file) < 0) + /* Allocate a large data block with gaddr4 */ + gaddr4 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + /* Should be page aligned */ + if(gaddr4 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Free the block with gaddr3--will merge with remaining free space to become 2 pages + section (size 2) in previous page */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr3, (hsize_t)TBLOCK_SIZE8000); + + /* Try extending the block with gaddr2 crossing page boundary--there is free-space section big enough to fulfill the request */ + was_extended = H5MF_try_extend(f, H5AC_ind_read_dxpl_id, H5FD_MEM_DRAW, (haddr_t)gaddr2, (hsize_t)TBLOCK_SIZE8190, (hsize_t)TBLOCK_SIZE5); + /* Should succeed */ + if(was_extended == FALSE) TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) FAIL_STACK_ERROR PASSED() - } /* end if */ - else { - SKIPPED(); - puts(" Current VFD doesn't support metadata aggregator"); - } /* end else */ + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation strategy"); + } return(0); error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(fid); + H5Pclose(fcpl); } H5E_END_TRY; return(1); -} /* test_dichotomy() */ + +} /* test_page_large_try_extend() */ + +/*------------------------------------------------------------------------- + * Function: test_page_large + * + * Purpose: To verify that allocations and de-allocations for large data + * are done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_large(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t gaddr1, gaddr2, gaddr3, gaddr4; /* Addresses for blocks */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + h5_stat_size_t file_size; /* File size */ + char filename[FILENAME_LEN]; /* Filename to use */ + + TESTING("Paged aggregation for file space: large allocations and de-allocations"); + + /* Current VFD that does not support continuous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); + + if(contig_addr_vfd) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate a large data block with gaddr1 */ + /* 1 page + 1904 bytes; 2192 bytes in free-space manager */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE6000); + + /* Should be page aligned */ + if(gaddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a large data block with gaddr2--should be on another page */ + /* Allocate 1 page + 3904 bytes; 192 bytes in free-space manager */ + gaddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + /* Should be page aligned */ + if(gaddr2 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a large data block with gaddr3--should be on another page */ + /* Allocate 2 pages + 3808 bytes; 288 bytes in free-space manager */ + gaddr3 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE12000); + if(!H5F_addr_defined(gaddr3)) + TEST_ERROR + + /* Free the block with gaddr2 */ + /* Merged sections: 2192 + 8000 + 192 = 10384 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr2, (hsize_t)TBLOCK_SIZE8000); + + /* Get free-space info */ + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + + /* Verify that the manager contains 2 free-space sections: 10384 and 288 */ + if(fs_stat.tot_sect_count != 2) + TEST_ERROR + if(fs_stat.tot_space != (10384+288)) + TEST_ERROR + + /* Allocate a large data block with gaddr4--there is a free-space section able to fulfill the request */ + /* Free-space sections: 2192 + 3192 + 288 = 5672 bytes */ + gaddr4 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + + /* Should be page aligned */ + if(gaddr4 % TBLOCK_SIZE4096) + TEST_ERROR + if(gaddr4 != gaddr2) + TEST_ERROR + + /* Get free-space info */ + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + /* Verify that that there are 3 free-space sections */ + if(fs_stat.tot_sect_count != 3) + TEST_ERROR + if(fs_stat.tot_space != (2192+3192+288)) + TEST_ERROR + + /* Free the two blocks with gaddr1 and gaddr4 */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr1, (hsize_t)TBLOCK_SIZE6000); + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, gaddr4, (hsize_t)TBLOCK_SIZE5000); + + /* Get free-space info */ + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + /* Verify that that there are 2 free-space sections: 16384 (4 pages) + 288 */ + if(fs_stat.tot_sect_count != 2) + TEST_ERROR + if(fs_stat.tot_space != (16384+288)) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + /* Get the size of the file */ + if((file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + + /* Verify that file size end on a page boundary */ + if(file_size % TBLOCK_SIZE4096) + TEST_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + } H5E_END_TRY; + return(1); + +} /* test_page_large() */ + +/*------------------------------------------------------------------------- + * Function: test_page_small + * + * Purpose: To verify allocations and de-allocations for small meta/raw data + * are done properly when paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_small(const char *env_h5_drvr, hid_t fapl) +{ + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr2, addr3, addr4, addr5; /* Addresses for blocks */ + haddr_t addr9, addr10, addr11; /* Address for small meta data blocks */ + haddr_t saddr1, saddr2; /* Addresses for small raw data blocks */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ + char filename[FILENAME_LEN]; /* Filename to use */ + hbool_t multi= FALSE, split = FALSE, family = FALSE; + + TESTING("Paged aggregation for file space: small allocations and de-allocations"); + + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + else if(!HDstrcmp(env_h5_drvr, "family")) + family = TRUE; + + if(!multi && !split) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set the strategy to paged aggregation */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + FAIL_STACK_ERROR + + /* Allocate 2 small meta data blocks: addr1, addr2 */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + addr2 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Allocate a small raw data block with saddr1 */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + /* Should be on the second page and page aligned */ + if(saddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a small raw data block with saddr2 */ + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + /* Should not be page aligned */ + if(!(saddr2 % TBLOCK_SIZE4096)) + TEST_ERROR + /* Should be next to the block with saddr1 */ + if(saddr2 != (saddr1 + TBLOCK_SIZE30)) + TEST_ERROR + + /* Allocate a small meta data block with addr3--there is no free-space section big enough to fulfill the request */ + addr3 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE4020); + /* Should be on the third page and page aligned */ + if(addr3 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a small meta data block with addr4--there is a free-space section big enough to fulfill the request */ + addr4 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + /* Should not be page aligned */ + if(!(addr4 % TBLOCK_SIZE4096)) + TEST_ERROR + /* Should be next to the block with addr2 */ + if(addr4 != (addr2 + TBLOCK_SIZE50)) + TEST_ERROR + + /* Allocate a small meta data block with addr5--there is a free-space section big enough to fulfill the request */ + addr5 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE40); + /* Should not be page aligned */ + if(!(addr5 % TBLOCK_SIZE4096)) + TEST_ERROR + + /* Should be next to the block with addr3 */ + if(addr5 != (addr3 + TBLOCK_SIZE4020)) + TEST_ERROR + + /* Allocate a small meta data block with addr6--taking up the remaining space in the first page */ + if(family) + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3080); + else + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3088); + + /* Allocate a small meta data block with addr7--taking up the remaining space in the third page */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE36); + + /* Allocate 2 small meta data blocks: addr8, addr9--there is no free-space to fulfill the request */ + H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + addr9 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + + /* Free the block with saddr1 and saddr2--merge with remaining section to become a page which will be returned to the large manager */ + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr1, (hsize_t)TBLOCK_SIZE30); + H5MF_xfree(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, saddr2, (hsize_t)TBLOCK_SIZE50); + + /* Verify that the large manager does contain a section with file space page size (default is 4096) */ + if(!f->shared->fs_man[H5F_MEM_PAGE_GENERIC]) + TEST_ERROR + if(H5FS_stat_info(f, f->shared->fs_man[H5F_MEM_PAGE_GENERIC], &fs_stat) < 0) + FAIL_STACK_ERROR + if(fs_stat.tot_space != TBLOCK_SIZE4096) + TEST_ERROR + + /* Allocate a small meta data block with addr10--there is a free-space section big enough to fulfill the request */ + addr10 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE3900); + /* The block should be next to the block with addr9 */ + if(addr10 != (addr9 + TBLOCK_SIZE80)) + TEST_ERROR + + /* Allocate a small meta data block with addr11 */ + /* The current free-space section is unable to fulfill the request; obtain a page from the large manager */ + addr11 = H5MF_alloc(f, H5FD_MEM_OHDR, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + /* The address of the block should be the same the freed block with saddr1 */ + if(addr11 != saddr1) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR + + /* Close the property list */ + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + } H5E_END_TRY; + return(1); + +} /* test_page_small() */ + +/*------------------------------------------------------------------------- + * Function: test_page_alignment + * + * Purpose: To verify the proper alignment is used when H5Pset_alignment() + * is set and paged aggregation is enabled. + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Vailin Choi; Jan 2013 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_page_alignment(const char *env_h5_drvr, hid_t fapl) +{ + + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fcpl2 = -1; /* File creation property list ID */ + hid_t fapl_new = -1; /* File access property list ID */ + H5F_t *f = NULL; /* Internal file object pointer */ + haddr_t addr1, addr2; /* Addresses for small meta data blocks */ + haddr_t saddr1, saddr2; /* Addresses for small raw data blocks */ + haddr_t gaddr1, gaddr2; /* Addresses for blocks */ + char filename[FILENAME_LEN]; /* Filename to use */ + hbool_t split = FALSE, multi = FALSE; + + TESTING("Paged aggregation and H5Pset_alignment: verify proper alignment is used"); + + /* Check for split or multi driver */ + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + + if(!multi && !split) { + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); + + /* + * Case 1: Verify that the alignment in use is the default file space + * page size when paged aggregation is enabled. + */ + + if((fapl_new = H5Pcopy(fapl)) < 0) TEST_ERROR + + /* The alignment to use will be the library's default file space page size */ + if(H5Pset_libver_bounds(fapl_new, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + /* Set alignment value to 16 */ + if(H5Pset_alignment(fapl_new, (hsize_t)0, (hsize_t)TEST_ALIGN16) < 0) + TEST_ERROR + + if(split || multi) { + hid_t memb_fapl; + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + /* Create fapl */ + if((memb_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR + + /* Set alignment */ + if(H5Pset_alignment(memb_fapl, 0, (hsize_t)TEST_ALIGN16) < 0) + TEST_ERROR + + HDmemset(memb_name, 0, sizeof memb_name); + + if(split) { + /* Set split driver with new FAPLs */ + if(H5Pset_fapl_split(fapl_new, "-m.h5", memb_fapl, "-r.h5", memb_fapl) < 0) + TEST_ERROR + + /* Get current multi settings */ + if(H5Pget_fapl_multi(fapl_new, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + TBLOCK_SIZE4096 - 1) / TBLOCK_SIZE4096) * TBLOCK_SIZE4096; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + TBLOCK_SIZE4096 - 1) / TBLOCK_SIZE4096) * TBLOCK_SIZE4096; + + /* Set split driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + } else { + /* Get current multi settings */ + if(H5Pget_fapl_multi(fapl_new, memb_map, NULL, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + /* Populate memb_fapl_arr */ + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { + memb_fapl_arr[mt] = memb_fapl; + memb_addr[mt] = ((memb_addr[mt] + TBLOCK_SIZE4096 - 1) / TBLOCK_SIZE4096) * TBLOCK_SIZE4096; + } + + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl_new, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + } /* end else */ + + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); + + /* Close memb_fapl */ + if(H5Pclose(memb_fapl) < 0) + TEST_ERROR + } /* end if */ + + /* File creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set the strategy to paged aggregation and persisting free space */ + /* The alignment to use will be the library's default file space page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + TEST_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 2 small raw data blocks */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Should be on the second page and page aligned on 4096 (default file space page size) */ + if(saddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Should be next to the block with saddr1 */ + if(saddr2 != (saddr1 + TBLOCK_SIZE30)) + TEST_ERROR + + /* Allocate 2 large raw data blocks */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + gaddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE8000); + + /* Should be on the 3rd page and page aligned */ + if(gaddr1 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Should be on the 4th page and page aligned */ + if(gaddr2 % TBLOCK_SIZE4096) + TEST_ERROR + + /* Close the file creation property list */ + if(H5Pclose(fcpl) < 0) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + + /* + * Case 2: Verify that the alignment in use is the alignment set + * via H5Pset_alignment when paged aggregation not enabled. + */ + /* fapl_new has latest format and H5Pset_alignment set */ + /* Disable small data block mechanism */ + if(H5Pset_small_data_block_size(fapl_new, (hsize_t)0) < 0) + TEST_ERROR + /* Disable meta data block mechanism */ + if(H5Pset_meta_block_size(fapl_new, (hsize_t)0) < 0) + TEST_ERROR + + /* Create the file to work on */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 2 small meta data blocks */ + addr1 = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + addr2 = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Should be aligned on 16 */ + if(addr1 % TEST_ALIGN16 || addr2 % TEST_ALIGN16) + TEST_ERROR + + /* addr2 should be right next to the block with addr1 */ + if((addr1 + TBLOCK_SIZE30) % TEST_ALIGN16) + if(addr2 != (((addr1 + TBLOCK_SIZE30) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* Allocate 2 small raw data blocks */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE80); + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE100); + + /* Should be aligned on 16 */ + if(saddr1 % TEST_ALIGN16 || saddr2 % TEST_ALIGN16) + TEST_ERROR + + if(!multi && !split) { + /* saddr1 should be right next to the block with addr2 */ + if((addr2 + TBLOCK_SIZE50) % TEST_ALIGN16) + if(saddr1 != (((addr2 + TBLOCK_SIZE50) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + } + + /* saddr2 should be right next to the block with saddr1 */ + if((saddr1 + TBLOCK_SIZE80) % TEST_ALIGN16) + if(saddr2 != (((saddr1 + TBLOCK_SIZE80) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* Close file */ + if(H5Fclose(fid) < 0) + TEST_ERROR + + /* + * Case 3: Verify that the alignment in use is the alignment set + * via H5Pset_alignment when paged aggregation not enabled. + */ + /* File creation property list */ + if((fcpl2 = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR + + /* Set file space page size */ + if(H5Pset_file_space_page_size(fcpl2, (hsize_t)TBLOCK_SIZE8192) < 0) + TEST_ERROR + + /* Set strategy to H5F_FSPACE_STRATEGY_AGGR but meta/raw data block is 0 as set in fapl_new */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_AGGR, FALSE, (hsize_t)1) < 0) + TEST_ERROR + + /* fapl_new has latest format, H5Pset_alignment set, and disable meta/raw block */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl2, fapl_new)) < 0) + TEST_ERROR + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) + TEST_ERROR + + /* Allocate 2 small raw data blocks */ + saddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE30); + saddr2 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE50); + + /* Should be aligned on 16 */ + if(saddr1 % TEST_ALIGN16) + TEST_ERROR + if(saddr2 % TEST_ALIGN16) + TEST_ERROR + + /* saddr2 should be right next to the block with saddr1 */ + if((saddr1 + TBLOCK_SIZE30) % TEST_ALIGN16) + if(saddr2 != (((saddr1 + TBLOCK_SIZE30) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* Allocate a large raw data block */ + gaddr1 = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, (hsize_t)TBLOCK_SIZE5000); + + /* Should be aligned on 16 */ + if(gaddr1 % TEST_ALIGN16) + TEST_ERROR + + /* gaddr1 is right next to the block with saddr2 */ + if((saddr2 + TBLOCK_SIZE50) % TEST_ALIGN16) + if(gaddr1 != (((saddr2 + TBLOCK_SIZE50) / TEST_ALIGN16) + 1) * TEST_ALIGN16) + TEST_ERROR + + /* There is no free-space manager involved for H5F_FSPACE_STRATEGY_AGGR strategy */ + if(f->shared->fs_man[H5FD_MEM_DRAW] || f->shared->fs_man[H5FD_MEM_SUPER]) + TEST_ERROR + + /* Closing */ + if(H5Fclose(fid) < 0) + TEST_ERROR + if(H5Pclose(fcpl2) < 0) + TEST_ERROR + if(H5Pclose(fapl_new) < 0) + TEST_ERROR + + PASSED() + + } else { + SKIPPED(); + puts(" Current VFD doesn't support persisting free-space or paged aggregation strategy"); + } + + return(0); + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fcpl); + H5Pclose(fapl_new); + } H5E_END_TRY; + return(1); + +} /* test_page_alignment() */ int main(void) @@ -7625,33 +8869,21 @@ main(void) /* Make a copy of the FAPL before adjusting the alignment */ if((new_fapl = H5Pcopy(fapl)) < 0) TEST_ERROR - - /* alignment is not set for the following tests */ - if(H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1) < 0) - TEST_ERROR - - /* meta/small data is set to 2048 for the following tests */ - if(H5Pset_meta_block_size(fapl, (hsize_t)TEST_BLOCK_SIZE2048) < 0) - TEST_ERROR - if(H5Pset_small_data_block_size(fapl, (hsize_t)TEST_BLOCK_SIZE2048) < 0) - TEST_ERROR - - /* interaction with file allocation */ + /* For old library format--interaction with file allocation */ nerrors += test_mf_eoa(env_h5_drvr, fapl); nerrors += test_mf_eoa_shrink(env_h5_drvr, fapl); nerrors += test_mf_eoa_extend(env_h5_drvr, fapl); - /* interaction with temporary file space allocation */ - nerrors += test_mf_tmp(env_h5_drvr, fapl); + /* For old library format */ + nerrors += test_dichotomy(new_fapl); - /* interaction with free-space manager */ + /* For old library format--interaction with free-space manager */ nerrors += test_mf_fs_start(fapl); nerrors += test_mf_fs_alloc_free(fapl); nerrors += test_mf_fs_extend(fapl); nerrors += test_mf_fs_absorb(env_h5_drvr, fapl); - nerrors += test_dichotomy(env_h5_drvr, new_fapl); - /* interaction with meta/sdata aggregator */ + /* For old library format--interaction with meta/sdata aggregator */ nerrors += test_mf_aggr_alloc1(env_h5_drvr, fapl); nerrors += test_mf_aggr_alloc2(env_h5_drvr, fapl); nerrors += test_mf_aggr_alloc3(env_h5_drvr, fapl); @@ -7662,7 +8894,7 @@ main(void) nerrors += test_mf_aggr_extend(env_h5_drvr, fapl); nerrors += test_mf_aggr_absorb(env_h5_drvr, fapl); - /* Tests for alignment */ + /* For old library format--tests for alignment */ for(curr_test = TEST_NORMAL; curr_test < TEST_NTESTS; H5_INC_ENUM(test_type_t, curr_test)) { switch(curr_test) { @@ -7692,11 +8924,54 @@ main(void) nerrors += test_mf_align_alloc6(env_h5_drvr, fapl, new_fapl); } /* end for */ - /* tests to verify that file's free-space managers are persistent */ - nerrors += test_mf_fs_drivers(fapl); + /* For old and new format--interaction with temporary file space allocation */ + nerrors += test_mf_tmp(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_tmp(env_h5_drvr, fapl, TRUE); + + /* For old and new format--free-space merge/shrunk away */ + + /* Temporary: modify to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_fs_gone(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_fs_gone(env_h5_drvr, fapl, TRUE); + + /* Temporary: modify to skip testing multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_strat_thres_gone(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_strat_thres_gone(env_h5_drvr, fapl, TRUE); - /* tests for file space management */ - nerrors += test_filespace_drivers(fapl); + /* For old and new format--persisting free-space */ + + /* Temporary: Modify to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_fs_persist(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_fs_persist(env_h5_drvr, fapl, TRUE); + + /* Temporary: modify to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_mf_strat_thres_persist(env_h5_drvr, fapl, FALSE); + nerrors += test_mf_strat_thres_persist(env_h5_drvr, fapl, TRUE); + + /* Temporary skipped for multi/split drivers: + fail file create when persisting free-space or using paged aggregation strategy */ +#ifdef PB_OUT + /* Tests specific for multi and split files--persisting free-space */ + nerrors += test_mf_fs_persist_split(); + nerrors += test_mf_fs_persist_multi(); +#endif + + /* + * Tests specific for file space paging + */ + /* Temporary: The following 7 tests are modified to skip testing for multi/split driver: + fail file create when persisting free-space or using paged aggregation strategy */ + nerrors += test_page_small(env_h5_drvr, fapl); + nerrors += test_page_large(env_h5_drvr, fapl); + nerrors += test_page_large_try_extend(env_h5_drvr, fapl); + nerrors += test_page_small_try_extend(env_h5_drvr, fapl); + nerrors += test_page_try_shrink(env_h5_drvr, fapl); + nerrors += test_page_alloc_xfree(env_h5_drvr, fapl); /* can handle multi/split */ + nerrors += test_page_alignment(env_h5_drvr, fapl); /* can handle multi/split */ /* tests for specific bugs */ nerrors += test_mf_bug1(env_h5_drvr, fapl); diff --git a/test/none.h5 b/test/none.h5 Binary files differnew file mode 100644 index 0000000..bb6ff56 --- /dev/null +++ b/test/none.h5 diff --git a/test/page_buffer.c b/test/page_buffer.c new file mode 100644 index 0000000..b94ad03 --- /dev/null +++ b/test/page_buffer.c @@ -0,0 +1,2087 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*********************************************************** +* +* Test program: cache_page_buffer +* +* Tests the Page Buffer Feature. +* +*************************************************************/ + +#include "h5test.h" + +#include "H5PBprivate.h" +#include "H5Iprivate.h" + +/* + * This file needs to access private information from the H5F package. + */ +#define H5MF_FRIEND /*suppress error about including H5MFpkg */ +#include "H5MFpkg.h" + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" + + +#define FILENAME_LEN 1024 +#define NUM_DSETS 5 +#define NX 100 +#define NY 50 + +/* helper routines */ +static unsigned create_file(char *filename, hid_t fcpl, hid_t fapl); +static unsigned open_file(char *filename, hid_t fapl, hsize_t page_size, size_t page_buffer_size); + +/* test routines */ +static unsigned test_args(hid_t fapl, const char *env_h5_drvr); +static unsigned test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr); +static unsigned test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr); +static unsigned test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr); +static unsigned test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr); + +const char *FILENAME[] = { + "filepaged", + NULL +}; + + +/*------------------------------------------------------------------------- + * Function: create_file() + * + * Purpose: The purpose of this function appears to be a smoke check + * intended to exercise the page buffer. + * + * Specifically, the function creates a file, and then goes + * through a loop in which it creates four data sets, write + * data to one of them, verifies the data written, and then + * deletes the three that it didn't write to. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +static unsigned +create_file(char *filename, hid_t fcpl, hid_t fapl) +{ + hid_t file_id = -1; + hid_t dset_id = -1; + hid_t grp_id = -1; + hid_t filespace = -1; + hsize_t dimsf[2] = {NX, NY}; /* dataset dimensions */ + int *data = NULL; /* pointer to data buffer to write */ + hid_t dcpl = -1; + int i; + int num_elements; + int j; + char dset_name[10]; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + if((grp_id = H5Gcreate2(file_id, "GROUP", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + num_elements = NX * NY; + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR + for (i=0; i < (int)num_elements; i++) + data[i] = i; + + if((filespace = H5Screate_simple(2, dimsf, NULL)) < 0) + FAIL_STACK_ERROR; + + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR; + if(H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) + FAIL_STACK_ERROR; + + for(i=0 ; i<NUM_DSETS; i++) { + + HDsprintf(dset_name, "D1dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDsprintf(dset_name, "D2dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDsprintf(dset_name, "D3dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDsprintf(dset_name, "dset%d", i); + if((dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, filespace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if(H5Dwrite(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + HDmemset(data, 0, (size_t)num_elements * sizeof(int)); + if((dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if(H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + FAIL_STACK_ERROR; + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + for (j=0; j < num_elements; j++) { + if(data[j] != j) { + HDfprintf(stderr, "Read different values than written\n"); + FAIL_STACK_ERROR; + } + } + + HDsprintf(dset_name, "D1dset%d", i); + if(H5Ldelete(grp_id, dset_name, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + HDsprintf(dset_name, "D2dset%d", i); + if(H5Ldelete(grp_id, dset_name, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + HDsprintf(dset_name, "D3dset%d", i); + if(H5Ldelete(grp_id, dset_name, H5P_DEFAULT) < 0) + FAIL_STACK_ERROR; + } + + if(H5Gclose(grp_id) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Sclose(filespace) < 0) + FAIL_STACK_ERROR; + + HDfree(data); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(filespace); + H5Gclose(grp_id); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return(1); +} /* create_file */ + + +/*------------------------------------------------------------------------- + * Function: open_file() + * + * Purpose: The purpose of this function appears to be a smoke check + * intended to exercise the page buffer. + * + * Specifically, the function opens a file (created by + * create_file()?), and verify the contents of its datasets. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +static unsigned +open_file(char *filename, hid_t fapl, hsize_t page_size, + size_t page_buffer_size) +{ + hid_t file_id = -1; + hid_t dset_id = -1; + hid_t grp_id = -1; + int *data = NULL; /* pointer to data buffer to write */ + int i; + int j; + int num_elements; + char dset_name[10]; + H5F_t *f = NULL; + + if((file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + if(f->shared->page_buf == NULL) + FAIL_STACK_ERROR; + if(f->shared->page_buf->page_size != page_size) + FAIL_STACK_ERROR; + if(f->shared->page_buf->max_size != page_buffer_size) + FAIL_STACK_ERROR; + + if((grp_id = H5Gopen2(file_id, "GROUP", H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + num_elements = NX * NY; + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR + + for(i=0 ; i<NUM_DSETS; i++) { + + HDsprintf(dset_name, "dset%d", i); + if((dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + if(H5Dread(dset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + FAIL_STACK_ERROR; + + if(H5Dclose(dset_id) < 0) + FAIL_STACK_ERROR; + + for (j=0; j < num_elements; j++) { + if(data[j] != j) { + HDfprintf(stderr, "Read different values than written\n"); + FAIL_STACK_ERROR; + } + } + } + + if(H5Gclose(grp_id) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Gclose(grp_id); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return 1; +} + +/* + * + * set_multi_split(): + * Internal routine to set up page-aligned address space for multi/split driver + * when testing paged aggregation. + * + */ +static unsigned +set_multi_split(const char *env_h5_drvr, hid_t fapl, hsize_t pagesize) +{ + hbool_t split = FALSE; + hbool_t multi = FALSE; + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + /* Check for split or multi driver */ + if(!HDstrcmp(env_h5_drvr, "split")) + split = TRUE; + else if(!HDstrcmp(env_h5_drvr, "multi")) + multi = TRUE; + + if(split || multi) { + + HDmemset(memb_name, 0, sizeof memb_name); + + /* Get current split settings */ + if(H5Pget_fapl_multi(fapl, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR + + if(split) { + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + pagesize - 1) / pagesize) * pagesize; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + pagesize - 1) / pagesize) * pagesize; + } else { + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + memb_addr[mt] = ((memb_addr[mt] + pagesize - 1) / pagesize) * pagesize; + } /* end else */ + + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR + + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); + + } /* end if */ + + return 0; + +error: + return 1; + +} /* set_multi_split() */ + + +/*------------------------------------------------------------------------- + * Function: test_args() + * + * Purpose: This test appears to be a quick smoke check directed at: + * + * 1) verifying that API errors are caught. + * + * 2) verifying that the page buffer behaves more or less + * as advertized. + * + * Any data mis-matches or unexpected failures or successes + * reported by the HDF5 library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +static unsigned +test_args(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + herr_t ret; + + TESTING("Settings for Page Buffering"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) TEST_ERROR + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + + + /* Test setting a page buffer without Paged Aggregation enabled - + * should fail + */ + if(H5Pset_page_buffer_size(fapl, 512, 0, 0) < 0) + TEST_ERROR; + + H5E_BEGIN_TRY { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + + if(file_id >= 0) + TEST_ERROR; + + + /* Test setting a page buffer with a size smaller than a single + * page size - should fail + */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 511, 0, 0) < 0) + TEST_ERROR; + + H5E_BEGIN_TRY { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + + if(file_id >= 0) + TEST_ERROR; + + + /* Test setting a page buffer with sum of min meta and raw + * data percentage > 100 - should fail + */ + H5E_BEGIN_TRY { + ret = H5Pset_page_buffer_size(fapl, 512, 50, 51); + } H5E_END_TRY; + + if(ret >= 0) + TEST_ERROR; + + if(set_multi_split(env_h5_drvr, fapl, 512) != 0) + TEST_ERROR; + + /* Test setting a page buffer with a size equal to a single page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 512, 0, 0) < 0) + TEST_ERROR; + + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + + if(open_file(filename, fapl, 512, 512) != 0) + TEST_ERROR; + + + /* Test setting a page buffer with a size slightly larger than a + * single page size + */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 513, 0, 0) < 0) + TEST_ERROR; + + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + + if(open_file(filename, fapl, 512, 512) != 0) + TEST_ERROR; + + if(set_multi_split(env_h5_drvr, fapl, 4194304) != 0) + TEST_ERROR; + + + /* Test setting a large page buffer size and page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 4194304) < 0) + TEST_ERROR; + + if(H5Pset_page_buffer_size(fapl, 16777216, 0, 0) < 0) + TEST_ERROR; + + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + + if(open_file(filename, fapl, 4194304, 16777216) != 0) + TEST_ERROR; + + if(set_multi_split(env_h5_drvr, fapl, 1) != 0) + TEST_ERROR; + + + /* Test setting a 512 byte page buffer size and page size */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 512) < 0) + TEST_ERROR; + if(H5Pset_page_buffer_size(fapl, 512, 0, 0) < 0) + TEST_ERROR; + if(create_file(filename, fcpl, fapl) != 0) + TEST_ERROR; + if(open_file(filename, fapl, 512, 512) != 0) + TEST_ERROR; + + + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + } H5E_END_TRY; + return 1; +} /* test_args */ + + +/*------------------------------------------------------------------------- + * Function: test_raw_data_handling() + * + * Purpose: The purpose of this function appears to be a smoke check + * of raw data reads and writes via the page buffer. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added base_page_cnt field as supporting code. This allows + * the test to adjust to the number of page buffer pages + * accessed during file open / create. + * + * The test for the value of base_page_cnt just after file + * open exists detect changes in library behavior. Assuming + * any such change is not indicative of other issues, these + * tests can be modified to reflect the change. + * + * JRM -- 2/23/17 + * + *------------------------------------------------------------------------- + */ + +/* Changes due to file space page size has a minimum size of 512 */ +static unsigned +test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + H5FD_io_info_t fdio_info; + H5P_genplist_t *meta_plist = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id); + H5P_genplist_t *raw_plist = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id); + size_t base_page_cnt; + size_t page_count = 0; + int i, num_elements = 2000; + haddr_t addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("Raw Data Handling"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + TEST_ERROR; + if(H5Pset_page_buffer_size(fapl, sizeof(int)*2000, 0, 0) < 0) + TEST_ERROR; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the number of pages inserted, and verify that it is the + * the expected value. + */ + base_page_cnt = H5SL_count(f->shared->page_buf->slist_ptr); + if(base_page_cnt != 1) + TEST_ERROR; + + /* allocate space for a 2000 elements */ + if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* update the first 50 elements to have values 0-49 - this will be + a page buffer update with 1 page resulting in the page + buffer. */ + /* Changes: 100 */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count ++; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* update elements 300 - 450, with values 300 - - this will + bring two more pages into the page buffer. */ + for(i=0 ; i<150 ; i++) + data[i] = i+300; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*150, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + page_count += 2; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* update elements 100 - 300, this will go to disk but also update + existing pages in the page buffer. */ + for(i=0 ; i<200 ; i++) + data[i] = i+100; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*100), sizeof(int)*200, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* Update elements 225-300 - this will update an existing page in the PB */ + /* Changes: 450 - 600; 150 */ + for(i=0 ; i<150 ; i++) + data[i] = i+450; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*450), sizeof(int)*150, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* Do a full page write to block 600-800 - should bypass the PB */ + for(i=0 ; i<200 ; i++) + data[i] = i+600; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*600), sizeof(int)*200, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* read elements 800 - 1200, this should not affect the PB, and should read -1s */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*800), sizeof(int)*400, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 400; i++) { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + FAIL_STACK_ERROR; + } + } + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + FAIL_STACK_ERROR; + + /* read elements 1200 - 1201, this should read -1 and bring in an + * entire page of addr 1200 + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*1200), sizeof(int)*1, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 1; i++) { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + page_count ++; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* read elements 175 - 225, this should use the PB existing pages */ + /* Changes: 350 - 450 */ + /* read elements 175 - 225, this should use the PB existing pages */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 100; i++) { + if(data[i] != i+350) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* read elements 0 - 800 using the VFD.. this should result in -1s + except for the writes that went through the PB (100-300 & 600-800) */ + fdio_info.file = f->shared->lf; + fdio_info.meta_dxpl = meta_plist; + fdio_info.raw_dxpl = raw_plist; + if(H5FD_read(&fdio_info, H5FD_MEM_DRAW, addr, sizeof(int)*800, data) < 0) + FAIL_STACK_ERROR; + i = 0; + while (i < 800) { + if((i>=100 && i<300) || (i>=600)) { + if(data[i] != i) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + else { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + i++; + } + + /* read elements 0 - 800 using the PB.. this should result in all + * what we have written so far and should get the updates from the PB + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int)*800, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + for (i=0; i < 800; i++) { + if(data[i] != i) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + + /* update elements 400 - 1400 to value 0, this will go to disk but + * also evict existing pages from the PB (page 400 & 1200 that are + * existing). + */ + for(i=0 ; i<1000 ; i++) + data[i] = 0; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*400), sizeof(int)*1000, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + page_count -= 2; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* read elements 0 - 1000.. this should go to disk then update the + * buffer result 200-400 with existing pages + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr, sizeof(int)*1000, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + i=0; + while (i < 1000) { + if(i<400) { + if(data[i] != i) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + else { + if(data[i] != 0) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } + } + i++; + } + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return 1; +} /* test_raw_data_handling */ + + +/*------------------------------------------------------------------------- + * Function: test_lru_processing() + * + * Purpose: Basic set of tests verifying expected page buffer LRU + * management. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added base_page_cnt field as supporting code. This allows + * the test to adjust to the number of page buffer pages + * accessed during file open / create. + * + * The test for the value of base_page_cnt just after file + * open exists detect changes in library behavior. Assuming + * any such change is not indicative of other issues, these + * tests can be modified to reflect the change. + * + * JRM -- 2/23/17 + * + * + *------------------------------------------------------------------------- + */ + +static unsigned +test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + size_t base_page_cnt; + size_t page_count = 0; + int i; + int num_elements = 2000; + haddr_t addr = HADDR_UNDEF; + haddr_t search_addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("LRU Processing"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + FAIL_STACK_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + FAIL_STACK_ERROR; + + /* keep 2 pages at max in the page buffer */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*400, 20, 0) < 0) + FAIL_STACK_ERROR; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the number of pages inserted, and verify that it is the + * the expected value. + */ + base_page_cnt = H5SL_count(f->shared->page_buf->slist_ptr); + if(base_page_cnt != 1) + TEST_ERROR; + + /* allocate space for a 2000 elements */ + if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* update the first 100 elements to have values 0-99 - this will be + * a page buffer update with 1 page resulting in the page + * buffer. + */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count ++; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count + base_page_cnt) + TEST_ERROR; + + /* update elements 300 - 450, with values 300 - 449 - this will + * bring two pages into the page buffer and evict 0. + */ + for(i=0 ; i<150 ; i++) + data[i] = i+300; + + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*150, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count = 2; + + /* at this point, the page buffer entry created at file open should + * have been evicted -- thus no further need to consider base_page_cnt. + */ + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* The two pages should be the ones with address 100 and 200; 0 + should have been evicted */ + /* Changes: 200, 400 */ + search_addr = addr; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + search_addr = addr + sizeof(int)*400; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* update elements 150-151, this will update existing pages in the + page buffer and move it to the top of the LRU. */ + /* Changes: 300 - 301 */ + for(i=0 ; i<1 ; i++) + data[i] = i+300; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*300), sizeof(int)*1, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* read elements 600 - 601, this should read -1 and bring in an + entire page of addr 600, and evict page 200 */ + /* Changes: 1200 - 1201; 1200, 400 */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*1200), sizeof(int)*1, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 1; i++) { + if(data[i] != -1) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } /* end if */ + } /* end for */ + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* Changes: 400 */ + search_addr = addr + sizeof(int)*400; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 200 */ + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 1200 */ + search_addr = addr + sizeof(int)*1200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + /* read elements 175 - 225, this should move 100 to the top, evict 600 and bring in 200 */ + /* Changes: 350 - 450; 200, 1200, 400 */ + if(H5F_block_read(f, H5FD_MEM_DRAW, addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + for (i=0; i < 100; i++) { + if(data[i] != i+350) { + HDfprintf(stderr, "Read different values than written\n"); + TEST_ERROR; + } /* end if */ + } /* end for */ + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* Changes: 1200 */ + search_addr = addr + sizeof(int)*1200; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 200 */ + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 400 */ + search_addr = addr + sizeof(int)*400; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* update elements 200 - 700 to value 0, this will go to disk but + also discarding existing pages from the PB (page 200). */ + /* Changes: 400 - 1400; 400 */ + for(i=0 ; i<1000 ; i++) + data[i] = 0; + if(H5F_block_write(f, H5FD_MEM_DRAW, addr+(sizeof(int)*400), sizeof(int)*1000, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + page_count -= 1; + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* Changes: 200 */ + search_addr = addr + sizeof(int)*200; + if(NULL == H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + /* Changes: 400 */ + search_addr = addr + sizeof(int)*400; + if(NULL != H5SL_search(f->shared->page_buf->slist_ptr, &(search_addr))) + FAIL_STACK_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + return 1; +} /* test_lru_processing */ + + +/*------------------------------------------------------------------------- + * Function: test_min_threshold() + * + * Purpose: Tests verifying observation of minimum and maximum + * raw and metadata page counts in the page buffer. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added the base_raw_cnt and base_meta_cnt fields and + * supporting code. This allows the test to adjust to the + * number of page buffer pages accessed during file open / + * create. + * + * The tests for the values of base_raw_cnt and base_meta_cnt + * just after file open exist detect changes in library + * behavior. Assuming any such change is not indicative of + * other issues, these tests can be modified to reflect the + * change. + * + * JRM -- 2/23/17 + * + *------------------------------------------------------------------------- + */ + +static unsigned +test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + size_t base_raw_cnt = 0; + size_t base_meta_cnt = 0; + size_t page_count = 0; + int i; + int num_elements = 1000; + H5PB_t *page_buf; + haddr_t meta_addr = HADDR_UNDEF; + haddr_t raw_addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("Minimum Metadata threshold Processing"); + HDprintf("\n"); + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR; + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + FAIL_STACK_ERROR; + + HDprintf("\tMinimum metadata threshold = 100%%\n"); + + /* keep 5 pages at max in the page buffer and 5 meta page minimum */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 100, 0) < 0) + FAIL_STACK_ERROR; + + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the raw and meta counts now, so we can adjust tests accordingly. + */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->page_buf); + + base_raw_cnt = f->shared->page_buf->raw_count; + base_meta_cnt = f->shared->page_buf->meta_count; + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 5) + TEST_ERROR; + + if(page_buf->min_raw_count != 0) + TEST_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* write all raw data, this would end up in page buffer since there + * is no metadata yet + * + * Not necessarily -- opening the file may may load a metadata page. + */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->raw_count != 5 - base_meta_cnt) + TEST_ERROR; + + /* write all meta data, this would end up in page buffer */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->meta_count != 5) + TEST_ERROR; + + if(page_buf->raw_count != 0) + TEST_ERROR; + + /* write and read more raw data and make sure that they don't end up in + * page buffer since the minimum metadata is actually the entire + * page buffer + */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->meta_count != 5) + TEST_ERROR; + + if(page_buf->raw_count != 0) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + + HDprintf("\tMinimum raw data threshold = 100%%\n"); + + page_count = 0; + + /* keep 5 pages at max in the page buffer and 5 raw page minimum */ + /* Changes: 1000 */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 0, 100) < 0) + TEST_ERROR; + + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the raw and meta counts now, so we can adjust tests accordingly. + */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->page_buf); + + base_raw_cnt = f->shared->page_buf->raw_count; + base_meta_cnt = f->shared->page_buf->meta_count; + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 0) + TEST_ERROR; + if(page_buf->min_raw_count != 5) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + TEST_ERROR; + + /* write all meta data, this would end up in page buffer since there + * is no raw data yet + */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + if(page_buf->meta_count != 5 - base_raw_cnt) + TEST_ERROR; + + /* write/read all raw data, this would end up in page buffer */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->raw_count != 5) + TEST_ERROR; + + if(page_buf->meta_count != 0) + TEST_ERROR; + + /* write and read more meta data and make sure that they don't end up in + * page buffer since the minimum metadata is actually the entire + * page buffer + */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*350), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*750), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*900), sizeof(int)*50, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(page_buf->raw_count != 5) + TEST_ERROR; + + if(page_buf->meta_count != 0) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + + HDprintf("\tMinimum metadata threshold = 40%%, Minimum rawdata threshold = 40%%\n"); + page_count = 0; + /* keep 5 pages at max in the page buffer 2 meta pages, 2 raw pages + * minimum + */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 40, 40) < 0) + TEST_ERROR; + + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * + * However, with the current 1 metadata page inserted into the + * the page buffer, it is not necessary to track the base raw and + * metadata entry counts. + */ + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 2) + TEST_ERROR; + + if(page_buf->min_raw_count != 2) + TEST_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* fill the page buffer with raw data */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 5 - base_meta_cnt) + TEST_ERROR; + + /* add 3 meta entries evicting 3 raw entries */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 3) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 2) + TEST_ERROR; + + /* adding more meta entires should replace meta entries since raw data + * is at its minimum + */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 3) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 2) + TEST_ERROR; + + /* bring existing raw entires up the LRU */ + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*750), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* adding 2 raw entries (even with 1 call) should only evict 1 meta + * entry and another raw entry + */ + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*350), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 2) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 3) + TEST_ERROR; + + /* adding 2 meta entries should replace 2 entires at the bottom of the LRU */ + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*98), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*242), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 2) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 3) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + HDprintf("\tMinimum metadata threshold = 20%%\n"); + page_count = 0; + /* keep 5 pages at max in the page buffer and 1 meta page minimum */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 39, 0) < 0) + TEST_ERROR; + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + page_buf = f->shared->page_buf; + + if(page_buf->min_meta_count != 1) + TEST_ERROR; + if(page_buf->min_raw_count != 0) + TEST_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* fill the page buffer with raw data */ + for(i=0 ; i<100 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + page_count += 5; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + /* add 2 meta entries evicting 2 raw entries */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 2) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 3) + TEST_ERROR; + + /* bring the rest of the raw entries up the LRU */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + /* write one more raw entry which replace one meta entry */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 1) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 4) + TEST_ERROR; + + /* write one more raw entry which should replace another raw entry + * keeping min threshold of meta entries + */ + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 1) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 4) + TEST_ERROR; + + /* write a metadata entry that should replace the metadata entry + * at the bottom of the LRU + */ + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5SL_count(f->shared->page_buf->slist_ptr) != page_count) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->meta_count != 1) + TEST_ERROR; + + if(f->shared->page_buf->raw_count != 4) + TEST_ERROR; + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + + HDfree(data); + + PASSED() + + return 0; + +error: + + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + + return 1; + +} /* test_min_threshold */ + + +/*------------------------------------------------------------------------- + * Function: test_stats_collection() + * + * Purpose: Tests verifying correct collection of statistics + * by the page buffer. + * + * Any data mis-matches or failures reported by the HDF5 + * library result in test failure. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + * Changes: Added the base_raw_cnt and base_meta_cnt fields and + * supporting code. This allows the test to adjust to the + * number of page buffer pages accessed during file open / + * create. + * + * The tests for the values of base_raw_cnt and base_meta_cnt + * just after file open exist detect changes in library + * behavior. Assuming any such change is not indicative of + * other issues, these tests can be modified to reflect the + * change. + * + * JRM -- 2/23/17 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + int i; + int num_elements = 1000; + size_t base_raw_cnt = 0; + size_t base_meta_cnt = 0; + haddr_t meta_addr = HADDR_UNDEF; + haddr_t raw_addr = HADDR_UNDEF; + int *data = NULL; + H5F_t *f = NULL; + + TESTING("Statistics Collection"); + + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + if((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if(set_multi_split(env_h5_drvr, fapl, sizeof(int)*200) != 0) + TEST_ERROR; + + if((data = (int *)HDcalloc((size_t)num_elements, sizeof(int))) == NULL) + TEST_ERROR + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + TEST_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + TEST_ERROR; + + if(H5Pset_file_space_page_size(fcpl, sizeof(int)*200) < 0) + TEST_ERROR; + + /* keep 5 pages at max in the page buffer */ + if(H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 20, 0) < 0) + TEST_ERROR; + + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Get a pointer to the internal file object */ + if(NULL == (f = (H5F_t *)H5I_object(file_id))) + FAIL_STACK_ERROR; + + /* opening the file inserts one or more pages into the page buffer. + * Get the raw and meta counts now, so we can adjust the expected + * statistics accordingly. + */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->page_buf); + + base_raw_cnt = f->shared->page_buf->raw_count; + base_meta_cnt = f->shared->page_buf->meta_count; + + if(base_raw_cnt != 0) + TEST_ERROR; + + if(base_meta_cnt != 1) + TEST_ERROR; + + /* reset statistics before we begin the tests */ + if(H5Freset_page_buffering_stats(file_id) < 0) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + if(HADDR_UNDEF == (raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements))) + FAIL_STACK_ERROR; + + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + for(i=0 ; i<200 ; i++) + data[i] = i; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*500), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*700), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*900), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*200, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*300), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*182, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*200), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*600), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_rawdata_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*400), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*600), sizeof(int)*200, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(H5F_block_read(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*800), sizeof(int)*100, H5AC_ind_read_dxpl_id, data) < 0) + FAIL_STACK_ERROR; + + if(f->shared->page_buf->accesses[0] != 8) + TEST_ERROR; + if(f->shared->page_buf->accesses[1] != 16) + TEST_ERROR; + + if(f->shared->page_buf->bypasses[0] != 3) + TEST_ERROR; + if(f->shared->page_buf->bypasses[1] != 1) + TEST_ERROR; + + if(f->shared->page_buf->hits[0] != 0) + TEST_ERROR; + if(f->shared->page_buf->hits[1] != 4) + TEST_ERROR; + + if(f->shared->page_buf->misses[0] != 8) + TEST_ERROR; + if(f->shared->page_buf->misses[1] != 11) + TEST_ERROR; + + if(f->shared->page_buf->evictions[0] != 5 + base_meta_cnt) + TEST_ERROR; + if(f->shared->page_buf->evictions[1] != 9 + base_raw_cnt) + TEST_ERROR; + + { + unsigned accesses[2]; + unsigned hits[2]; + unsigned misses[2]; + unsigned evictions[2]; + unsigned bypasses[2]; + + if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, evictions, bypasses) < 0) + FAIL_STACK_ERROR; + + if(accesses[0] != 8) + TEST_ERROR; + if(accesses[1] != 16) + TEST_ERROR; + if(bypasses[0] != 3) + TEST_ERROR; + if(bypasses[1] != 1) + TEST_ERROR; + if(hits[0] != 0) + TEST_ERROR; + if(hits[1] != 4) + TEST_ERROR; + if(misses[0] != 8) + TEST_ERROR; + if(misses[1] != 11) + TEST_ERROR; + if(evictions[0] != 5 + base_meta_cnt) + TEST_ERROR; + if(evictions[1] != 9 + base_raw_cnt) + TEST_ERROR; + + if(H5Freset_page_buffering_stats(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Fget_page_buffering_stats(file_id, accesses, hits, misses, evictions, bypasses) < 0) + FAIL_STACK_ERROR; + + if(accesses[0] != 0) + TEST_ERROR; + if(accesses[1] != 0) + TEST_ERROR; + if(bypasses[0] != 0) + TEST_ERROR; + if(bypasses[1] != 0) + TEST_ERROR; + if(hits[0] != 0) + TEST_ERROR; + if(hits[1] != 0) + TEST_ERROR; + if(misses[0] != 0) + TEST_ERROR; + if(misses[1] != 0) + TEST_ERROR; + if(evictions[0] != 0) + TEST_ERROR; + if(evictions[1] != 0) + TEST_ERROR; + } /* end block */ + + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + HDfree(data); + + + PASSED() + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + if(data) + HDfree(data); + } H5E_END_TRY; + + return 1; +} /* test_stats_collection */ + + +/*------------------------------------------------------------------------- + * Function: main() + * + * Purpose: Main function for the page buffer tests. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: unknown + * ?? / ?? / ?? + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + hid_t fapl = -1; /* File access property list for data files */ + unsigned nerrors = 0; /* Cumulative error count */ + const char *env_h5_drvr = NULL; /* File Driver value from environment */ + +#ifndef H5_HAVE_PARALLEL + + h5_reset(); + + /* Get the VFD to use */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* Temporary skip testing with multi/split drivers: + * Page buffering depends on paged aggregation which is + * currently disabled for multi/split drivers. + */ + if((0 == HDstrcmp(env_h5_drvr, "multi")) || + (0 == HDstrcmp(env_h5_drvr, "split"))) { + + SKIPPED() + HDputs("Skip page buffering test because paged aggregation is disabled for multi/split drivers"); + HDexit(EXIT_SUCCESS); + } /* end if */ + + if((fapl = h5_fileaccess()) < 0) { + nerrors++; + PUTS_ERROR("Can't get VFD-dependent fapl") + } /* end if */ + + nerrors += test_args(fapl, env_h5_drvr); + nerrors += test_raw_data_handling(fapl, env_h5_drvr); + nerrors += test_lru_processing(fapl, env_h5_drvr); + nerrors += test_min_threshold(fapl, env_h5_drvr); + nerrors += test_stats_collection(fapl, env_h5_drvr); + + h5_clean_files(FILENAME, fapl); + + if(nerrors) + goto error; + + HDputs("All Page Buffering tests passed."); + +#else + SKIPPED() + HDputs("Page Buffering is disabled for parallel."); +#endif + + HDexit(EXIT_SUCCESS); + +error: + + HDprintf("***** %d Page Buffering TEST%s FAILED! *****\n", + nerrors, nerrors > 1 ? "S" : ""); + + H5E_BEGIN_TRY { + H5Pclose(fapl); + } H5E_END_TRY; + + HDexit(EXIT_FAILURE); + +} /* main() */ + diff --git a/tools/test/h5diff/testfiles/h5diff_dset_idx1.h5 b/test/paged_nopersist.h5 Binary files differindex 3252303..b6c945a 100644 --- a/tools/test/h5diff/testfiles/h5diff_dset_idx1.h5 +++ b/test/paged_nopersist.h5 diff --git a/test/paged_persist.h5 b/test/paged_persist.h5 Binary files differnew file mode 100644 index 0000000..f0ae836 --- /dev/null +++ b/test/paged_persist.h5 diff --git a/test/set_extent.c b/test/set_extent.c index a992419..17dc6a0 100644 --- a/test/set_extent.c +++ b/test/set_extent.c @@ -139,6 +139,15 @@ int main( void ) unsigned new_format; /* Whether to use the latest file format */ unsigned chunk_cache; /* Whether to enable chunk caching */ int nerrors = 0; + const char *env_h5_drvr; /* File Driver value from environment */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + + + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); /* Initialize random number seed */ HDsrandom((unsigned)HDtime(NULL)); @@ -194,12 +203,15 @@ int main( void ) H5F_LIBVER_LATEST) < 0) TEST_ERROR /* Tests which use chunked datasets */ - nerrors += do_ranks( my_fapl, new_format ) < 0 ? 1 : 0; + if(!new_format || (new_format && contig_addr_vfd)) + nerrors += do_ranks( my_fapl, new_format ) < 0 ? 1 : 0; } /* end for */ /* Tests which do not use chunked datasets */ - nerrors += test_external( fapl ) < 0 ? 1 : 0; - nerrors += do_layouts( fapl ) < 0 ? 1 : 0; + if(!new_format || (new_format && contig_addr_vfd)) { + nerrors += test_external( fapl ) < 0 ? 1 : 0; + nerrors += do_layouts( fapl ) < 0 ? 1 : 0; + } } /* end for */ /* Close 2nd FAPL */ diff --git a/test/stab.c b/test/stab.c index f81bb5f..00aee21 100644 --- a/test/stab.c +++ b/test/stab.c @@ -96,7 +96,7 @@ const char *FILENAME[] = { *------------------------------------------------------------------------- */ static int -test_misc(hid_t fapl, hbool_t new_format) +test_misc(hid_t fcpl, hid_t fapl, hbool_t new_format) { hid_t fid = (-1); /* File ID */ hid_t g1 = (-1), g2 = (-1), g3 = (-1); @@ -110,7 +110,7 @@ test_misc(hid_t fapl, hbool_t new_format) /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR /* Create initial groups for testing, then close */ if((g1 = H5Gcreate2(fid, "test_1a", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR @@ -177,7 +177,7 @@ test_misc(hid_t fapl, hbool_t new_format) *------------------------------------------------------------------------- */ static int -test_long(hid_t fapl, hbool_t new_format) +test_long(hid_t fcpl, hid_t fapl, hbool_t new_format) { hid_t fid = (-1); /* File ID */ hid_t g1 = (-1), g2 = (-1); @@ -192,7 +192,7 @@ test_long(hid_t fapl, hbool_t new_format) /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR /* Group names */ name1 = (char *)HDmalloc((size_t)LONG_NAME_LEN); @@ -252,7 +252,7 @@ error: *------------------------------------------------------------------------- */ static int -test_large(hid_t fapl, hbool_t new_format) +test_large(hid_t fcpl, hid_t fapl, hbool_t new_format) { hid_t fid = (-1); /* File ID */ hid_t cwg = (-1), dir = (-1); /* Group IDs */ @@ -267,7 +267,7 @@ test_large(hid_t fapl, hbool_t new_format) /* Create file */ h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR /* * Create a directory that has so many entries that the root @@ -318,7 +318,7 @@ test_large(hid_t fapl, hbool_t new_format) *------------------------------------------------------------------------- */ static int -lifecycle(hid_t fapl2) +lifecycle(hid_t fcpl, hid_t fapl2) { hid_t fid = (-1); /* File ID */ hid_t gid = (-1); /* Group ID */ @@ -341,7 +341,7 @@ lifecycle(hid_t fapl2) /* Create file */ h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl2)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) TEST_ERROR /* Close file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -532,7 +532,7 @@ error: *------------------------------------------------------------------------- */ static int -long_compact(hid_t fapl2) +long_compact(hid_t fcpl, hid_t fapl2) { hid_t fid = (-1); /* File ID */ hid_t gid = (-1); /* Group ID */ @@ -546,7 +546,7 @@ long_compact(hid_t fapl2) /* Create file */ h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl2)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) TEST_ERROR /* Close file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -755,7 +755,7 @@ error: *------------------------------------------------------------------------- */ static int -no_compact(hid_t fapl2) +no_compact(hid_t fcpl, hid_t fapl2) { hid_t fid = (-1); /* File ID */ hid_t gid = (-1); /* Group ID */ @@ -772,7 +772,7 @@ no_compact(hid_t fapl2) /* Create file */ h5_fixname(FILENAME[0], fapl2, filename, sizeof(filename)); - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl2)) < 0) TEST_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl2)) < 0) TEST_ERROR /* Close file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -1165,9 +1165,20 @@ error: int main(void) { - hid_t fapl, fapl2; /* File access property list IDs */ - unsigned new_format; /* Whether to use the new format or not */ - int nerrors = 0; + hid_t fapl, fapl2; /* File access property list IDs */ + hid_t fcpl, fcpl2; /* File creation property list ID */ + unsigned new_format; /* Whether to use the new format or not */ + const char *env_h5_drvr; /* File Driver value from environment */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + int nerrors = 0; + + /* Get the VFD to use */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + + /* VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); /* Reset library */ h5_reset(); @@ -1179,20 +1190,42 @@ main(void) /* Set the "use the latest version of the format" bounds for creating objects in the file */ if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR + /* Set up file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) TEST_ERROR + if((fcpl2 = H5Pcopy(fcpl)) < 0) TEST_ERROR + + /* Set to use paged aggregation strategy and persisting free-space */ + /* Skip testing for multi/split drivers */ + if(H5Pset_file_space_strategy(fcpl2, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)1) < 0) + TEST_ERROR + /* Loop over using new group format */ for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl = fapl; + hid_t my_fcpl = fcpl; + + if(!contig_addr_vfd) + continue; + + if(new_format) { + my_fapl = fapl2; + my_fcpl = fcpl2; /* Set to use paged aggregation and persisting free-space */ + } + /* Perform basic tests, with old & new style groups */ - nerrors += test_misc((new_format ? fapl2 : fapl), new_format); - nerrors += test_long((new_format ? fapl2 : fapl), new_format); - nerrors += test_large((new_format ? fapl2 : fapl), new_format); + nerrors += test_misc(my_fcpl, my_fapl, new_format); + nerrors += test_long(my_fcpl, my_fapl, new_format); + nerrors += test_large(my_fcpl, my_fapl, new_format); } /* end for */ /* New format group specific tests (require new format features) */ - nerrors += lifecycle(fapl2); - nerrors += long_compact(fapl2); - nerrors += read_old(); - nerrors += no_compact(fapl2); - nerrors += gcpl_on_root(fapl2); + if(contig_addr_vfd) { + nerrors += lifecycle(fcpl2, fapl2); + nerrors += long_compact(fcpl2, fapl2); + nerrors += read_old(); + nerrors += no_compact(fcpl2, fapl2); + nerrors += gcpl_on_root(fapl2); + } /* Old group API specific tests */ nerrors += old_api(fapl); @@ -1200,6 +1233,8 @@ main(void) /* Close 2nd FAPL */ H5Pclose(fapl2); + H5Pclose(fcpl); + H5Pclose(fcpl2); /* Verify symbol table messages are cached */ nerrors += (h5_verify_cached_stabs(FILENAME, fapl) < 0 ? 1 : 0); diff --git a/test/testfiles/plist_files/def_fcpl_32be b/test/testfiles/plist_files/def_fcpl_32be Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_32be +++ b/test/testfiles/plist_files/def_fcpl_32be diff --git a/test/testfiles/plist_files/def_fcpl_32le b/test/testfiles/plist_files/def_fcpl_32le Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_32le +++ b/test/testfiles/plist_files/def_fcpl_32le diff --git a/test/testfiles/plist_files/def_fcpl_64be b/test/testfiles/plist_files/def_fcpl_64be Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_64be +++ b/test/testfiles/plist_files/def_fcpl_64be diff --git a/test/testfiles/plist_files/def_fcpl_64le b/test/testfiles/plist_files/def_fcpl_64le Binary files differindex 38dec23..9a67dd5 100644 --- a/test/testfiles/plist_files/def_fcpl_64le +++ b/test/testfiles/plist_files/def_fcpl_64le diff --git a/test/testfiles/plist_files/fcpl_32be b/test/testfiles/plist_files/fcpl_32be Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_32be +++ b/test/testfiles/plist_files/fcpl_32be diff --git a/test/testfiles/plist_files/fcpl_32le b/test/testfiles/plist_files/fcpl_32le Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_32le +++ b/test/testfiles/plist_files/fcpl_32le diff --git a/test/testfiles/plist_files/fcpl_64be b/test/testfiles/plist_files/fcpl_64be Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_64be +++ b/test/testfiles/plist_files/fcpl_64be diff --git a/test/testfiles/plist_files/fcpl_64le b/test/testfiles/plist_files/fcpl_64le Binary files differindex 3ce8bf4..4a8ac8a 100644 --- a/test/testfiles/plist_files/fcpl_64le +++ b/test/testfiles/plist_files/fcpl_64le diff --git a/test/tfile.c b/test/tfile.c index 6987493..295a29c 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -104,13 +104,13 @@ /* Declarations for test_filespace_*() */ #define FILENAME_LEN 1024 /* length of file name */ -#define CORE_INCREMENT 1024 /* core file */ -#define FAMILY_SIZE 1024 /* family file */ #define DSETNAME "dset" /* Name of dataset */ #define NELMTS(X) (sizeof(X)/sizeof(X[0])) /* # of elements */ #define READ_OLD_BUFSIZE 1024 /* Buffer for holding file data */ #define FILE5 "tfile5.h5" /* Test file */ #define TEST_THRESHOLD10 10 /* Free space section threshold */ +#define FSP_SIZE_DEF 4096 /* File space page size default */ +#define FSP_SIZE512 512 /* File space page size */ /* Declaration for test_libver_macros2() */ #define FILE6 "tfile6.h5" /* Test file */ @@ -125,17 +125,30 @@ const char *OLD_FILENAME[] = { "filespace_1_6.h5", /* 1.6 HDF5 file */ "filespace_1_8.h5" /* 1.8 HDF5 file */ }; -const char *FILESPACE_NAME[] = { - "tfilespace", - NULL + +/* Files created in 1.10.0 release --used in test_filespace_1.10.0_compatible() */ +/* These files are copied from release 1.10.0 tools/h5format_convert/testfiles */ +const char *OLD_1_10_0_FILENAME[] = { + "h5fc_ext1_i.h5", /* 0 */ + "h5fc_ext1_f.h5", /* 1 */ + "h5fc_ext2_if.h5", /* 2 */ + "h5fc_ext2_sf.h5", /* 3 */ + "h5fc_ext3_isf.h5", /* 4 */ + "h5fc_ext_none.h5" /* 5 */ +}; + +/* Files used in test_filespace_round_compatible() */ +const char *FSPACE_FILENAMES[] = { + "fsm_aggr_nopersist.h5", /* H5F_FILE_SPACE_AGGR, not persisting free-space */ + "fsm_aggr_persist.h5", /* H5F_FILE_SPACE_AGGR, persisting free-space */ + "paged_nopersist.h5", /* H5F_FILE_SPACE_PAGE, not persisting free-space */ + "paged_persist.h5", /* H5F_FILE_SPACE_PAGE, persisting free-space */ + "aggr.h5", /* H5F_FILE_SPACE_AGGR */ + "none.h5" /* H5F_FILE_SPACE_NONE */ }; -const char *FILENAME[] = { - "sec2_tfile", - "split_tfile", - "stdio_tfile", - "core_tfile", - "family_tfile", +const char *FILESPACE_NAME[] = { + "tfilespace", NULL }; @@ -1538,112 +1551,7 @@ test_file_perm2(void) CHECK(ret, FAIL, "H5Sclose"); } /* end test_file_perm2() */ -/**************************************************************** -** -** test_file_freespace(): low-level file test routine. -** This test checks the free space available in a file in various -** situations. -** -** Modifications: -** Vailin Choi; July 2012 -** Remove datasets in reverse order so that all file spaces are shrunk. -** (A change due to H5FD_FLMAP_DICHOTOMY.) -** -*****************************************************************/ -static void -test_file_freespace(void) -{ - hid_t file; /* File opened with read-write permission */ - h5_stat_size_t empty_filesize; /* Size of file when empty */ - h5_stat_size_t mod_filesize; /* Size of file after being modified */ - hssize_t free_space; /* Amount of free space in file */ - hid_t dspace; /* Dataspace ID */ - hid_t dset; /* Dataset ID */ - hid_t dcpl; /* Dataset creation property list */ - int k; /* Local index variable */ - unsigned u; /* Local index variable */ - char name[32]; /* Dataset name */ - herr_t ret; - - /* Output message about test being performed */ - MESSAGE(5, ("Testing Low-Level File Free Space\n")); - - /* Create an "empty" file */ - file = H5Fcreate(FILE1, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - CHECK(file, FAIL, "H5Fcreate"); - - ret = H5Fclose(file); - CHECK_I(ret, "H5Fclose"); - - /* Get the "empty" file size */ - empty_filesize = h5_get_file_size(FILE1, H5P_DEFAULT); - - /* Re-open the file (with read-write permission) */ - file = H5Fopen(FILE1, H5F_ACC_RDWR, H5P_DEFAULT); - CHECK_I(file, "H5Fopen"); - - /* Check that the free space is 0 */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - VERIFY(free_space, 0, "H5Fget_freespace"); - - /* Create dataspace for datasets */ - dspace = H5Screate(H5S_SCALAR); - CHECK(dspace, FAIL, "H5Screate"); - - /* Create a dataset creation property list */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - CHECK(dcpl, FAIL, "H5Pcreate"); - - /* Set the space allocation time to early */ - ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); - CHECK(ret, FAIL, "H5Pset_alloc_time"); - - /* Create datasets in file */ - for(u = 0; u < 10; u++) { - sprintf(name, "Dataset %u", u); - dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); - CHECK(dset, FAIL, "H5Dcreate2"); - - ret = H5Dclose(dset); - CHECK(ret, FAIL, "H5Dclose"); - } /* end for */ - - /* Close dataspace */ - ret = H5Sclose(dspace); - CHECK(ret, FAIL, "H5Sclose"); - - /* Close dataset creation property list */ - ret = H5Pclose(dcpl); - CHECK(ret, FAIL, "H5Pclose"); - - /* Check that there is the right amount of free space in the file */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - VERIFY(free_space, 2360, "H5Fget_freespace"); - /* Delete datasets in file */ - for(k = 9; k >= 0; k--) { - sprintf(name, "Dataset %u", (unsigned)k); - ret = H5Ldelete(file, name, H5P_DEFAULT); - CHECK(ret, FAIL, "H5Ldelete"); - } /* end for */ - - /* Check that there is the right amount of free space in the file */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - VERIFY(free_space, 0, "H5Fget_freespace"); - - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); - - /* Get the file size after modifications*/ - mod_filesize = h5_get_file_size(FILE1, H5P_DEFAULT); - - /* Check that the file reverted to empty size */ - VERIFY(mod_filesize, empty_filesize, "H5Fget_freespace"); -} /* end test_file_freespace() */ /**************************************************************** ** @@ -3185,419 +3093,1227 @@ test_userblock_alignment(void) /**************************************************************** ** -** test_free_sections(): -** This routine does the actual work of checking information for -** free space sections available in a file in various situations. +** test_userblock_alignment_paged(): low-level file test routine. +** This test checks to ensure that files with both a userblock and +** alignment interact properly: +** -- alignment via H5Pset_alignment +** -- alignment via paged aggregation +** +** Programmer: Vailin Choi; March 2013 ** *****************************************************************/ static void -test_free_sections(hid_t fapl, char *fname) +test_userblock_alignment_paged(void) { - hid_t file; /* File ID */ - hid_t fcpl; /* File creation property list template */ - hssize_t free_space; /* Amount of free space in file */ - hid_t dspace; /* Dataspace ID */ - hid_t dset; /* Dataset ID */ - hid_t dcpl; /* Dataset creation property list */ - unsigned u; /* Local index variable */ - char name[32]; /* Dataset name */ - hssize_t nsects; /* # of free-space sections */ - hssize_t saved_nsects; /* saved copy for the # of free-space sections */ - int i; /* local index variable */ - hsize_t total; /* sum of the free-space section sizes */ - hsize_t last_size; /* size of last free-space section */ - H5F_sect_info_t *sect_info; /* array to hold the free-space information */ - H5F_sect_info_t *saved_sect_info; /* array to hold the free-space information */ - herr_t ret; /* return value */ - - /* Create file-creation template */ + hid_t fid; /* File ID */ + hid_t fcpl; /* File creation property list ID */ + hid_t fapl; /* File access property list ID */ + herr_t ret; /* Generic return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("Testing interaction between userblock and alignment (via paged aggregation and H5Pset_alignment)\n")); + + /* + * Case 1: + * Userblock size = 0 + * Alignment in use = 4096 + * Strategy = H5F_FILE_SPACE_PAGE; fsp_size = alignment = 4096 + * Outcome: + * Should succeed: + * userblock is 0 and alignment != 0 + */ + /* Create file creation property list with user block */ fcpl = H5Pcreate(H5P_FILE_CREATE); CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)0); + CHECK(ret, FAIL, "H5Pset_userblock"); - /* Set file space strategy and free space section threshold */ - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Create file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); - /* Create the file */ - file = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); - CHECK(file, FAIL, "H5Fcreate"); + /* Set the "use the latest version of the format" bounds */ + ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); - /* Close the FCPL */ + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ ret = H5Pclose(fcpl); CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - /* Create dataspace for datasets */ - dspace = H5Screate(H5S_SCALAR); - CHECK(dspace, FAIL, "H5Screate"); + /* + * Case 2a: + * Userblock size = 1024 + * Alignment in use = 512 + * Strategy = H5F_FILE_SPACE_PAGE; fsp_size = alignment = 512 + * H5Pset_alignment() is 3 + * Outcome: + * Should succeed: + * userblock (1024) is integral mult. of alignment (512) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); - /* Create a dataset creation property list */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - CHECK(dcpl, FAIL, "H5Pcreate"); + /* Create file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); - /* Set the space allocation time to early */ - ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); - CHECK(ret, FAIL, "H5Pset_alloc_time"); + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - /* Create datasets in file */ - for(u = 0; u < 10; u++) { - sprintf(name, "Dataset %u", u); - dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); - CHECK(dset, FAIL, "H5Dcreate2"); + /* + * Case 2b: + * Userblock size = 1024 + * Alignment in use = 3 + * Strategy = H5F_FILE_SPACE_AGGR; fsp_size = 512 + * (via default file creation property) + * H5Pset_alignment() is 3 + * Outcome: + * Should fail at file creation: + * userblock (1024) is non-integral mult. of alignment (3) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); - ret = H5Dclose(dset); - CHECK(ret, FAIL, "H5Dclose"); - } /* end for */ + /* Create file access property list */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Close dataspace */ - ret = H5Sclose(dspace); - CHECK(ret, FAIL, "H5Sclose"); + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); - /* Close dataset creation property list */ - ret = H5Pclose(dcpl); + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); CHECK(ret, FAIL, "H5Pclose"); - /* Delete odd-numbered datasets in file */ - for(u = 0; u < 10; u++) { - sprintf(name, "Dataset %u", u); - if(u % 2) { - ret = H5Ldelete(file, name, H5P_DEFAULT); - CHECK(ret, FAIL, "H5Ldelete"); - } /* end if */ - } /* end for */ + /* + * Case 3a: + * Userblock size = 512 + * Alignment in use = 512 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 512 + * H5Pset_alignment() is 3 + * Outcome: + * Should succeed: + * userblock (512) is equal to alignment (512) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Re-open the file with read-only permission */ - file = H5Fopen(fname, H5F_ACC_RDONLY, fapl); - CHECK_I(file, "H5Fopen"); - - /* Get the amount of free space in the file */ - free_space = H5Fget_freespace(file); - CHECK(free_space, FAIL, "H5Fget_freespace"); - - /* Get the # of free-space sections in the file */ - saved_nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, NULL); - CHECK(saved_nsects, FAIL, "H5Fget_free_sections"); - - /* Allocate storage for the free space section information */ - saved_sect_info = (H5F_sect_info_t *)HDcalloc((size_t)saved_nsects, sizeof(H5F_sect_info_t)); - CHECK(saved_sect_info, NULL, "HDcalloc"); - - /* Should return failure when nsects is 0 with a nonnull sect_info */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, saved_sect_info); - VERIFY(nsects, FAIL, "H5Fget_free_sections"); - - /* Verify the correct # of free-space sections */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)saved_nsects, saved_sect_info); - VERIFY(nsects, saved_nsects, "H5Fget_free_sections"); - - /* Verify the amount of free-space is correct */ - total = 0; - for(i = 0; i < nsects; i++) - total += saved_sect_info[i].size; - VERIFY(free_space, total, "H5Fget_free_sections"); - - /* save the last section's size */ - last_size = saved_sect_info[nsects-1].size; - - /* Allocate storage for -1 free space section information */ - sect_info = (H5F_sect_info_t *)HDcalloc((size_t)(saved_nsects - 1), sizeof(H5F_sect_info_t)); - CHECK(sect_info, NULL, "HDcalloc"); - - /* Retrieve free space info for -1 sections */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(saved_nsects - 1), sect_info); - VERIFY(nsects, saved_nsects, "H5Fget_free_sections"); - - /* Verify the amount of free-space is correct */ - total = 0; - for(i = 0; i < (saved_nsects - 1); i++) { - VERIFY(sect_info[i].addr, saved_sect_info[i].addr, "H5Fget_free_sections"); - VERIFY(sect_info[i].size, saved_sect_info[i].size, "H5Fget_free_sections"); - total += sect_info[i].size; - } + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); - VERIFY(((hsize_t)free_space - last_size), total, "H5Fget_free_sections"); - HDfree(sect_info); + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - /* Allocate storage for +1 free space section information */ - sect_info = (H5F_sect_info_t *)HDcalloc((size_t)(saved_nsects + 1), sizeof(H5F_sect_info_t)); - CHECK(sect_info, NULL, "HDcalloc"); + /* + * Case 3b: + * Userblock size = 512 + * Alignment in use = 3 + * Strategy is H5F_FILE_SPACE_NONE; fsp_size = 512 + * H5Pset_alignment() is 3 + * Outcome: + * Should fail at file creation: + * userblock (512) is non-integral mult. of alignment (3) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); - /* Retrieve free-space info for +1 sections */ - nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(saved_nsects + 1), sect_info); - VERIFY(nsects, saved_nsects, "H5Fget_free_sections"); + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); - /* Verify free-space info is correct */ - total = 0; - for(i = 0; i < nsects; i++) { - VERIFY(sect_info[i].addr, saved_sect_info[i].addr, "H5Fget_free_sections"); - VERIFY(sect_info[i].size, saved_sect_info[i].size, "H5Fget_free_sections"); - total += sect_info[i].size; - } + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); - VERIFY(sect_info[nsects].addr, 0, "H5Fget_free_sections"); - VERIFY(sect_info[nsects].size, 0, "H5Fget_free_sections"); - VERIFY(free_space, total, "H5Fget_free_sections"); - HDfree(sect_info); + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); - /* Verify that there is no free-space section for this type */ - nsects = H5Fget_free_sections(file, H5FD_MEM_BTREE, (size_t)0, NULL); - VERIFY(nsects, 0, "H5Fget_free_sections"); + /* + * Case 4a: + * Userblock size = 1024 + * Alignment in use = 1023 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 1023 + * H5Pset_alignment() is 16 + * Outcome: + * Should fail at file creation: + * userblock (1024) is non-integral multiple of alignment (1023) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1023); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); - /* Close file */ - ret = H5Fclose(file); - CHECK(ret, FAIL, "H5Fclose"); + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); - HDfree(saved_sect_info); + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); -} /* end test_free_sections() */ + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 4b: + * Userblock size = 1024 + * Alignment in use = 16 + * Strategy is H5F_FILE_SPACE_FSM_AGGR; fsp_size = 1023 + * H5Pset_alignment() is 16 + * Outcome: + * Should succeed: + * userblock (512) is integral multiple of alignment (16) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1023); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 5a: + * Userblock size = 512 + * Alignment in use = 1024 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 1024 + * H5Pset_alignment() is 16 + * Outcome: + * Should fail at file creation: + * userblock (512) is less than alignment (1024) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Create a file with FAPL & FCPL */ + H5E_BEGIN_TRY { + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VERIFY(fid, FAIL, "H5Fcreate"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 5b: + * Userblock size = 512 + * Alignment in use = 16 + * Strategy is H5F_FILE_SPACE_NONE; fsp_size = 1024 + * H5Pset_alignment() is 16 + * Outcome: + * Should succed: + * userblock (512) is integral multiple of alignment (16) + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)16); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case 6: + * Userblock size = 512 + * Alignment in use = 512 + * Strategy is H5F_FILE_SPACE_PAGE; fsp_size = alignment = 512 + * H5Pset_alignment() is 3 + * Reopen the file; H5Pset_alignment() is 1024 + * Outcome: + * Should succed: + * Userblock (512) is the same as alignment (512); + * The H5Pset_alignment() calls have no effect + */ + /* Create file creation property list with user block */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + ret = H5Pset_userblock(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_userblock"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + + /* Create file access property list with alignment */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + CHECK(fapl, FAIL, "H5Pcreate"); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)3); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper1(fcpl, fapl); + CHECK(ret, FAIL, "test_userblock_alignment_helper1"); + + /* Change alignment in FAPL */ + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_alignment"); + + /* Call helper routines to perform file manipulations */ + ret = test_userblock_alignment_helper2(fapl, FALSE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + ret = test_userblock_alignment_helper2(fapl, TRUE); + CHECK(ret, FAIL, "test_userblock_alignment_helper2"); + + /* Release property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fapl); + CHECK(ret, FAIL, "H5Pclose"); + +} /* end test_userblock_alignment_paged() */ /**************************************************************** ** -** test_filespace_sects(): -** This test checks free space section info for -** files created with sec2 and split drivers. +** test_filespace_info(): +** Verify the following public routines retrieve and set file space +** information correctly: +** (1) H5Pget/set_file_space_strategy(): +** Retrieve and set file space strategy, persisting free-space, +** and free-space section threshold as specified +** (2) H5Pget/set_file_space_page_size(): +** Retrieve and set the page size for paged aggregation ** -*****************************************************************/ +****************************************************************/ static void -test_filespace_sects(void) +test_filespace_info(const char *env_h5_drvr) { - hid_t fapl_sec2; /* File access property id with sec2 driver */ - hid_t fapl_split; /* File access property id with split driver */ - hid_t fapl_core; /* File access property id with core driver */ - hid_t fapl_stdio; /* File access property id with stdio driver */ - hid_t fapl_family; /* File access property id with family driver */ - char filename[FILENAME_LEN]; /* Filename to use */ - herr_t ret; /* Return value */ + hid_t fid; /* File IDs */ + hid_t fapl, new_fapl; /* File access property lists */ + hid_t fcpl, fcpl1, fcpl2; /* File creation property lists */ + H5F_fspace_strategy_t strategy; /* File space strategy */ + hbool_t persist; /* Persist free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + unsigned new_format; /* New or old format */ + H5F_fspace_strategy_t fs_strategy; /* File space strategy--iteration variable */ + unsigned fs_persist; /* Persist free-space or not--iteration variable */ + hsize_t fs_threshold; /* Free-space section threshold--iteration variable */ + hsize_t fsp_size; /* File space page size */ + char filename[FILENAME_LEN]; /* Filename to use */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + herr_t ret; /* Return value */ - /* SEC2 */ - MESSAGE(5, ("Testing File free space information for a sec2 file\n")); + /* Output message about test being performed */ + MESSAGE(5, ("Testing file creation public routines: H5Pget/set_file_space_strategy & H5Pget/set_file_space_page_size\n")); - fapl_sec2 = H5Pcreate(H5P_FILE_ACCESS); + contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); - ret = H5Pset_fapl_sec2(fapl_sec2); - CHECK(ret, FAIL, "H5Pset_fapl_sec2"); + fapl = h5_fileaccess(); + h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[0], fapl_sec2, filename, sizeof(filename)); + /* Get a copy of the file access property list */ + new_fapl = H5Pcopy(fapl); + CHECK(new_fapl, FAIL, "H5Pcopy"); - /* perform free space information test for file with sec2 driver */ - test_free_sections(fapl_sec2, filename); + /* Set the "use the latest version of the format" bounds */ + ret = H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); - /* close fapl_sec2 and remove the file */ - h5_clean_files(FILENAME, fapl_sec2); + /* + * Case (1) + * Check file space information from a default file creation property list. + * Values expected: + * strategy--H5F_FILE_SPACE_AGGR + * persist--FALSE + * threshold--1 + * file space page size--4096 + */ + /* Create file creation property list template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* SPLIT */ - MESSAGE(5, ("Testing File free space information for a split file\n")); + /* Verify file space information */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); - fapl_split = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_split, FAIL, "h5_fileaccess"); + /* Retrieve file space page size */ + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE_DEF, "H5Pget_file_space_page_size"); - ret = H5Pset_fapl_split(fapl_split, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT); - CHECK(ret, FAIL, "H5Pset_fapl_split"); + /* Close property list */ + H5Pclose(fcpl); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[1], fapl_split, filename, sizeof(filename)); + /* + * Case (2) + * File space page size has a minimum size of 512. + * Setting value less than 512 will return an error; + * --setting file space page size to 0 + * --setting file space page size to 511 + */ + /* Create file creation property list template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); - /* perform free space information test for file with split driver */ - test_free_sections(fapl_split, filename); + /* Setting to 0: should fail */ + H5E_BEGIN_TRY { + ret = H5Pset_file_space_page_size(fcpl, 0); + } H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); - /* close fapl and remove the file */ - h5_clean_files(FILENAME, fapl_split); + /* Setting to 511: should fail */ + H5E_BEGIN_TRY { + ret = H5Pset_file_space_page_size(fcpl, 511); + } H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 512: should succeed */ + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); - /* STDIO */ - MESSAGE(5, ("Testing File free space information for a stdio file\n")); + /* Close property list */ + H5Pclose(fcpl); - fapl_stdio = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_stdio, FAIL, "h5_fileaccess"); + /* + * Case (3) + * Check file space information when creating a file with default properties. + * Values expected: + * strategy--H5F_FILE_SPACE_AGGR + * persist--FALSE + * threshold--1 + * file space page size--4096 + */ + /* Create a file with default file creation and access property lists */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fcreate"); - ret = H5Pset_fapl_stdio(fapl_stdio); - CHECK(ret, FAIL, "H5Pset_fapl_split"); + /* Get the file's creation property list */ + fcpl1 = H5Fget_create_plist(fid); + CHECK(fcpl1, FAIL, "H5Fget_create_plist"); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[2], fapl_stdio, filename, sizeof(filename)); + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl1, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* perform free space information test for file with stdio driver */ - test_free_sections(fapl_stdio, filename); + /* Verify file space information */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); - /* close fapl and remove the file */ - h5_clean_files(FILENAME, fapl_stdio); + /* Retrieve file space page size */ + ret = H5Pget_file_space_page_size(fcpl1, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE_DEF, "H5Pget_file_space_page_size"); + /* Close property lists */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + ret = H5Pclose(fcpl1); + CHECK(ret, FAIL, "H5Pclose"); - /* CORE */ - MESSAGE(5, ("Testing File free space information for a core file\n")); + /* + * Case (4) + * Check file space information when creating a file with the + * latest library format and default properties. + * Values expected: + * strategy--H5F_FILE_SPACE_AGGR + * persist--FALSE + * threshold--1 + * file space page size--4096 + */ + /* Create a file with the latest library format */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, new_fapl); + CHECK(fid, FAIL, "H5Fcreate"); - fapl_core = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_core, FAIL, "h5_fileaccess"); + /* Get the file's creation property */ + fcpl1 = H5Fget_create_plist(fid); + CHECK(fcpl1, FAIL, "H5Fget_create_plist"); - ret = H5Pset_fapl_core(fapl_core, (size_t)CORE_INCREMENT, TRUE); - CHECK(ret, FAIL, "H5Pset_fapl_core"); + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl1, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[3], fapl_core, filename, sizeof(filename)); + /* Verify file space information */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); - /* perform free space information test for file with core driver */ - test_free_sections(fapl_core, filename); + /* Retrieve file space page size */ + ret = H5Pget_file_space_page_size(fcpl1, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE_DEF, "H5Pget_file_space_page_size"); - /* close fapl_ and remove the file */ - h5_clean_files(FILENAME, fapl_core); + /* Close property lists */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + ret = H5Pclose(fcpl1); + CHECK(ret, FAIL, "H5Pclose"); + + /* + * Case (5) + * Check file space information with the following combinations: + * Create file with -- + * New or old format + * Persist or not persist free-space + * Different sizes for free-space section threshold (0 to 10) + * The four file space strategies: + * H5F_FSPACE_STRATEGY_FSM_AGGR, H5F_FSPACE_STRATEGY_PAGE, + * H5F_FSPACE_STRATEGY_AGGR, H5F_FSPACE_STRATEGY_NONE + * File space page size: set to 512 + * + */ + for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl; + + /* Set the FAPL for the type of format */ + if(new_format) { + MESSAGE(5, ("Testing with new group format\n")); + my_fapl = new_fapl; + } /* end if */ + else { + MESSAGE(5, ("Testing with old group format\n")); + my_fapl = fapl; + } /* end else */ + + /* Test with TRUE or FALSE for persisting free-space */ + for(fs_persist = FALSE; fs_persist <= TRUE; fs_persist++) { + + /* Test with free-space section threshold size: 0 to 10 */ + for(fs_threshold = 0; fs_threshold <= TEST_THRESHOLD10; fs_threshold++) { + + /* Test with 4 file space strategies */ + for(fs_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; fs_strategy < H5F_FSPACE_STRATEGY_NTYPES; H5_INC_ENUM(H5F_fspace_strategy_t, fs_strategy)) { + + if(!contig_addr_vfd && (fs_strategy == H5F_FSPACE_STRATEGY_PAGE || fs_persist)) + continue; + + /* Create file creation property list template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + + /* Set file space information */ + ret = H5Pset_file_space_strategy(fcpl, fs_strategy, (hbool_t)fs_persist, fs_threshold); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Verify file space information */ + VERIFY(strategy, fs_strategy, "H5Pget_file_space_strategy"); + + if(fs_strategy < H5F_FSPACE_STRATEGY_AGGR) { + VERIFY(persist, (hbool_t)fs_persist, "H5Pget_file_space_strategy"); + VERIFY(threshold, fs_threshold, "H5Pget_file_space_strategy"); + } else { + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + } + + /* Retrieve and verify file space page size */ + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + + /* Create the file with the specified file space info */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, my_fapl); + CHECK(fid, FAIL, "H5Fcreate"); + + /* Get the file's creation property */ + fcpl1 = H5Fget_create_plist(fid); + CHECK(fcpl1, FAIL, "H5Fget_create_plist"); + + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl1, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Verify file space information */ + VERIFY(strategy, fs_strategy, "H5Pget_file_space_strategy"); + + if(fs_strategy < H5F_FSPACE_STRATEGY_AGGR) { + VERIFY(persist, fs_persist, "H5Pget_file_space_strategy"); + VERIFY(threshold, fs_threshold, "H5Pget_file_space_strategy"); + } else { + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + } + + /* Retrieve and verify file space page size */ + ret = H5Pget_file_space_page_size(fcpl1, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Re-open the file */ + fid = H5Fopen(filename, H5F_ACC_RDWR, my_fapl); + CHECK(ret, FAIL, "H5Fopen"); + + /* Get the file's creation property */ + fcpl2 = H5Fget_create_plist(fid); + CHECK(fcpl2, FAIL, "H5Fget_create_plist"); + + /* Retrieve file space information */ + ret = H5Pget_file_space_strategy(fcpl2, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Verify file space information */ + VERIFY(strategy, fs_strategy, "H5Pget_file_space_strategy"); + if(fs_strategy < H5F_FSPACE_STRATEGY_AGGR) { + VERIFY(persist, fs_persist, "H5Pget_file_space_strategy"); + VERIFY(threshold, fs_threshold, "H5Pget_file_space_strategy"); + } else { + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + } + + /* Retrieve and verify file space page size */ + ret = H5Pget_file_space_page_size(fcpl2, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Release file creation property lists */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fcpl1); + CHECK(ret, FAIL, "H5Pclose"); + ret = H5Pclose(fcpl2); + CHECK(ret, FAIL, "H5Pclose"); + } /* end for file space strategy type */ + } /* end for free-space section threshold */ + } /* end for fs_persist */ + + /* close fapl_ and remove the file */ + h5_clean_files(FILESPACE_NAME, my_fapl); + } /* end for new_format */ + +} /* test_filespace_info() */ + +/**************************************************************** +** +** set_multi_split(): +** Internal routine to set up page-aligned address space for multi/split driver +** when testing paged aggregation. +** This is used by test_file_freespace() and test_sects_freespace(). +** +*****************************************************************/ +static int +set_multi_split(hid_t fapl, hsize_t pagesize, hbool_t multi, hbool_t split) +{ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl_arr[H5FD_MEM_NTYPES]; + char *memb_name[H5FD_MEM_NTYPES]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + hbool_t relax; + H5FD_mem_t mt; + + HDassert(split || multi); + HDmemset(memb_name, 0, sizeof memb_name); - /* FAMILY */ - MESSAGE(5, ("Testing File free space information for a family file\n")); + /* Get current split settings */ + if(H5Pget_fapl_multi(fapl, memb_map, memb_fapl_arr, memb_name, memb_addr, &relax) < 0) + TEST_ERROR - fapl_family = H5Pcreate(H5P_FILE_ACCESS); - CHECK(fapl_family, FAIL, "h5_fileaccess"); + if(split) { + /* Set memb_addr aligned */ + memb_addr[H5FD_MEM_SUPER] = ((memb_addr[H5FD_MEM_SUPER] + pagesize - 1) / pagesize) * pagesize; + memb_addr[H5FD_MEM_DRAW] = ((memb_addr[H5FD_MEM_DRAW] + pagesize - 1) / pagesize) * pagesize; + } else { + /* Set memb_addr aligned */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + memb_addr[mt] = ((memb_addr[mt] + pagesize - 1) / pagesize) * pagesize; + } /* end else */ - ret = H5Pset_fapl_family(fapl_family, (hsize_t)FAMILY_SIZE, H5P_DEFAULT); - CHECK(ret, FAIL, "H5Pset_fapl_family"); + /* Set multi driver with new FAPLs */ + if(H5Pset_fapl_multi(fapl, memb_map, memb_fapl_arr, (const char * const *)memb_name, memb_addr, relax) < 0) + TEST_ERROR - /* Set the filename to use for this test (dependent on fapl) */ - h5_fixname(FILENAME[4], fapl_family, filename, sizeof(filename)); + /* Free memb_name */ + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) + free(memb_name[mt]); - /* perform free space information test for file with family driver */ - test_free_sections(fapl_family, filename); + return 0; - /* close fapl and remove the file */ - h5_clean_files(FILENAME, fapl_family); +error: + return(-1); -} /* end test_filespace_sects() */ +} /* set_multi_split() */ /**************************************************************** ** -** test_filespace_info(): -** Verify that the public routines H5Pget/set_file_space() -** retrieve and set the file space strategy and free space -** section threshold as specified. +** test_file_freespace(): +** This routine checks the free space available in a file as +** returned by the public routine H5Fget_freespace(). ** -****************************************************************/ +** Modifications: +** Vailin Choi; July 2012 +** Remove datasets in reverse order so that all file spaces are shrunk. +** (A change due to H5FD_FLMAP_DICHOTOMY.) +** +** Vailin Choi; Dec 2012 +** Add changes due to paged aggregation via new format: +** the amount of freespace is different. +** +*****************************************************************/ static void -test_filespace_info(void) +test_file_freespace(const char *env_h5_drvr) { - hid_t fid1, fid2; /* HDF5 File IDs */ - hid_t fapl, new_fapl; /* File access property */ - hid_t fcpl, fcpl1, fcpl2; /* File creation property */ - char filename[FILENAME_LEN]; /* Filename to use */ - H5F_file_space_type_t strategy, fs_type, def_type; /* File space handling strategy */ - hsize_t threshold, fs_size, def_size; /* Free space section threshold */ - unsigned new_format; /* new format or old format */ - herr_t ret; /* return value */ + hid_t file; /* File opened with read-write permission */ + h5_stat_size_t empty_filesize; /* Size of file when empty */ + h5_stat_size_t mod_filesize; /* Size of file after being modified */ + hssize_t free_space; /* Amount of free space in file */ + hid_t fcpl; /* File creation property list */ + hid_t fapl, new_fapl; /* File access property list IDs */ + hid_t dspace; /* Dataspace ID */ + hid_t dset; /* Dataset ID */ + hid_t dcpl; /* Dataset creation property list */ + int k; /* Local index variable */ + unsigned u; /* Local index variable */ + char filename[FILENAME_LEN]; /* Filename to use */ + char name[32]; /* Dataset name */ + unsigned new_format; /* To use old or new format */ + hbool_t split_vfd, multi_vfd; /* Indicate multi/split driver */ + hsize_t expected_freespace; /* Freespace expected */ + hsize_t expected_fs_del; /* Freespace expected after delete */ + herr_t ret; /* Return value */ + + split_vfd = !HDstrcmp(env_h5_drvr, "split"); + multi_vfd = !HDstrcmp(env_h5_drvr, "multi"); + + if(!split_vfd && !multi_vfd) { + fapl = h5_fileaccess(); + h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); + + new_fapl = H5Pcopy(fapl); + CHECK(new_fapl, FAIL, "H5Pcopy"); + + /* Set the "use the latest version of the format" bounds */ + ret = H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); - /* Output message about test being performed */ - MESSAGE(5, ("Testing File Space Management public routines: H5Pget/set_file_space()\n")); + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + + /* Test with old & new format */ + for(new_format = FALSE; new_format <= TRUE; new_format++) { + hid_t my_fapl; + + /* Set the FAPL for the type of format */ + if(new_format) { + MESSAGE(5, ("Testing with new group format\n")); + + my_fapl = new_fapl; + + if(multi_vfd || split_vfd) { + ret = set_multi_split(new_fapl, FSP_SIZE_DEF, multi_vfd, split_vfd); + CHECK(ret, FAIL, "set_multi_split"); + } + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); + CHECK(ret, FAIL, "H5P_set_file_space_strategy"); + + expected_freespace = 4534; + if(split_vfd) expected_freespace = 427; + if(multi_vfd) expected_freespace = 248; + expected_fs_del = 0; + } /* end if */ + else { + MESSAGE(5, ("Testing with old group format\n")); + /* Default: non-paged aggregation, non-persistent free-space */ + my_fapl = fapl; + expected_freespace = 2464; + if(split_vfd) expected_freespace = 264; + if(multi_vfd) expected_freespace = 0; + expected_fs_del = 4096; + + } /* end else */ + + /* Create an "empty" file */ + file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, my_fapl); + CHECK(file, FAIL, "H5Fcreate"); + + ret = H5Fclose(file); + CHECK_I(ret, "H5Fclose"); + + /* Get the "empty" file size */ + empty_filesize = h5_get_file_size(filename, H5P_DEFAULT); + + /* Re-open the file (with read-write permission) */ + file = H5Fopen(filename, H5F_ACC_RDWR, my_fapl); + CHECK_I(file, "H5Fopen"); + + /* Check that the free space is 0 */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + VERIFY(free_space, 0, "H5Fget_freespace"); + + /* Create dataspace for datasets */ + dspace = H5Screate(H5S_SCALAR); + CHECK(dspace, FAIL, "H5Screate"); + + /* Create a dataset creation property list */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + CHECK(dcpl, FAIL, "H5Pcreate"); + + /* Set the space allocation time to early */ + ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); + CHECK(ret, FAIL, "H5Pset_alloc_time"); + + /* Create datasets in file */ + for(u = 0; u < 10; u++) { + sprintf(name, "Dataset %u", u); + dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset, FAIL, "H5Dcreate2"); + + ret = H5Dclose(dset); + CHECK(ret, FAIL, "H5Dclose"); + } /* end for */ - fapl = h5_fileaccess(); - h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); + /* Close dataspace */ + ret = H5Sclose(dspace); + CHECK(ret, FAIL, "H5Sclose"); - new_fapl = H5Pcopy(fapl); - CHECK(new_fapl, FAIL, "H5Pcopy"); + /* Close dataset creation property list */ + ret = H5Pclose(dcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Set the "use the latest version of the format" bounds */ - ret = H5Pset_libver_bounds(new_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); - CHECK(ret, FAIL, "H5Pset_libver_bounds"); + /* Check that there is the right amount of free space in the file */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + VERIFY(free_space, expected_freespace, "H5Fget_freespace"); - /* Create file-creation template */ - fcpl = H5Pcreate(H5P_FILE_CREATE); - CHECK(fcpl, FAIL, "H5Pcreate"); + /* Delete datasets in file */ + for(k = 9; k >= 0; k--) { + sprintf(name, "Dataset %u", (unsigned)k); + ret = H5Ldelete(file, name, H5P_DEFAULT); + CHECK(ret, FAIL, "H5Ldelete"); + } /* end for */ - /* Get default file space information */ - ret = H5Pget_file_space(fcpl, &def_type, &def_size); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Check that there is the right amount of free space in the file */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + if(new_format) + VERIFY(free_space, expected_fs_del, "H5Fget_freespace"); + else + VERIFY(free_space, expected_fs_del, "H5Fget_freespace"); - /* Test with old & new format groups */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { - hid_t my_fapl; + /* Close file */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); + + /* Get the file size after modifications*/ + mod_filesize = h5_get_file_size(filename, H5P_DEFAULT); + + /* Check that the file reverted to empty size */ + VERIFY(mod_filesize, empty_filesize, "H5Fget_freespace"); + + h5_clean_files(FILESPACE_NAME, my_fapl); + + } /* end for */ + } + +} /* end test_file_freespace() */ + +/**************************************************************** +** +** test_sects_freespace(): +** This routine checks free-space section information for the +** file as returned by the public routine H5Fget_free_sections(). +** +*****************************************************************/ +static void +test_sects_freespace(const char *env_h5_drvr, hbool_t new_format) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file; /* File ID */ + hid_t fcpl; /* File creation property list template */ + hid_t fapl; /* File access property list template */ + hssize_t free_space; /* Amount of free-space in the file */ + hid_t dspace; /* Dataspace ID */ + hid_t dset; /* Dataset ID */ + hid_t dcpl; /* Dataset creation property list */ + char name[32]; /* Dataset name */ + hssize_t nsects = 0; /* # of free-space sections */ + hssize_t nall; /* # of free-space sections for all types of data */ + hssize_t nmeta = 0, nraw = 0; /* # of free-space sections for meta/raw/generic data */ + H5F_sect_info_t sect_info[15]; /* Array to hold free-space information */ + H5F_sect_info_t all_sect_info[15]; /* Array to hold free-space information for all types of data */ + H5F_sect_info_t meta_sect_info[15]; /* Array to hold free-space information for metadata */ + H5F_sect_info_t raw_sect_info[15]; /* Array to hold free-space information for raw data */ + hsize_t total = 0; /* sum of the free-space section sizes */ + hsize_t tmp_tot = 0; /* Sum of the free-space section sizes */ + hsize_t last_size; /* Size of last free-space section */ + hsize_t dims[1]; /* Dimension sizes */ + unsigned u; /* Local index variable */ + H5FD_mem_t type; + hbool_t split_vfd = FALSE, multi_vfd = FALSE; + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("Testing H5Fget_free_sections()--free-space section info in the file\n")); + + split_vfd = !HDstrcmp(env_h5_drvr, "split"); + multi_vfd = !HDstrcmp(env_h5_drvr, "multi"); + + if(!split_vfd && !multi_vfd) { + + fapl = h5_fileaccess(); + h5_fixname(FILESPACE_NAME[0], fapl, filename, sizeof filename); + + /* Create file-creation template */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); - /* Set the FAPL for the type of format */ if(new_format) { - MESSAGE(5, ("Testing with new group format\n")); - my_fapl = new_fapl; - } /* end if */ - else { - MESSAGE(5, ("Testing with old group format\n")); - my_fapl = fapl; - } /* end else */ + ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + CHECK(ret, FAIL, "H5Pset_libver_bounds"); + + /* Set to paged aggregation and persistent free-space */ + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + /* Set up paged aligned address space for multi/split driver */ + if(multi_vfd || split_vfd) { + ret = set_multi_split(fapl, FSP_SIZE_DEF, multi_vfd, split_vfd); + CHECK(ret, FAIL, "set_multi_split"); + } + + } else { + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + } + + /* Create the file */ + file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + CHECK(file, FAIL, "H5Fcreate"); + + /* Create a dataset creation property list */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + CHECK(dcpl, FAIL, "H5Pcreate"); + + /* Set the space allocation time to early */ + ret = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); + CHECK(ret, FAIL, "H5Pset_alloc_time"); + + /* Create 1 large dataset */ + dims[0] = 1200; + dspace = H5Screate_simple(1, dims, NULL); + dset = H5Dcreate2(file, "Dataset_large", H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset, FAIL, "H5Dcreate2"); - /* Test with different sized free space section threshold */ - for(fs_size = 0; fs_size <= TEST_THRESHOLD10; fs_size++) { + /* Close dataset */ + ret = H5Dclose(dset); + CHECK(ret, FAIL, "H5Dclose"); - /* Test with different file space handling strategies */ - for(fs_type = H5F_FILE_SPACE_DEFAULT; fs_type < H5F_FILE_SPACE_NTYPES; H5_INC_ENUM(H5F_file_space_type_t, fs_type)) { + /* Close dataspace */ + ret = H5Sclose(dspace); + CHECK(ret, FAIL, "H5Sclose"); - /* Get a copy of the default file creation property */ - fcpl1 = H5Pcopy(fcpl); - CHECK(fcpl1, FAIL, "H5Pcopy"); + /* Create dataspace for datasets */ + dspace = H5Screate(H5S_SCALAR); + CHECK(dspace, FAIL, "H5Screate"); - /* Set file space strategy and free space section threshold */ - ret = H5Pset_file_space(fcpl1, fs_type, fs_size); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Create datasets in file */ + for(u = 0; u < 10; u++) { + sprintf(name, "Dataset %u", u); + dset = H5Dcreate2(file, name, H5T_STD_U32LE, dspace, H5P_DEFAULT, dcpl, H5P_DEFAULT); + CHECK(dset, FAIL, "H5Dcreate2"); - /* Get the file space info from the creation property */ - ret = H5Pget_file_space(fcpl1, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); + ret = H5Dclose(dset); + CHECK(ret, FAIL, "H5Dclose"); + } /* end for */ - /* A 0 value for strategy retains existing strategy in use */ - VERIFY(strategy, (H5F_file_space_type_t)(fs_type ? fs_type : def_type), "H5Pget_file_space"); - /* A 0 value for threshold retains existing threshold in use */ - VERIFY(threshold, (hsize_t)(fs_size ? fs_size : def_size), "H5Pget_file_space"); + /* Close dataspace */ + ret = H5Sclose(dspace); + CHECK(ret, FAIL, "H5Sclose"); - /* Create the file with the specified file space info */ - fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl1, my_fapl); - CHECK(fid1, FAIL, "H5Fcreate"); + /* Close dataset creation property list */ + ret = H5Pclose(dcpl); + CHECK(ret, FAIL, "H5Pclose"); - /* Close the file */ - ret = H5Fclose(fid1); - CHECK(ret, FAIL, "H5Fclose"); + /* Delete odd-numbered datasets in file */ + for(u = 0; u < 10; u++) { + sprintf(name, "Dataset %u", u); + if(u % 2) { + ret = H5Ldelete(file, name, H5P_DEFAULT); + CHECK(ret, FAIL, "H5Ldelete"); + } /* end if */ + } /* end for */ - /* Re-open the file */ - fid2 = H5Fopen(filename, H5F_ACC_RDWR, my_fapl); - CHECK(fid2, FAIL, "H5Fopen"); + /* Close file */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); - /* Get the file's creation property */ - fcpl2 = H5Fget_create_plist(fid2); - CHECK(fcpl2, FAIL, "H5Fget_create_plist"); + /* Re-open the file with read-only permission */ + file = H5Fopen(filename, H5F_ACC_RDONLY, fapl); + CHECK_I(file, "H5Fopen"); + + /* Get the amount of free space in the file */ + free_space = H5Fget_freespace(file); + CHECK(free_space, FAIL, "H5Fget_freespace"); + + /* Get the total # of free-space sections in the file */ + nall = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, NULL); + CHECK(nall, FAIL, "H5Fget_free_sections"); + + /* Should return failure when nsects is 0 with a nonnull sect_info */ + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)0, all_sect_info); + VERIFY(nsects, FAIL, "H5Fget_free_sections"); + + /* Retrieve and verify free space info for all the sections */ + HDmemset(all_sect_info, 0, sizeof(all_sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)nall, all_sect_info); + VERIFY(nsects, nall, "H5Fget_free_sections"); + + /* Verify the amount of free-space is correct */ + for(u = 0; u < nall; u++) + total += all_sect_info[u].size; + VERIFY(free_space, total, "H5Fget_free_sections"); + + /* Save the last section's size */ + last_size = all_sect_info[nall-1].size; + + /* Retrieve and verify free space info for -1 sections */ + HDmemset(sect_info, 0, sizeof(sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(nall - 1), sect_info); + VERIFY(nsects, nall, "H5Fget_free_sections"); + + /* Verify the amount of free-space is correct */ + total = 0; + for(u = 0; u < (nall - 1); u++) { + VERIFY(sect_info[u].addr, all_sect_info[u].addr, "H5Fget_free_sections"); + VERIFY(sect_info[u].size, all_sect_info[u].size, "H5Fget_free_sections"); + total += sect_info[u].size; + } + VERIFY(((hsize_t)free_space - last_size), total, "H5Fget_free_sections"); + + /* Retrieve and verify free-space info for +1 sections */ + HDmemset(sect_info, 0, sizeof(sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DEFAULT, (size_t)(nall + 1), sect_info); + VERIFY(nsects, nall, "H5Fget_free_sections"); + + /* Verify amount of free-space is correct */ + total = 0; + for(u = 0; u < nall; u++) { + VERIFY(sect_info[u].addr, all_sect_info[u].addr, "H5Fget_free_sections"); + VERIFY(sect_info[u].size, all_sect_info[u].size, "H5Fget_free_sections"); + total += sect_info[u].size; + } + VERIFY(sect_info[nall].addr, 0, "H5Fget_free_sections"); + VERIFY(sect_info[nall].size, 0, "H5Fget_free_sections"); + VERIFY(free_space, total, "H5Fget_free_sections"); + + HDmemset(meta_sect_info, 0, sizeof(meta_sect_info)); + if(multi_vfd) { + hssize_t ntmp; + + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + continue; + /* Get the # of free-space sections in the file for metadata */ + ntmp = H5Fget_free_sections(file, type, (size_t)0, NULL); + CHECK(ntmp, FAIL, "H5Fget_free_sections"); + + if(ntmp > 0) { + nsects = H5Fget_free_sections(file, type, (size_t)ntmp, &meta_sect_info[nmeta]); + VERIFY(nsects, ntmp, "H5Fget_free_sections"); + nmeta += ntmp; + } + } + } else { + /* Get the # of free-space sections in the file for metadata */ + nmeta = H5Fget_free_sections(file, H5FD_MEM_SUPER, (size_t)0, NULL); + CHECK(nmeta, FAIL, "H5Fget_free_sections"); + + /* Retrieve and verify free-space sections for metadata */ + nsects = H5Fget_free_sections(file, H5FD_MEM_SUPER, (size_t)nmeta, meta_sect_info); + VERIFY(nsects, nmeta, "H5Fget_free_sections"); + } - strategy = H5F_FILE_SPACE_DEFAULT; - threshold = 0; + /* Get the # of free-space sections in the file for raw data */ + nraw = H5Fget_free_sections(file, H5FD_MEM_DRAW, (size_t)0, NULL); + CHECK(nraw, FAIL, "H5Fget_free_sections"); - /* Get the file space info from the creation property list */ - ret = H5Pget_file_space(fcpl2, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Retrieve and verify free-space sections for raw data */ + HDmemset(raw_sect_info, 0, sizeof(raw_sect_info)); + nsects = H5Fget_free_sections(file, H5FD_MEM_DRAW, (size_t)nraw, raw_sect_info); + VERIFY(nsects, nraw, "H5Fget_free_sections"); - VERIFY(strategy, (H5F_file_space_type_t)(fs_type ? fs_type : def_type), "H5Pget_file_space"); - VERIFY(threshold, (hsize_t)(fs_size ? fs_size : def_size), "H5Pget_file_space"); + /* Sum all the free-space sections */ + for(u = 0; u < nmeta; u++) + tmp_tot += meta_sect_info[u].size; - /* Close the file */ - ret = H5Fclose(fid2); - CHECK(ret, FAIL, "H5Fclose"); + for(u = 0; u < nraw; u++) + tmp_tot += raw_sect_info[u].size; - /* Release file-creation template */ - ret = H5Pclose(fcpl1); - CHECK(ret, FAIL, "H5Pclose"); - ret = H5Pclose(fcpl2); - CHECK(ret, FAIL, "H5Pclose"); - } /* end for file space strategy type */ - } /* end for free space threshold */ + /* Verify free-space info */ + VERIFY(nmeta+nraw, nall, "H5Fget_free_sections"); + VERIFY(tmp_tot, total, "H5Fget_free_sections"); - h5_clean_files(FILESPACE_NAME, my_fapl); + /* Closing */ + ret = H5Fclose(file); + CHECK(ret, FAIL, "H5Fclose"); + ret = H5Pclose(fcpl); + CHECK(fcpl, FAIL, "H5Pclose"); - } /* end for new/old format */ + h5_clean_files(FILESPACE_NAME, fapl); + } + +} /* end test_sects_freespace() */ - /* Close the file creation property list */ - ret = H5Pclose(fcpl); - CHECK(ret, FAIL, "H5Pclose"); -} /* test_filespace_info() */ /**************************************************************** ** ** test_filespace_compatible(): -** Verify that the branch with file space management enhancement +** Verify that the trunk with the latest file space management ** can open, read and modify 1.6 HDF5 file and 1.8 HDF5 file. -** Also verify the correct file space strategy/threshold in use +** Also verify the correct file space handling information ** and the amount of free space. ** ****************************************************************/ @@ -3614,8 +4330,9 @@ test_filespace_compatible(void) ssize_t nread; /* Number of bytes read in */ unsigned i, j; /* Local index variable */ hssize_t free_space; /* Amount of free-space in the file */ + hbool_t persist; /* Persist free-space or not */ hsize_t threshold; /* Free-space section threshold */ - H5F_file_space_type_t strategy; /* File space handling strategy */ + H5F_fspace_strategy_t strategy; /* File space handling strategy */ herr_t ret; /* Return value */ /* Output message about test being performed */ @@ -3655,14 +4372,16 @@ test_filespace_compatible(void) fcpl = H5Fget_create_plist(fid); CHECK(fcpl, FAIL, "H5Fget_create_plist"); - /* Retrieve the file space handling strategy and threshold */ - ret = H5Pget_file_space(fcpl, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); + /* Retrieve the file space info */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); - /* File space handling strategy should be H5F_FILE_SPACE_ALL = 2 */ - /* Free space section threshold should be 1 */ - VERIFY(strategy, 2, "H5Pget_file_space"); - VERIFY(threshold, 1, "H5Pget_file_space"); + /* File space handling strategy should be H5F_FSPACE_STRATEGY_FSM_AGGR */ + /* Persisting free-space should be FALSE */ + /* Free-space section threshold should be 1 */ + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); /* Generate raw data */ for(i = 0; i < 100; i++) @@ -3715,6 +4434,387 @@ test_filespace_compatible(void) /**************************************************************** ** +** test_filespace_1.10.0_compatible(): +** Verify that the latest file space management can open, read and +** modify 1.10.0 HDF5 files : +** h5fc_ext1_i.h5: H5F_FILE_SPACE_ALL, default threshold; has superblock extension but no fsinfo message +** h5fc_ext1_f.h5: H5F_FILE_SPACE_ALL_PERSIST, default threshold; has superblock extension with fsinfo message +** h5fc_ext2_if.h5: H5F_FILE_SPACE_ALL, non-default threshold; has superblock extension with fsinfo message +** h5fc_ext2_sf.h5: H5F_FILE_SPACE_VFD, default threshold; has superblock extension with fsinfo message +** h5fc_ext3_isf.h5: H5F_FILE_SPACE_AGGR_VFD, default threshold; has superblock extension with fsinfo message +** h5fc_ext_none.h5: H5F_FILE_SPACE_ALL, default threshold; without superblock extension +** The above files are copied from release 1.10.0 tools/h5format_convert/testfiles. +** +****************************************************************/ +static void +test_filespace_1_10_0_compatible(void) +{ + hid_t fid = -1; /* File id */ + hid_t did = -1; /* Dataset id */ + hid_t fcpl; /* File creation property list */ + hbool_t persist; /* Persist free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + H5F_fspace_strategy_t strategy; /* File space handling strategy */ + int wbuf[24]; /* Buffer for dataset data */ + int rdbuf[24]; /* Buffer for dataset data */ + int status; /* Status from copying the existing file */ + unsigned i, j; /* Local index variable */ + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("File space compatibility testing for 1.10.0 files\n")); + + for(j = 0; j < NELMTS(OLD_1_10_0_FILENAME); j++) { + const char *filename = H5_get_srcdir_filename(OLD_1_10_0_FILENAME[j]); /* Corrected test file name */ + + /* Make a copy of the test file */ + status = h5_make_local_copy(filename, FILE5); + CHECK(status, FAIL, "h5_make_local_copy"); + + /* Open the temporary test file */ + fid = H5Fopen(FILE5, H5F_ACC_RDWR, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + /* Get the file's file creation property list */ + fcpl = H5Fget_create_plist(fid); + CHECK(fcpl, FAIL, "H5Fget_create_plist"); + + /* Retrieve the file space info */ + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + + switch(j) { + case 0: + VERIFY(strategy, H5F_FILE_SPACE_STRATEGY_DEF, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/DSET_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 1: + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, TRUE, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/DSET_NDATA_BT2", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 2: + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, 2, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/DSET_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 3: + VERIFY(strategy, H5F_FSPACE_STRATEGY_NONE, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 4: + VERIFY(strategy, H5F_FSPACE_STRATEGY_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_FA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + case 5: + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, H5F_FREE_SPACE_PERSIST_DEF, "H5Pget_file_space_strategy"); + VERIFY(threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space_strategy"); + + /* Open the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + for(i = 0; i < 24; i++) + wbuf[i] = (int)j+1; + + /* Write to the dataset */ + ret = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + default: + break; + } + + /* Close the plist */ + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + /* Re-Open the file */ + fid = H5Fopen(FILE5, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + switch(j) { + case 0: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/DSET_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 1: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/DSET_NDATA_BT2", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 2: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/DSET_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 3: + /* Open and read the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_EA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 4: + + /* Open and read the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_FA", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + case 5: + + /* Open and read the dataset */ + did = H5Dopen2(fid, "/GROUP/DSET_NDATA_NONE", H5P_DEFAULT); + CHECK(did, FAIL, "H5Dopen"); + + ret = H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Verify the data read is correct */ + for(i = 0; i < 24; i++) + VERIFY(rdbuf[i], j+1, "test_compatible"); + + /* Close the dataset */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + break; + + default: + break; + } + + /* Close the file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + } /* end for */ + +} /* test_filespace_1_10_0_compatible */ + +/**************************************************************** +** +** test_filespace_round_compatible(): +** Verify that the trunk can open, read and modify these files-- +** 1) They are initially created (via gen_filespace.c) in the trunk +** with combinations of file space strategies, default/non-default +** threshold, and file spacing paging enabled/disbled. +** The library creates the file space info message with +** "mark if unknown" in these files. +** 2) They are copied to the 1.8 branch, and are opened/read/modified +** there via test_filespace_compatible() in test/tfile.c. +** The 1.8 library marks the file space info message as "unknown" +** in these files. +** 3) They are then copied back from the 1.8 branch to the trunk for +** compatibility testing via this routine. +** 4) Upon encountering the file space info message which is marked +** as "unknown", the library will use the default file space management +** from then on: non-persistent free-space managers, default threshold, +** and non-paging file space. +** +****************************************************************/ +static void +test_filespace_round_compatible(void) +{ + hid_t fid = -1; /* File id */ + hid_t fcpl = -1; /* File creation property list ID */ + unsigned j; /* Local index variable */ + H5F_fspace_strategy_t strategy; /* File space strategy */ + hbool_t persist; /* Persist free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + hssize_t free_space; /* Amount of free space in the file */ + int status; /* Status from copying the existing file */ + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("File space compatibility testing for files from trunk to 1_8 to trunk\n")); + + for(j = 0; j < NELMTS(FSPACE_FILENAMES); j++) { + const char *filename = H5_get_srcdir_filename(FSPACE_FILENAMES[j]); + + /* Make a copy of the test file */ + status = h5_make_local_copy(filename, FILE5); + CHECK(status, FAIL, "h5_make_local_copy"); + + /* Open the temporary test file */ + fid = H5Fopen(FILE5, H5F_ACC_RDWR, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + /* Get the file's creation property list */ + fcpl = H5Fget_create_plist(fid); + CHECK(fcpl, FAIL, "H5Fget_create_plist"); + + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + VERIFY(strategy, H5F_FSPACE_STRATEGY_FSM_AGGR, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); + + /* There should not be any free space in the file */ + free_space = H5Fget_freespace(fid); + CHECK(free_space, FAIL, "H5Fget_freespace"); + VERIFY(free_space, (hssize_t)0, "H5Fget_freespace"); + + /* Closing */ + ret = H5Fclose(fid); + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Fclose"); + } /* end for */ + +} /* test_filespace_round_compatible */ + + +/**************************************************************** +** ** test_libver_bounds_real(): ** Verify that a file created and modified with the ** specified libver bounds has the specified object header @@ -3951,6 +5051,9 @@ test_deprec(void) { hid_t file; /* File IDs for old & new files */ hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File creation property list */ + hid_t new_fapl; + hsize_t align; unsigned super; /* Superblock version # */ unsigned freelist; /* Free list version # */ unsigned stab; /* Symbol table entry version # */ @@ -4003,17 +5106,25 @@ test_deprec(void) CHECK(fcpl, FAIL, "H5Pcreate"); /* Set a property in the FCPL that will push the superblock version up */ - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_VFD, (hsize_t)0); - CHECK(ret, FAIL, "H5Pset_file_space"); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0); + ret = H5Pset_file_space_page_size(fcpl, (hsize_t)512); + CHECK(ret, FAIL, "H5Pset_file_space_strategy"); + + fapl = H5Pcreate(H5P_FILE_ACCESS); + ret = H5Pset_alignment(fapl, (hsize_t)1, (hsize_t)1024); + CHECK(ret, FAIL, "H5Pset_alignment"); /* Creating a file with the non-default file creation property list should * create a version 2 superblock */ /* Create file with custom file creation property list */ - file= H5Fcreate(FILE1, H5F_ACC_TRUNC , fcpl, H5P_DEFAULT); + file= H5Fcreate(FILE1, H5F_ACC_TRUNC , fcpl, fapl); CHECK(file, FAIL, "H5Fcreate"); + new_fapl = H5Fget_access_plist(file); + H5Pget_alignment(new_fapl, NULL, &align); + /* Close FCPL */ ret=H5Pclose(fcpl); CHECK(ret, FAIL, "H5Pclose"); @@ -4021,7 +5132,7 @@ test_deprec(void) /* Get the file's version information */ ret = H5Fget_info1(file, &finfo); CHECK(ret, FAIL, "H5Fget_info1"); - VERIFY(finfo.super_ext_size, 40,"H5Fget_info1"); + VERIFY(finfo.super_ext_size, 152,"H5Fget_info1"); VERIFY(finfo.sohm.hdr_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.index_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.heap_size, 0,"H5Fget_info1"); @@ -4053,7 +5164,7 @@ test_deprec(void) /* Get the file's version information */ ret = H5Fget_info1(file, &finfo); CHECK(ret, FAIL, "H5Fget_info1"); - VERIFY(finfo.super_ext_size, 40,"H5Fget_info1"); + VERIFY(finfo.super_ext_size, 152,"H5Fget_info1"); VERIFY(finfo.sohm.hdr_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.index_size, 0,"H5Fget_info1"); VERIFY(finfo.sohm.msgs_info.heap_size, 0,"H5Fget_info1"); @@ -4077,6 +5188,81 @@ test_deprec(void) /* Close file */ ret=H5Fclose(file); CHECK(ret, FAIL, "H5Fclose"); + + { /* Test deprecated H5Pget/set_file_space() */ + + H5F_file_space_type_t old_strategy; + hsize_t old_threshold; + hid_t fid; + hid_t ffcpl; + + fcpl = H5Pcreate(H5P_FILE_CREATE); + CHECK(fcpl, FAIL, "H5Pcreate"); + + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL, "H5Pget_file_space"); + VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); + + /* Set file space strategy and free space section threshold */ + ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); + CHECK(ret, FAIL, "H5Pget_file_space"); + + /* Get the file space info from the creation property */ + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, H5F_FREE_SPACE_THRESHOLD_DEF, "H5Pget_file_space"); + + ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_DEFAULT, (hsize_t)3); + CHECK(ret, FAIL, "H5Pget_file_space"); + + ret = H5Pget_file_space(fcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); + + /* Create a file */ + fid = H5Fcreate(FILE1, H5F_ACC_TRUNC , fcpl, H5P_DEFAULT); + CHECK(file, FAIL, "H5Fcreate"); + + old_strategy = H5F_FILE_SPACE_DEFAULT; + old_threshold = 0; + ffcpl = H5Fget_create_plist(fid); + ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); + + /* Close file */ + ret=H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + + ret = H5Pclose(ffcpl); + CHECK(ret, FAIL, "H5Pclose"); + + ret = H5Pclose(fcpl); + CHECK(ret, FAIL, "H5Pclose"); + + /* Reopen the file */ + fid = H5Fopen(FILE1, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fcreate"); + + old_strategy = H5F_FILE_SPACE_DEFAULT; + old_threshold = 0; + ffcpl = H5Fget_create_plist(fid); + ret = H5Pget_file_space(ffcpl, &old_strategy, &old_threshold); + CHECK(ret, FAIL, "H5Pget_file_space"); + VERIFY(old_strategy, H5F_FILE_SPACE_ALL_PERSIST, "H5Pget_file_space"); + VERIFY(old_threshold, 3, "H5Pget_file_space"); + + ret = H5Pclose(ffcpl); + CHECK(ret, FAIL, "H5Pclose"); + + ret=H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + } + } /* test_deprec */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ @@ -4088,9 +5274,16 @@ test_deprec(void) void test_file(void) { + const char *env_h5_drvr; /* File Driver value from environment */ + /* Output message about test being performed */ MESSAGE(5, ("Testing Low-Level File I/O\n")); + /* Get the VFD to use */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + if(env_h5_drvr == NULL) + env_h5_drvr = "nomatch"; + test_file_create(); /* Test file creation(also creation templates)*/ test_file_open(); /* Test file opening */ test_file_reopen(); /* Test file reopening */ @@ -4099,7 +5292,6 @@ test_file(void) test_get_obj_ids(); /* Test H5Fget_obj_ids for Jira Issue 8528 */ test_file_perm(); /* Test file access permissions */ test_file_perm2(); /* Test file access permission again */ - test_file_freespace(); /* Test file free space information */ test_file_ishdf5(); /* Test detecting HDF5 files correctly */ test_file_open_dot(); /* Test opening objects with "." for a name */ test_file_open_overlap(); /* Test opening files in an overlapping manner */ @@ -4114,9 +5306,21 @@ test_file(void) test_cached_stab_info(); /* Tests that files are created with cached stab info in the superblock */ test_rw_noupdate(); /* Test to ensure that RW permissions don't write the file unless dirtied */ test_userblock_alignment(); /* Tests that files created with a userblock and alignment interact properly */ - test_filespace_sects(); /* Test file free space section information */ - test_filespace_info(); /* Test file creation public routines:H5Pget/set_file_space */ + test_userblock_alignment_paged(); /* Tests files created with a userblock and alignment (via paged aggregation) interact properly */ + test_filespace_info(env_h5_drvr); /* Test file creation public routines: */ + /* H5Pget/set_file_space_strategy() & H5Pget/set_file_space_page_size() */ + /* Skipped testing for multi/split drivers */ + test_file_freespace(env_h5_drvr); /* Test file public routine H5Fget_freespace() */ + /* Skipped testing for multi/split drivers */ + /* Setup for multi/split drivers are there already */ + test_sects_freespace(env_h5_drvr, TRUE); /* Test file public routine H5Fget_free_sections() for new format */ + /* Skipped testing for multi/split drivers */ + /* Setup for multi/split drivers are there already */ + test_sects_freespace(env_h5_drvr, FALSE); /* Test file public routine H5Fget_free_sections() */ + /* Skipped testing for multi/split drivers */ test_filespace_compatible(); /* Test compatibility for file space management */ + test_filespace_round_compatible(); /* Testing file space compatibility for files from trunk to 1_8 to trunk */ + test_filespace_1_10_0_compatible(); /* Testing file space compatibility for files from release 1.10.0 */ test_libver_bounds(); /* Test compatibility for file space management */ test_libver_macros(); /* Test the macros for library version comparison */ test_libver_macros2(); /* Test the macros for library version comparison */ diff --git a/test/tmisc.c b/test/tmisc.c index f93500d..6012588 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -1819,8 +1819,9 @@ test_misc11(void) unsigned sym_lk; /* Symbol table B-tree leaf 'K' value */ unsigned nindexes; /* Shared message number of indexes */ H5F_info2_t finfo; /* global information about file */ - H5F_file_space_type_t strategy; /* File/free space strategy */ + H5F_fspace_strategy_t strategy; /* File space strategy */ hsize_t threshold; /* Free-space section threshold */ + hbool_t persist; /* To persist free-space or not */ herr_t ret; /* Generic return value */ /* Output message about test being performed */ @@ -1878,11 +1879,11 @@ test_misc11(void) ret=H5Pset_shared_mesg_nindexes(fcpl,MISC11_NINDEXES); CHECK(ret, FAIL, "H5Pset_shared_mesg"); - ret = H5Pset_file_space(fcpl, H5F_FILE_SPACE_VFD, (hsize_t)0); + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); CHECK(ret, FAIL, "H5Pset_file_space"); /* Creating a file with the non-default file creation property list should - * create a version 1 superblock + * create a version 2 superblock */ /* Create file with custom file creation property list */ @@ -1942,10 +1943,11 @@ test_misc11(void) CHECK(ret, FAIL, "H5Pget_shared_mesg_nindexes"); VERIFY(nindexes, MISC11_NINDEXES, "H5Pget_shared_mesg_nindexes"); - ret = H5Pget_file_space(fcpl, &strategy, &threshold); - CHECK(ret, FAIL, "H5Pget_file_space"); - VERIFY(strategy, 4, "H5Pget_file_space"); - VERIFY(threshold, 1, "H5Pget_file_space"); + ret = H5Pget_file_space_strategy(fcpl, &strategy, &persist, &threshold); + CHECK(ret, FAIL, "H5Pget_file_space_strategy"); + VERIFY(strategy, 3, "H5Pget_file_space_strategy"); + VERIFY(persist, FALSE, "H5Pget_file_space_strategy"); + VERIFY(threshold, 1, "H5Pget_file_space_strategy"); /* Close file */ ret=H5Fclose(file); diff --git a/testpar/Makefile.am b/testpar/Makefile.am index 4fe0ba8..3889630 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -25,7 +25,7 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test # Test programs. These are our main targets. # -TEST_PROG_PARA=t_mpi testphdf5 t_cache t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame +TEST_PROG_PARA=t_mpi testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame check_PROGRAMS = $(TEST_PROG_PARA) diff --git a/testpar/t_cache_image.c b/testpar/t_cache_image.c new file mode 100644 index 0000000..7283fa7 --- /dev/null +++ b/testpar/t_cache_image.c @@ -0,0 +1,2161 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: John Mainzer + * 7/13/15 + * + * This file contains tests specific to the cache image + * feature implemented in H5C.c + */ +#include "h5test.h" +#include "testphdf5.h" +#include "testpar.h" +#include "cache_common.h" +#include "genall5.h" + +#define CHUNK_SIZE 10 +#define DSET_SIZE (40 * CHUNK_SIZE) +#define MAX_NUM_DSETS 256 + + +/* global variable declarations: */ + + +const char *FILENAMES[] = { + "t_cache_image_00", + "t_cache_image_01", + NULL +}; + +/* local utility function declarations */ + +static void create_data_sets(hid_t file_id, int min_dset, int max_dset); +static void delete_data_sets(hid_t file_id, int min_dset, int max_dset); + +static void open_hdf5_file(const hbool_t create_file, + const hbool_t mdci_sbem_expected, const hbool_t read_only, + const hbool_t set_mdci_fapl, const hbool_t config_fsm, + const char * hdf_file_name, const unsigned cache_image_flags, + hid_t * file_id_ptr, H5F_t ** file_ptr_ptr, H5C_t ** cache_ptr_ptr, + MPI_Comm comm, MPI_Info info, int l_facc_type, + const hbool_t all_coll_metadata_ops, const hbool_t coll_metadata_write, + const int md_write_strat); + +static void verify_data_sets(hid_t file_id, int min_dset, int max_dset); + +/* local test function declarations */ + +static hbool_t parse_flags(int argc, char * argv[], hbool_t * setup_ptr, + hbool_t display); +static void usage(void); +static unsigned construct_test_file(int test_file_index); + + +/* top level test function declarations */ +static unsigned verify_cache_image_RW(int file_name_id, + int md_write_strat, int mpi_rank); + + +/****************************************************************************/ +/***************************** Utility Functions ****************************/ +/****************************************************************************/ + +/*------------------------------------------------------------------------- + * Function: construct_test_file() + * + * Purpose: This function attempts to mimic the typical "poor man's + * parallel use case in which the file is passed between + * processes, each of which open the file, write some data, + * close the file, and then pass control on to the next + * process. + * + * In this case, we create one group for each process, and + * populate it with a "zoo" of HDF5 objects selected to + * (ideally) exercise all HDF5 on disk data structures. + * + * The end result is a test file used verify that PHDF5 + * can open a file with a cache image. + * + * Cycle of operation + * + * 1) Create a HDF5 file with the cache image FAPL entry. + * + * Verify that the cache is informed of the cache image + * FAPL entry. + * + * Set all cache image flags, forcing full functionality. + * + * 2) Create a data set in the file. + * + * 3) Close the file. + * + * 4) Open the file. + * + * Verify that the metadata cache is instructed to load + * the metadata cache image. + * + * 5) Create a data set in the file. + * + * 6) Close the file. If enough datasets have been created + * goto 7. Otherwise return to 4. + * + * 7) Open the file R/O. + * + * Verify that the file contains a metadata cache image + * superblock extension message. + * + * 8) Verify all data sets. + * + * Verify that the cache image has been loaded. + * + * 9) close the file. + * + * Return: void + * + * Programmer: John Mainzer + * 1/25/17 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +construct_test_file(int test_file_index) +{ + const char * fcn_name = "construct_test_file()"; + char filename[512]; + hbool_t show_progress = FALSE; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + int cp = 0; + int min_dset = 0; + int max_dset = 0; + MPI_Comm dummy_comm = MPI_COMM_WORLD; + MPI_Info dummy_info = MPI_INFO_NULL; + + pass = TRUE; + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + HDassert(FILENAMES[test_file_index]); + + if ( h5_fixname(FILENAMES[test_file_index], H5P_DEFAULT, + filename, sizeof(filename)) + == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 1) Create a HDF5 file with the cache image FAPL entry. + * + * Verify that the cache is informed of the cache image FAPL entry. + * + * Set flags forcing full function of the cache image feature. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ TRUE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ TRUE, + /* config_fsm */ TRUE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ dummy_comm, + /* info */ dummy_info, + /* l_facc_type */ 0, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ 0); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Create a data set in the file. */ + + if ( pass ) { + + create_data_sets(file_id, min_dset++, max_dset++); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(1)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 3) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + while ( ( pass ) && ( max_dset < MAX_NUM_DSETS ) ) + { + + /* 4) Open the file. + * + * Verify that the metadata cache is instructed to load the + * metadata cache image. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ FALSE, + /* set_mdci_fapl */ TRUE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ dummy_comm, + /* info */ dummy_info, + /* l_facc_type */ 0, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ 0); + } + + if ( show_progress ) + HDfprintf(stdout, "%s:L1 cp = %d, max_dset = %d, pass = %d.\n", + fcn_name, cp, max_dset, pass); + + + /* 5) Create a data set in the file. */ + + if ( pass ) { + + create_data_sets(file_id, min_dset++, max_dset++); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded == 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block not loaded(1)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s:L2 cp = %d, max_dset = %d, pass = %d.\n", + fcn_name, cp + 1, max_dset, pass); + + + /* 6) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s:L3 cp = %d, max_dset = %d, pass = %d.\n", + fcn_name, cp + 2, max_dset, pass); + } /* end while */ + cp += 3; + + + /* 7) Open the file R/O. + * + * Verify that the file contains a metadata cache image + * superblock extension message. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ TRUE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ dummy_comm, + /* info */ dummy_info, + /* l_facc_type */ 0, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ 0); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 8) Open and close all data sets. + * + * Verify that the cache image has been loaded. + */ + + if ( pass ) { + + verify_data_sets(file_id, 0, max_dset - 1); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded == 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block not loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* 9) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + } + } + + return !pass; + +} /* construct_test_file() */ + + +/*------------------------------------------------------------------------- + * Function: create_data_sets() + * + * Purpose: If pass is TRUE on entry, create the specified data sets + * in the indicated file. + * + * Data sets and their contents must be well know, as we + * will verify that they contain the expected data later. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 7/15/15 + * + * Modifications: + * + * Added min_dset and max_dset parameters and supporting + * code. This allows the caller to specify a range of + * datasets to create. + * JRM -- 8/20/15 + * + *------------------------------------------------------------------------- + */ + +static void +create_data_sets(hid_t file_id, int min_dset, int max_dset) +{ + const char * fcn_name = "create_data_sets()"; + char dset_name[64]; + hbool_t show_progress = FALSE; + hbool_t valid_chunk; + hbool_t verbose = FALSE; + int cp = 0; + int i, j, k, l, m; + int data_chunk[CHUNK_SIZE][CHUNK_SIZE]; + herr_t status; + hid_t dataspace_id = -1; + hid_t filespace_ids[MAX_NUM_DSETS]; + hid_t memspace_id = -1; + hid_t dataset_ids[MAX_NUM_DSETS]; + hid_t properties = -1; + hsize_t dims[2]; + hsize_t a_size[2]; + hsize_t offset[2]; + hsize_t chunk_size[2]; + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + HDassert(0 <= min_dset); + HDassert(min_dset <= max_dset); + HDassert(max_dset < MAX_NUM_DSETS); + + /* create the datasets */ + + if ( pass ) { + + i = min_dset; + + while ( ( pass ) && ( i <= max_dset ) ) + { + /* create a dataspace for the chunked dataset */ + dims[0] = DSET_SIZE; + dims[1] = DSET_SIZE; + dataspace_id = H5Screate_simple(2, dims, NULL); + + if ( dataspace_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Screate_simple() failed."; + } + + /* set the dataset creation plist to specify that the raw data is + * to be partioned into 10X10 element chunks. + */ + + if ( pass ) { + + chunk_size[0] = CHUNK_SIZE; + chunk_size[1] = CHUNK_SIZE; + properties = H5Pcreate(H5P_DATASET_CREATE); + + if ( properties < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pcreate() failed."; + } + } + + if ( pass ) { + + if ( H5Pset_chunk(properties, 2, chunk_size) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_chunk() failed."; + } + } + + /* create the dataset */ + if ( pass ) { + + sprintf(dset_name, "/dset%03d", i); + dataset_ids[i] = H5Dcreate2(file_id, dset_name, H5T_STD_I32BE, + dataspace_id, H5P_DEFAULT, + properties, H5P_DEFAULT); + + if ( dataset_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dcreate() failed."; + } + } + + /* get the file space ID */ + if ( pass ) { + + filespace_ids[i] = H5Dget_space(dataset_ids[i]); + + if ( filespace_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dget_space() failed."; + } + } + + i++; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* create the mem space to be used to read and write chunks */ + if ( pass ) { + + dims[0] = CHUNK_SIZE; + dims[1] = CHUNK_SIZE; + memspace_id = H5Screate_simple(2, dims, NULL); + + if ( memspace_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Screate_simple() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* select in memory hyperslab */ + if ( pass ) { + + offset[0] = 0; /*offset of hyperslab in memory*/ + offset[1] = 0; + a_size[0] = CHUNK_SIZE; /*size of hyperslab*/ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, offset, NULL, + a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sselect_hyperslab() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* initialize all datasets on a round robin basis */ + i = 0; + while ( ( pass ) && ( i < DSET_SIZE ) ) + { + j = 0; + while ( ( pass ) && ( j < DSET_SIZE ) ) + { + m = min_dset; + while ( ( pass ) && ( m <= max_dset ) ) + { + /* initialize the slab */ + for ( k = 0; k < CHUNK_SIZE; k++ ) + { + for ( l = 0; l < CHUNK_SIZE; l++ ) + { + data_chunk[k][l] = (DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l; + } + } + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; /*offset of hyperslab in file*/ + offset[1] = (hsize_t)j; + a_size[0] = CHUNK_SIZE; /*size of hyperslab*/ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk H5Sselect_hyperslab() failed."; + } + + /* write the chunk to file */ + status = H5Dwrite(dataset_ids[m], H5T_NATIVE_INT, memspace_id, + filespace_ids[m], H5P_DEFAULT, data_chunk); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dwrite() failed."; + } + m++; + } + j += CHUNK_SIZE; + } + + i += CHUNK_SIZE; + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* read data from data sets and validate it */ + i = 0; + while ( ( pass ) && ( i < DSET_SIZE ) ) + { + j = 0; + while ( ( pass ) && ( j < DSET_SIZE ) ) + { + m = min_dset; + while ( ( pass ) && ( m <= max_dset ) ) + { + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; /* offset of hyperslab in file */ + offset[1] = (hsize_t)j; + a_size[0] = CHUNK_SIZE; /* size of hyperslab */ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + + /* read the chunk from file */ + if ( pass ) { + + status = H5Dread(dataset_ids[m], H5T_NATIVE_INT, + memspace_id, filespace_ids[m], + H5P_DEFAULT, data_chunk); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + } + + /* validate the slab */ + if ( pass ) { + + valid_chunk = TRUE; + for ( k = 0; k < CHUNK_SIZE; k++ ) + { + for ( l = 0; l < CHUNK_SIZE; l++ ) + { + if ( data_chunk[k][l] + != + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l) ) { + + valid_chunk = FALSE; + + if ( verbose ) { + + HDfprintf(stdout, + "data_chunk[%0d][%0d] = %0d, expect %0d.\n", + k, l, data_chunk[k][l], + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l)); + HDfprintf(stdout, + "m = %d, i = %d, j = %d, k = %d, l = %d\n", + m, i, j, k, l); + } + } + } + } + + if ( ! valid_chunk ) { + + pass = FALSE; + failure_mssg = "slab validation failed."; + + if ( verbose ) { + + fprintf(stdout, + "Chunk (%0d, %0d) in /dset%03d is invalid.\n", + i, j, m); + } + } + } + m++; + } + j += CHUNK_SIZE; + } + i += CHUNK_SIZE; + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* close the file spaces */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Sclose(filespace_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose() failed."; + } + i++; + } + + + /* close the datasets */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Dclose(dataset_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dclose() failed."; + } + i++; + } + + /* close the mem space */ + if ( pass ) { + + if ( H5Sclose(memspace_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose(memspace_id) failed."; + } + } + + return; + +} /* create_data_sets() */ + + +/*------------------------------------------------------------------------- + * Function: delete_data_sets() + * + * Purpose: If pass is TRUE on entry, verify and then delete the + * dataset(s) indicated by min_dset and max_dset in the + * indicated file. + * + * Data sets and their contents must be well know, as we + * will verify that they contain the expected data later. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 10/31/16 + * + * Modifications: + * + * None. + * JRM -- 8/20/15 + * + *------------------------------------------------------------------------- + */ + +static void +delete_data_sets(hid_t file_id, int min_dset, int max_dset) +{ + const char * fcn_name = "delete_data_sets()"; + char dset_name[64]; + hbool_t show_progress = FALSE; + int cp = 0; + int i; + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + HDassert(0 <= min_dset); + HDassert(min_dset <= max_dset); + HDassert(max_dset < MAX_NUM_DSETS); + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* first, verify the contents of the target dataset(s) */ + verify_data_sets(file_id, min_dset, max_dset); + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* now delete the target datasets */ + if ( pass ) { + + i = min_dset; + + while ( ( pass ) && ( i <= max_dset ) ) + { + sprintf(dset_name, "/dset%03d", i); + + if ( H5Ldelete(file_id, dset_name, H5P_DEFAULT) < 0) { + + pass = FALSE; + failure_mssg = "H5Ldelete() failed."; + } + + i++; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + return; + +} /* delete_data_sets() */ + + +/*------------------------------------------------------------------------- + * Function: open_hdf5_file() + * + * Purpose: If pass is true on entry, create or open the specified HDF5 + * and test to see if it has a metadata cache image superblock + * extension message. + * + * Set pass to FALSE and issue a suitable failure + * message if either the file contains a metadata cache image + * superblock extension and mdci_sbem_expected is TRUE, or + * vise versa. + * + * If mdci_sbem_expected is TRUE, also verify that the metadata + * cache has been advised of this. + * + * If read_only is TRUE, open the file read only. Otherwise + * open the file read/write. + * + * If set_mdci_fapl is TRUE, set the metadata cache image + * FAPL entry when opening the file, and verify that the + * metadata cache is notified. + * + * If config_fsm is TRUE, setup the persistant free space + * manager. Note that this flag may only be set if + * create_file is also TRUE. + * + * Return pointers to the cache data structure and file data + * structures. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 7/14/15 + * + * Modifications: + * + * Modified function to handle parallel file creates / opens. + * + * JRM -- 2/1/17 + * + *------------------------------------------------------------------------- + */ + +static void +open_hdf5_file(const hbool_t create_file, + const hbool_t mdci_sbem_expected, + const hbool_t read_only, + const hbool_t set_mdci_fapl, + const hbool_t config_fsm, + const char * hdf_file_name, + const unsigned cache_image_flags, + hid_t * file_id_ptr, + H5F_t ** file_ptr_ptr, + H5C_t ** cache_ptr_ptr, + MPI_Comm comm, + MPI_Info info, + int l_facc_type, + const hbool_t all_coll_metadata_ops, + const hbool_t coll_metadata_write, + const int md_write_strat) +{ + const char * fcn_name = "open_hdf5_file()"; + hbool_t show_progress = FALSE; + hbool_t verbose = FALSE; + int cp = 0; + hid_t fapl_id = -1; + hid_t fcpl_id = -1; + hid_t file_id = -1; + herr_t result; + H5F_t * file_ptr = NULL; + H5C_t * cache_ptr = NULL; + H5C_cache_image_ctl_t image_ctl; + H5AC_cache_image_config_t cache_image_config = { + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, + TRUE, + FALSE, + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; + + if ( pass ) + { + /* opening the file both read only and with a cache image + * requested is a contradiction. We resolve it by ignoring + * the cache image request silently. + */ + if ( ( create_file && mdci_sbem_expected ) || + ( create_file && read_only ) || + ( config_fsm && !create_file ) || + ( hdf_file_name == NULL ) || + ( ( set_mdci_fapl ) && ( cache_image_flags == 0 ) ) || + ( ( set_mdci_fapl ) && + ( (cache_image_flags & ~H5C_CI__ALL_FLAGS) != 0 ) ) || + ( file_id_ptr == NULL ) || + ( file_ptr_ptr == NULL ) || + ( cache_ptr_ptr == NULL ) || + ( l_facc_type != (l_facc_type & (FACC_MPIO)) ) ) { + + failure_mssg = + "Bad param(s) on entry to open_hdf5_file().\n"; + + pass = FALSE; + } else if ( verbose ) { + + HDfprintf(stdout, "%s: HDF file name = \"%s\".\n", + fcn_name, hdf_file_name); + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* create a file access propertly list. */ + if ( pass ) { + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + + if ( fapl_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pcreate() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* call H5Pset_libver_bounds() on the fapl_id */ + if ( pass ) { + + if ( H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) + < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_libver_bounds() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* get metadata cache image config -- verify that it is the default */ + if ( pass ) { + + result = H5Pget_mdc_image_config(fapl_id, &cache_image_config); + + if ( result < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pget_mdc_image_config() failed.\n"; + } + + if ( ( cache_image_config.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( cache_image_config.generate_image != FALSE ) || + ( cache_image_config.save_resize_status != FALSE ) || + ( cache_image_config.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) ) { + + pass = FALSE; + failure_mssg = "Unexpected default cache image config.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* set metadata cache image fapl entry if indicated */ + if ( ( pass ) && ( set_mdci_fapl ) ) { + + /* set cache image config fields to taste */ + cache_image_config.generate_image = TRUE; + cache_image_config.save_resize_status = FALSE; + cache_image_config.entry_ageout = H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE; + + result = H5Pset_mdc_image_config(fapl_id, &cache_image_config); + + if ( result < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_mdc_image_config() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* setup the persistant free space manager if indicated */ + if ( ( pass ) && ( config_fsm ) ) { + + fcpl_id = H5Pcreate(H5P_FILE_CREATE); + + if ( fcpl_id <= 0 ) { + + pass = FALSE; + failure_mssg = "H5Pcreate(H5P_FILE_CREATE) failed."; + } + } + + if ( ( pass ) && ( config_fsm ) ) { + + if ( H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, + TRUE, (hsize_t)1) == FAIL ) { + pass = FALSE; + failure_mssg = "H5Pset_file_space_strategy() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + /* set Parallel access with communicator */ + if ( H5Pset_fapl_mpio(fapl_id, comm, info) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_fapl_mpio() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + if (H5Pset_all_coll_metadata_ops(fapl_id, all_coll_metadata_ops) < 0) { + + pass = FALSE; + failure_mssg = "H5Pset_all_coll_metadata_ops() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + if ( H5Pset_coll_metadata_write(fapl_id, coll_metadata_write) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_coll_metadata_write() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( l_facc_type == FACC_MPIO ) ) { + + /* set the desired parallel metadata write strategy */ + H5AC_cache_config_t mdc_config; + + mdc_config.version = H5C__CURR_AUTO_SIZE_CTL_VER; + + if ( H5Pget_mdc_config(fapl_id, &mdc_config) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pget_mdc_config() failed.\n"; + } + + mdc_config.metadata_write_strategy = md_write_strat; + + if ( H5Pset_mdc_config(fapl_id, &mdc_config) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_mdc_config() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* open the file */ + if ( pass ) { + + if ( create_file ) { + + if ( fcpl_id != -1 ) + + file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, + fcpl_id, fapl_id); + else + + file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, + H5P_DEFAULT, fapl_id); + + } else { + + if ( read_only ) + + file_id = H5Fopen(hdf_file_name, H5F_ACC_RDONLY, fapl_id); + + else + + file_id = H5Fopen(hdf_file_name, H5F_ACC_RDWR, fapl_id); + } + + if ( file_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fcreate() or H5Fopen() failed.\n"; + + } else { + + file_ptr = (struct H5F_t *)H5I_object_verify(file_id, H5I_FILE); + + if ( file_ptr == NULL ) { + + pass = FALSE; + failure_mssg = "Can't get file_ptr."; + + if ( verbose ) { + HDfprintf(stdout, "%s: Can't get file_ptr.\n", fcn_name); + } + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* get a pointer to the files internal data structure and then + * to the cache structure + */ + if ( pass ) { + + if ( file_ptr->shared->cache == NULL ) { + + pass = FALSE; + failure_mssg = "can't get cache pointer(1).\n"; + + } else { + + cache_ptr = file_ptr->shared->cache; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* verify expected metadata cache status */ + + /* get the cache image control structure from the cache, and verify + * that it contains the expected values. + * + * Then set the flags in this structure to the specified value. + */ + if ( pass ) { + + if ( H5C_get_cache_image_config(cache_ptr, &image_ctl) < 0 ) { + + pass = FALSE; + failure_mssg = "error returned by H5C_get_cache_image_config()."; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + if ( set_mdci_fapl ) { + + if ( read_only ) { + + if ( ( image_ctl.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( image_ctl.generate_image != FALSE ) || + ( image_ctl.save_resize_status != FALSE ) || + ( image_ctl.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) || + ( image_ctl.flags != H5C_CI__ALL_FLAGS ) ) { + + pass = FALSE; + failure_mssg = "Unexpected image_ctl values(1).\n"; + } + } else { + + if ( ( image_ctl.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( image_ctl.generate_image != TRUE ) || + ( image_ctl.save_resize_status != FALSE ) || + ( image_ctl.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) || + ( image_ctl.flags != H5C_CI__ALL_FLAGS ) ) { + + pass = FALSE; + failure_mssg = "Unexpected image_ctl values(2).\n"; + } + } + } else { + + if ( ( image_ctl.version != + H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION ) || + ( image_ctl.generate_image != FALSE ) || + ( image_ctl.save_resize_status != FALSE ) || + ( image_ctl.entry_ageout != + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE ) || + ( image_ctl.flags != H5C_CI__ALL_FLAGS ) ) { + + pass = FALSE; + failure_mssg = "Unexpected image_ctl values(3).\n"; + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( ( pass ) && ( set_mdci_fapl ) ) { + + image_ctl.flags = cache_image_flags; + + if ( H5C_set_cache_image_config(file_ptr, cache_ptr, &image_ctl) < 0 ) { + + pass = FALSE; + failure_mssg = "error returned by H5C_set_cache_image_config()."; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + if ( cache_ptr->close_warning_received == TRUE ) { + + pass = FALSE; + failure_mssg = "Unexpected value of close_warning_received.\n"; + } + + if ( mdci_sbem_expected ) { + + if ( read_only ) { + + if ( ( cache_ptr->load_image != TRUE ) || + ( cache_ptr->delete_image != FALSE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message not present?\n"; + } + } else { + + if ( ( cache_ptr->load_image != TRUE ) || + ( cache_ptr->delete_image != TRUE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message not present?\n"; + } + } + } else { + + if ( ( cache_ptr->load_image == TRUE ) || + ( cache_ptr->delete_image == TRUE ) ) { + + pass = FALSE; + failure_mssg = "mdci sb extension message present?\n"; + } + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + *file_id_ptr = file_id; + *file_ptr_ptr = file_ptr; + *cache_ptr_ptr = cache_ptr; + } + + if ( show_progress ) { + HDfprintf(stdout, "%s: cp = %d, pass = %d -- exiting.\n", + fcn_name, cp++, pass); + + if ( ! pass ) + HDfprintf(stdout, "%s: failure_mssg = %s\n", + fcn_name, failure_mssg); + } + + return; + +} /* open_hdf5_file() */ + + +/*------------------------------------------------------------------------- + * Function: parse_flags + * + * Purpose: Parse the flags passed to this program, and load the + * values into the supplied field. + * + * Return: Success: 1 + * Failure: 0 + * + * Programmer: J Mainzer + * 4/28/11 + * + *------------------------------------------------------------------------- + */ +hbool_t +parse_flags(int argc, char * argv[], hbool_t * setup_ptr, hbool_t display) +{ + const char * fcn_name = "parse_flags()"; + const char * (ops[]) = {"setup"}; + int success = TRUE; + + if ( setup_ptr == NULL ) { + + success = FALSE; + HDfprintf(stdout, "%s: bad arg(s) on entry.\n", fcn_name); + } + + if ( ( success ) && + ( ( argc < 1 ) || ( argc > 2 ) ) ) { + + success = FALSE; + usage(); + } + + if ( success ) { + + if ( argc == 2 ) { + + if ( strcmp(argv[1], ops[0]) == 0 ) { + + *setup_ptr = TRUE; + + } else { + + success = FALSE; + usage(); + } + } else { + + *setup_ptr = FALSE; + } + } + + if ( ( success ) && ( display ) ) { + + if ( *setup_ptr ) + HDfprintf(stdout, "t_cache_image setup\n"); + else + HDfprintf(stdout, "t_cache_image\n"); + } + + return(success); + +} /* parse_flags() */ + + +/*------------------------------------------------------------------------- + * Function: usage + * + * Purpose: Display a brief message describing the purpose and use + * of the program. + * + * Return: void + * + * Programmer: John Mainzer + * 4/28/11 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +void +usage(void) +{ + const char * s[] = + { + "\n", + "t_cache_image:\n", + "\n", + "Run the parallel cache image tests. \n" + "\n" + "In general, this program is run via MPI. However, at present, files\n" + "with cache images can only be constructed by serial processes.\n", + "\n", + "To square this circle, one process in the parallel computation \n" + "forks a serial version of the test program to handle this detail.\n", + "The \"setup\" parameter indicates that t_cache_image is being \n", + "invokde for this purpose.\n", + "\n", + "usage: t_cache_image [setup]\n", + "\n", + "where:\n", + "\n", + " setup parameter forces creation of test file\n", + "\n", + "Returns 0 on success, 1 on failure.\n", + "\n", + NULL, + }; + int i = 0; + + while(s[i] != NULL) { + fprintf(stdout, "%s", s[i]); + i++; + } + + return; +} /* usage() */ + + +/*------------------------------------------------------------------------- + * Function: verify_data_sets() + * + * Purpose: If pass is TRUE on entry, verify that the data sets in the + * file exist and contain the expected data. + * + * Note that these data sets were created by + * create_data_sets() above. Thus any changes in that + * function must be reflected in this function, and + * vise-versa. + * + * On failure, set pass to FALSE, and set failure_mssg + * to point to an appropriate failure message. + * + * Do nothing if pass is FALSE on entry. + * + * Return: void + * + * Programmer: John Mainzer + * 7/15/15 + * + * Modifications: + * + * Added min_dset and max_dset parameters and supporting + * code. This allows the caller to specify a range of + * datasets to verify. + * JRM -- 8/20/15 + * + *------------------------------------------------------------------------- + */ + +static void +verify_data_sets(hid_t file_id, int min_dset, int max_dset) +{ + const char * fcn_name = "verify_data_sets()"; + char dset_name[64]; + hbool_t show_progress = FALSE; + hbool_t valid_chunk; + hbool_t verbose = FALSE; + int cp = 0; + int i, j, k, l, m; + int data_chunk[CHUNK_SIZE][CHUNK_SIZE]; + herr_t status; + hid_t filespace_ids[MAX_NUM_DSETS]; + hid_t memspace_id = -1; + hid_t dataset_ids[MAX_NUM_DSETS]; + hsize_t dims[2]; + hsize_t a_size[2]; + hsize_t offset[2]; + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + HDassert(0 <= min_dset); + HDassert(min_dset <= max_dset); + HDassert(max_dset < MAX_NUM_DSETS); + + /* open the datasets */ + + if ( pass ) { + + i = min_dset; + + while ( ( pass ) && ( i <= max_dset ) ) + { + /* open the dataset */ + if ( pass ) { + + sprintf(dset_name, "/dset%03d", i); + dataset_ids[i] = H5Dopen2(file_id, dset_name, H5P_DEFAULT); + + if ( dataset_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dopen2() failed."; + } + } + + /* get the file space ID */ + if ( pass ) { + + filespace_ids[i] = H5Dget_space(dataset_ids[i]); + + if ( filespace_ids[i] < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dget_space() failed."; + } + } + + i++; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* create the mem space to be used to read and write chunks */ + if ( pass ) { + + dims[0] = CHUNK_SIZE; + dims[1] = CHUNK_SIZE; + memspace_id = H5Screate_simple(2, dims, NULL); + + if ( memspace_id < 0 ) { + + pass = FALSE; + failure_mssg = "H5Screate_simple() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* select in memory hyperslab */ + if ( pass ) { + + offset[0] = 0; /*offset of hyperslab in memory*/ + offset[1] = 0; + a_size[0] = CHUNK_SIZE; /*size of hyperslab*/ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, offset, NULL, + a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sselect_hyperslab() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + + /* read data from data sets and validate it */ + i = 0; + while ( ( pass ) && ( i < DSET_SIZE ) ) + { + j = 0; + while ( ( pass ) && ( j < DSET_SIZE ) ) + { + m = min_dset; + while ( ( pass ) && ( m <= max_dset ) ) + { + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; /* offset of hyperslab in file */ + offset[1] = (hsize_t)j; + a_size[0] = CHUNK_SIZE; /* size of hyperslab */ + a_size[1] = CHUNK_SIZE; + status = H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + + /* read the chunk from file */ + if ( pass ) { + + status = H5Dread(dataset_ids[m], H5T_NATIVE_INT, + memspace_id, filespace_ids[m], + H5P_DEFAULT, data_chunk); + + if ( status < 0 ) { + + pass = FALSE; + failure_mssg = "disk hyperslab create failed."; + } + } + + /* validate the slab */ + if ( pass ) { + + valid_chunk = TRUE; + for ( k = 0; k < CHUNK_SIZE; k++ ) + { + for ( l = 0; l < CHUNK_SIZE; l++ ) + { + if ( data_chunk[k][l] + != + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l) ) { + + valid_chunk = FALSE; + + if ( verbose ) { + + HDfprintf(stdout, + "data_chunk[%0d][%0d] = %0d, expect %0d.\n", + k, l, data_chunk[k][l], + ((DSET_SIZE * DSET_SIZE * m) + + (DSET_SIZE * (i + k)) + j + l)); + HDfprintf(stdout, + "m = %d, i = %d, j = %d, k = %d, l = %d\n", + m, i, j, k, l); + } + } + } + } + + if ( ! valid_chunk ) { + + pass = FALSE; + failure_mssg = "slab validation failed."; + + if ( verbose ) { + + fprintf(stdout, + "Chunk (%0d, %0d) in /dset%03d is invalid.\n", + i, j, m); + } + } + } + m++; + } + j += CHUNK_SIZE; + } + i += CHUNK_SIZE; + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + + /* close the file spaces */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Sclose(filespace_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose() failed."; + } + i++; + } + + + /* close the datasets */ + i = min_dset; + while ( ( pass ) && ( i <= max_dset ) ) + { + if ( H5Dclose(dataset_ids[i]) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Dclose() failed."; + } + i++; + } + + /* close the mem space */ + if ( pass ) { + + if ( H5Sclose(memspace_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Sclose(memspace_id) failed."; + } + } + + return; + +} /* verify_data_sets() */ + + +/****************************************************************************/ +/******************************* Test Functions *****************************/ +/****************************************************************************/ + +/*------------------------------------------------------------------------- + * Function: verify_cache_image_RW() + * + * Purpose: Verify that a HDF5 file containing a cache image is + * opened and read correctly by PHDF5 with the specified + * metadata write strategy. + * + * Basic cycle of operation is as follows: + * + * 1) Open the test file created at the beginning of this + * test. + * + * Verify that the file contains a cache image. + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. + * + * 2) Verify that the file contains the expected data. + * + * 3) Close the file. + * + * 4) Open the file, and verify that it doesn't contain + * a cache image. + * + * 5) Verify that the file contains the expected data. + * + * 6) Close the file. + * + * 7) Delete the file. + * + * Return: void + * + * Programmer: John Mainzer + * 1/25/17 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ + +static unsigned +verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank) +{ + const char * fcn_name = "verify_cache_imageRW()"; + char filename[512]; + hbool_t show_progress = FALSE; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + int cp = 0; + int i; + + pass = TRUE; + + if ( mpi_rank == 0 ) { + + switch(md_write_strat) { + + case H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY: + TESTING("parallel CI load test -- proc0 md write -- R/W"); + break; + + case H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED: + TESTING("parallel CI load test -- dist md write -- R/W"); + break; + + default: + TESTING("parallel CI load test -- unknown md write -- R/W"); + pass = FALSE; + break; + } + } + + show_progress = ( ( show_progress ) && ( mpi_rank == 0 ) ); + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + if ( h5_fixname(FILENAMES[file_name_id], H5P_DEFAULT, + filename, sizeof(filename)) == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 1) Open the test file created at the beginning of this test. + * + * Verify that the file contains a cache image. + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. + */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ MPI_COMM_WORLD, + /* info */ MPI_INFO_NULL, + /* l_facc_type */ FACC_MPIO, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ md_write_strat); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that only process 0 reads the cache image. */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that all other processes receive the cache image block + * from process 0. + */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Verify that the file contains the expected data. */ + if ( pass ) { + + verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded == 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block not loaded(2)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + + /* 3) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 4) Open the file, and verify that it doesn't contain a cache image. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr, + /* comm */ MPI_COMM_WORLD, + /* info */ MPI_INFO_NULL, + /* l_facc_type */ FACC_MPIO, + /* all_coll_metadata_ops */ FALSE, + /* coll_metadata_write */ FALSE, + /* md_write_strat */ md_write_strat); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 5) Verify that the file contains the expected data. */ + + if ( pass ) { + + verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + } + +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 0 ) { + + pass = FALSE; + failure_mssg = "metadata cache image block loaded(1)."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + + /* 6) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed.\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 7) Delete the file. */ + + if ( pass ) { + + /* wait for everyone to close the file */ + MPI_Barrier(MPI_COMM_WORLD); + + if ( ( mpi_rank == 0 ) && ( HDremove(filename) < 0 ) ) { + + pass = FALSE; + failure_mssg = "HDremove() failed.\n"; + } + } + + + /* report results */ + if ( mpi_rank == 0 ) { + + if ( pass ) { + + PASSED(); + + } else { + + H5_FAILED(); + + if ( show_progress ) + HDfprintf(stdout, "%s: failure_mssg = \"%s\"\n", failure_mssg); + } + } + + + return !pass; + +} /* verify_cache_imageRW() */ + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Run parallel tests on the cache image feature. + * + * At present, cache image is disabled in parallel, and + * thus these tests are restructed to verifying that a + * file with a cache image can be opened in the parallel + * case, and verifying that instructions to create a + * cache image are ignored in the parallel case. + * + * WARNING: This test uses fork() and execve(), and + * therefore will not run on Windows. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: John Mainzer + * 1/25/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +int +main(int argc, char **argv) +{ + hbool_t setup = FALSE; + unsigned nerrs = 0; + int i; + int mpi_size; + int mpi_rank; + + if ( ! parse_flags(argc, argv, &setup, FALSE) ) + exit(1); /* exit now if unable to parse flags */ + + if ( setup ) { /* construct test file and exit */ + + H5open(); + HDfprintf(stdout, "Constructing test files: \n"); + HDfflush(stdout); + + i = 0; + while ( FILENAMES[i] != NULL ) { + + HDfprintf(stdout, " writing %s ... ", FILENAMES[i]); + HDfflush(stdout); + construct_test_file(i); + + if ( pass ) { + + printf("done.\n"); + HDfflush(stdout); + + } else { + + printf("failed.\n"); + exit(1); + } + i++; + } + HDfprintf(stdout, "Test file construction complete.\n"); + exit(0); + } + + HDassert(!setup); + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + /* Attempt to turn off atexit post processing so that in case errors + * happen during the test and the process is aborted, it will not get + * hang in the atexit post processing in which it may try to make MPI + * calls. By then, MPI calls may not work. + */ + if (H5dont_atexit() < 0){ + printf("%d:Failed to turn off atexit processing. Continue.\n", + mpi_rank); + }; + + H5open(); + + if ( mpi_rank == 0 ) { + printf("===================================\n"); + printf("Parallel metadata cache image tests\n"); + printf(" mpi_size = %d\n", mpi_size); + printf("===================================\n"); + } + + if ( mpi_size < 2 ) { + + if ( mpi_rank == 0 ) { + + printf(" Need at least 2 processes. Exiting.\n"); + } + goto finish; + } + + if ( mpi_rank == 0 ) { /* create test files */ + + int child_status; + pid_t child_pid; + + child_pid = fork(); + + if ( child_pid == 0 ) { /* this is the child process */ + + /* fun and games to shutup the compiler */ + char param0[32] = "t_cache_image"; + char param1[32] = "setup"; + char * child_argv[] = {param0, param1, NULL}; + + /* we may need to play with the path here */ + if ( execv("t_cache_image", child_argv) == -1 ) { + + HDfprintf(stdout, + "execl() of setup process failed. errno = %d(%s)\n", + errno, strerror(errno)); + exit(1); + } + + } else if ( child_pid != -1 ) { + /* this is the parent process -- wait until child is done */ + if ( -1 == waitpid(child_pid, &child_status, WUNTRACED)) { + + HDfprintf(stdout, "can't wait on setup process.\n"); + + } else if ( ! WIFEXITED(child_status) ) { + + HDfprintf(stdout, "setup process hasn't exitied.\n"); + + } else if ( WEXITSTATUS(child_status) != 0 ) { + + HDfprintf(stdout, "setup process reports failure.\n"); + + } else { + + HDfprintf(stdout, + "testfile construction complete -- proceeding with tests.\n"); + } + } else { /* fork failed */ + + HDfprintf(stdout, "can't create process to construct test file.\n"); + } + } + + /* can't start test until test file exists */ + MPI_Barrier(MPI_COMM_WORLD); + + nerrs += verify_cache_image_RW(0, + H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, mpi_rank); + nerrs += verify_cache_image_RW(1, + H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, mpi_rank); + +finish: + + /* make sure all processes are finished before final report, cleanup + * and exit. + */ + MPI_Barrier(MPI_COMM_WORLD); + + if ( mpi_rank == 0 ) { /* only process 0 reports */ + sleep(10); + printf("===================================\n"); + if ( nerrs > 0 ) { + printf("***metadata cache image tests detected %d failures***\n", + nerrs); + } + else { + printf("metadata cache image tests finished with no failures\n"); + } + printf("===================================\n"); + } + + /* takedown_derived_types(); */ + + /* close HDF5 library */ + H5close(); + + /* MPI_Finalize must be called AFTER H5close which may use MPI calls */ + MPI_Finalize(); + + /* cannot just return (nerrs) because exit code is limited to 1byte */ + return(nerrs > 0); + +} /* main() */ + diff --git a/testpar/t_file.c b/testpar/t_file.c index b2f1d5e..4a923d4 100644 --- a/testpar/t_file.c +++ b/testpar/t_file.c @@ -19,6 +19,30 @@ #include "testphdf5.h" +#include "H5PBprivate.h" +#include "H5Iprivate.h" + +/* + * This file needs to access private information from the H5F package. + */ +#define H5C_FRIEND /*suppress error about including H5Cpkg */ +#include "H5Cpkg.h" +#define H5AC_FRIEND /*suppress error about including H5ACpkg */ +#include "H5ACpkg.h" +#define H5MF_FRIEND /*suppress error about including H5MFpkg */ +#include "H5MFpkg.h" +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_TESTING +#include "H5Fpkg.h" + +#define NUM_DSETS 5 + +int mpi_size, mpi_rank; + +static int create_file(const char *filename, hid_t fcpl, hid_t fapl, int metadata_write_strategy); +static int open_file(const char *filename, hid_t fapl, int metadata_write_strategy, + hsize_t page_size, size_t page_buffer_size); + /* * test file access by communicator besides COMM_WORLD. * Split COMM_WORLD into two, one (even_comm) contains the original @@ -33,7 +57,6 @@ void test_split_comm_access(void) { - int mpi_size, mpi_rank; MPI_Comm comm; MPI_Info info = MPI_INFO_NULL; int is_old, mrc; @@ -95,6 +118,595 @@ test_split_comm_access(void) } void +test_page_buffer_access(void) +{ + hid_t file_id = -1; /* File ID */ + hid_t fcpl, fapl, fapl_self; + hid_t dxpl_id = H5P_DATASET_XFER_DEFAULT; + size_t page_count = 0; + int i, num_elements = 200; + haddr_t raw_addr, meta_addr; + int *data; + H5F_t *f = NULL; + herr_t ret; /* generic return value */ + const char *filename; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + filename = (const char *)GetTestParameters(); + + if (VERBOSE_MED) + printf("Page Buffer Usage in Parallel %s\n", filename); + + fapl = create_faccess_plist(MPI_COMM_WORLD, MPI_INFO_NULL, facc_type); + VRFY((fapl >= 0), "create_faccess_plist succeeded"); + fcpl = H5Pcreate(H5P_FILE_CREATE); + VRFY((fcpl >= 0), ""); + + ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 1, (hsize_t)0); + VRFY((ret == 0), ""); + ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*100); + VRFY((ret == 0), ""); + ret = H5Pset_page_buffer_size(fapl, sizeof(int)*100000, 0, 0); + VRFY((ret == 0), ""); + + /* This should fail because collective metadata writes are not supported with page buffering */ + H5E_BEGIN_TRY { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + VRFY((file_id < 0), "H5Fcreate failed"); + + /* disable collective metadata writes for page buffering to work */ + ret = H5Pset_coll_metadata_write(fapl, FALSE); + VRFY((ret >= 0), ""); + + ret = create_file(filename, fcpl, fapl, H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + VRFY((ret == 0), ""); + ret = open_file(filename, fapl, H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, sizeof(int)*100, sizeof(int)*100000); + VRFY((ret == 0), ""); + + ret = create_file(filename, fcpl, fapl, H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); + VRFY((ret == 0), ""); + ret = open_file(filename, fapl, H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY, sizeof(int)*100, sizeof(int)*100000); + VRFY((ret == 0), ""); + + ret = H5Pset_file_space_page_size(fcpl, sizeof(int)*100); + VRFY((ret == 0), ""); + + data = (int *) HDmalloc(sizeof(int)*(size_t)num_elements); + + /* intialize all the elements to have a value of -1 */ + for(i=0 ; i<num_elements ; i++) + data[i] = -1; + + /* MSC - why this stopped working ? */ +#if 0 + if(MAINPROCESS) { + hid_t fapl_self; + + fapl_self = create_faccess_plist(MPI_COMM_SELF, MPI_INFO_NULL, facc_type); + + ret = H5Pset_page_buffer_size(fapl_self, sizeof(int)*1000, 0, 0); + VRFY((ret == 0), ""); + /* collective metadata writes do not work with page buffering */ + ret = H5Pset_coll_metadata_write(fapl_self, FALSE); + VRFY((ret >= 0), ""); + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl_self); + VRFY((file_id >= 0), ""); + + /* Get a pointer to the internal file object */ + f = (H5F_t *)H5I_object(file_id); + + VRFY((f->shared->page_buf != NULL), "Page Buffer created with 1 process"); + + /* allocate space for 200 raw elements */ + raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((raw_addr != HADDR_UNDEF), ""); + + /* allocate space for 200 metadata elements */ + meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((meta_addr != HADDR_UNDEF), ""); + + page_count = 0; + + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data); + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the first 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + page_count += 2; + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the second 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i+50; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update 100 - 200 */ + for(i=0 ; i<100 ; i++) + data[i] = i+100; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + ret = H5PB_flush(f, dxpl_id, FALSE); + VRFY((ret == 0), ""); + + /* read elements 0 - 200 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*200, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*200, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + + /* read elements 0 - 50 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + + /* close the file */ + ret = H5Fclose(file_id); + VRFY((ret >= 0), "H5Fclose succeeded"); + ret = H5Pclose(fapl_self); + VRFY((ret>=0), "H5Pclose succeeded"); + } +#endif + + MPI_Barrier(MPI_COMM_WORLD); + + if(mpi_size > 1) { + ret = H5Pset_page_buffer_size(fapl, sizeof(int)*1000, 0, 0); + VRFY((ret == 0), ""); + /* collective metadata writes do not work with page buffering */ + ret = H5Pset_coll_metadata_write(fapl, FALSE); + VRFY((ret >= 0), ""); + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + VRFY((file_id >= 0), ""); + + /* Get a pointer to the internal file object */ + f = (H5F_t *)H5I_object(file_id); + + VRFY((f->shared->page_buf != NULL), "Page Buffer created with 1 process"); + + /* allocate space for 200 raw elements */ + raw_addr = H5MF_alloc(f, H5FD_MEM_DRAW, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((raw_addr != HADDR_UNDEF), ""); + /* allocate space for 200 metadata elements */ + meta_addr = H5MF_alloc(f, H5FD_MEM_SUPER, H5AC_ind_read_dxpl_id, sizeof(int)*(size_t)num_elements); + VRFY((meta_addr != HADDR_UNDEF), ""); + + page_count = 0; + + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*(size_t)num_elements, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*(size_t)num_elements, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the first 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update the second 50 elements */ + for(i=0 ; i<50 ; i++) + data[i] = i+50; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*50), sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* update 100 - 200 */ + for(i=0 ; i<100 ; i++) + data[i] = i+100; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr+(sizeof(int)*100), sizeof(int)*100, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + /* read elements 0 - 200 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*200, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*200, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 200; i++) + VRFY((data[i] == i), "Read different values than written"); + + /* read elements 0 - 50 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + page_count += 1; + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == i), "Read different values than written"); + + MPI_Barrier(MPI_COMM_WORLD); + /* reset the first 50 elements to -1*/ + for(i=0 ; i<50 ; i++) + data[i] = -1; + ret = H5F_block_write(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + ret = H5F_block_write(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + + /* read elements 0 - 50 */ + ret = H5F_block_read(f, H5FD_MEM_DRAW, raw_addr, sizeof(int)*50, H5AC_rawdata_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == -1), "Read different values than written"); + ret = H5F_block_read(f, H5FD_MEM_SUPER, meta_addr, sizeof(int)*50, H5AC_ind_read_dxpl_id, data); + VRFY((ret == 0), ""); + VRFY((H5SL_count(f->shared->page_buf->slist_ptr) == page_count), "Wrong number of pages in PB"); + for (i=0; i < 50; i++) + VRFY((data[i] == -1), "Read different values than written"); + + /* close the file */ + ret = H5Fclose(file_id); + VRFY((ret >= 0), "H5Fclose succeeded"); + } + + ret = H5Pclose(fapl); + VRFY((ret>=0), "H5Pclose succeeded"); + ret = H5Pclose(fcpl); + VRFY((ret>=0), "H5Pclose succeeded"); + + HDfree(data); + data = NULL; + MPI_Barrier(MPI_COMM_WORLD); +} + +static int +create_file(const char *filename, hid_t fcpl, hid_t fapl, int metadata_write_strategy) +{ + hid_t file_id, dset_id, grp_id; + hid_t sid, mem_dataspace; + hsize_t start[RANK]; + hsize_t count[RANK]; + hsize_t stride[RANK]; + hsize_t block[RANK]; + DATATYPE *data_array = NULL; + hsize_t dims[RANK], i; + hsize_t num_elements; + int k; + char dset_name[10]; + H5F_t *f = NULL; + H5C_t *cache_ptr = NULL; + H5AC_cache_config_t config; + herr_t ret; + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + VRFY((file_id >= 0), ""); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + f = (H5F_t *)H5I_object(file_id); + VRFY((f != NULL), ""); + + cache_ptr = f->shared->cache; + VRFY((cache_ptr->magic == H5C__H5C_T_MAGIC), ""); + + cache_ptr->ignore_tags = TRUE; + H5C_stats__reset(cache_ptr); + config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + + ret = H5AC_get_cache_auto_resize_config(cache_ptr, &config); + VRFY((ret == 0), ""); + + config.metadata_write_strategy = metadata_write_strategy; + + ret = H5AC_set_cache_auto_resize_config(cache_ptr, &config); + VRFY((ret == 0), ""); + + grp_id = H5Gcreate2(file_id, "GROUP", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((grp_id >= 0), ""); + + dims[0] = ROW_FACTOR*mpi_size; + dims[1] = COL_FACTOR*mpi_size; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + + /* Each process takes a slabs of rows. */ + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + num_elements = block[0] * block[1]; + /* allocate memory for data buffer */ + data_array = (DATATYPE *)HDmalloc(num_elements*sizeof(DATATYPE)); + VRFY((data_array != NULL), "data_array HDmalloc succeeded"); + /* put some trivial data in the data_array */ + for(i=0 ; i<num_elements; i++) + data_array[i] = mpi_rank + 1; + + ret = H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (1, &num_elements, NULL); + VRFY((mem_dataspace >= 0), ""); + + for(k=0 ; k<NUM_DSETS; k++) { + sprintf(dset_name, "D1dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + sprintf(dset_name, "D2dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + sprintf(dset_name, "D3dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + sprintf(dset_name, "dset%d", k); + dset_id = H5Dcreate2(grp_id, dset_name, H5T_NATIVE_INT, sid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + + ret = H5Dwrite(dset_id, H5T_NATIVE_INT, mem_dataspace, sid, H5P_DEFAULT, data_array); + VRFY((ret == 0), ""); + + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + HDmemset(data_array, 0, num_elements*sizeof(DATATYPE)); + dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, mem_dataspace, sid, H5P_DEFAULT, data_array); + VRFY((ret == 0), ""); + + ret = H5Dclose(dset_id); + VRFY((ret == 0), ""); + + for (i=0; i < num_elements; i++) + VRFY((data_array[i] == mpi_rank+1), "Dataset Verify failed"); + + sprintf(dset_name, "D1dset%d", k); + ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); + VRFY((ret == 0), ""); + sprintf(dset_name, "D2dset%d", k); + ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); + VRFY((ret == 0), ""); + sprintf(dset_name, "D3dset%d", k); + ret = H5Ldelete(grp_id, dset_name, H5P_DEFAULT); + VRFY((ret == 0), ""); + } + + ret = H5Gclose(grp_id); + VRFY((ret == 0), ""); + ret = H5Fclose(file_id); + VRFY((ret == 0), ""); + ret = H5Sclose(sid); + VRFY((ret == 0), ""); + ret = H5Sclose(mem_dataspace); + VRFY((ret == 0), ""); + + MPI_Barrier(MPI_COMM_WORLD); + HDfree(data_array); + return 0; +} /* create_file */ + +static int +open_file(const char *filename, hid_t fapl, int metadata_write_strategy, + hsize_t page_size, size_t page_buffer_size) +{ + hid_t file_id, dset_id, grp_id, grp_id2; + hid_t sid, mem_dataspace; + DATATYPE *data_array = NULL; + hsize_t dims[RANK]; + hsize_t start[RANK]; + hsize_t count[RANK]; + hsize_t stride[RANK]; + hsize_t block[RANK]; + int i, k, ndims; + hsize_t num_elements; + char dset_name[10]; + H5F_t *f = NULL; + H5C_t *cache_ptr = NULL; + H5AC_cache_config_t config; + herr_t ret; + + config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + ret = H5Pget_mdc_config(fapl, &config); + VRFY((ret == 0), ""); + + config.metadata_write_strategy = metadata_write_strategy; + + ret = H5Pget_mdc_config(fapl, &config); + VRFY((ret == 0), ""); + + file_id = H5Fopen(filename, H5F_ACC_RDWR, fapl); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((file_id >= 0), ""); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + f = (H5F_t *)H5I_object(file_id); + VRFY((f != NULL), ""); + + cache_ptr = f->shared->cache; + VRFY((cache_ptr->magic == H5C__H5C_T_MAGIC), ""); + + MPI_Barrier(MPI_COMM_WORLD); + + VRFY((f->shared->page_buf != NULL), ""); + VRFY((f->shared->page_buf->page_size == page_size), ""); + VRFY((f->shared->page_buf->max_size == page_buffer_size), ""); + + grp_id = H5Gopen2(file_id, "GROUP", H5P_DEFAULT); + VRFY((grp_id >= 0), ""); + + dims[0] = ROW_FACTOR*mpi_size; + dims[1] = COL_FACTOR*mpi_size; + + /* Each process takes a slabs of rows. */ + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + num_elements = block[0] * block[1]; + /* allocate memory for data buffer */ + data_array = (DATATYPE *)HDmalloc(num_elements*sizeof(DATATYPE)); + VRFY((data_array != NULL), "data_array HDmalloc succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (1, &num_elements, NULL); + VRFY((mem_dataspace >= 0), ""); + + for(k=0 ; k<NUM_DSETS; k++) { + sprintf(dset_name, "dset%d", k); + dset_id = H5Dopen2(grp_id, dset_name, H5P_DEFAULT); + VRFY((dset_id >= 0), ""); + + sid = H5Dget_space(dset_id); + VRFY((dset_id >= 0), "H5Dget_space succeeded"); + + ndims = H5Sget_simple_extent_dims(sid, dims, NULL); + VRFY((ndims == 2), "H5Sget_simple_extent_dims succeeded"); + VRFY(dims[0] == ROW_FACTOR*mpi_size, "Wrong dataset dimensions"); + VRFY(dims[1] == COL_FACTOR*mpi_size, "Wrong dataset dimensions"); + + ret = H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + ret = H5Dread(dset_id, H5T_NATIVE_INT, mem_dataspace, sid, H5P_DEFAULT, data_array); + VRFY((ret >= 0), ""); + + ret = H5Dclose(dset_id); + VRFY((ret >= 0), ""); + ret = H5Sclose(sid); + VRFY((ret == 0), ""); + + for (i=0; i < num_elements; i++) + VRFY((data_array[i] == mpi_rank+1), "Dataset Verify failed"); + } + + grp_id2 = H5Gcreate2(file_id, "GROUP/GROUP2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((grp_id2 >= 0), ""); + ret = H5Gclose(grp_id2); + VRFY((ret == 0), ""); + + ret = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((ret == 0), ""); + + MPI_Barrier(MPI_COMM_WORLD); + /* flush invalidate each ring, starting from the outermost ring and + * working inward. + */ + for ( i = 0; i < H5C__HASH_TABLE_LEN; i++ ) { + H5C_cache_entry_t * entry_ptr = NULL; + + entry_ptr = cache_ptr->index[i]; + + while ( entry_ptr != NULL ) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(entry_ptr->is_dirty == FALSE); + + if(!entry_ptr->is_pinned && !entry_ptr->is_protected) { + ret = H5AC_expunge_entry(f, H5AC_ind_read_dxpl_id, entry_ptr->type, entry_ptr->addr, 0); + VRFY((ret == 0), ""); + } + + entry_ptr = entry_ptr->ht_next; + } + } + MPI_Barrier(MPI_COMM_WORLD); + + grp_id2 = H5Gopen2(file_id, "GROUP/GROUP2", H5P_DEFAULT); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((grp_id2 >= 0), ""); + ret = H5Gclose(grp_id2); + H5Eprint2(H5E_DEFAULT, stderr); + VRFY((ret == 0), ""); + + ret = H5Gclose(grp_id); + VRFY((ret == 0), ""); + ret = H5Fclose(file_id); + VRFY((ret == 0), ""); + ret = H5Sclose(mem_dataspace); + VRFY((ret == 0), ""); + HDfree(data_array); + + return nerrors; +} + +void test_file_properties(void) { hid_t fid; /* HDF5 file ID */ @@ -103,7 +715,6 @@ test_file_properties(void) const char *filename; MPI_Comm comm = MPI_COMM_WORLD; MPI_Info info = MPI_INFO_NULL; - int mpi_size, mpi_rank; herr_t ret; /* Generic return value */ filename = (const char *)GetTestParameters(); diff --git a/testpar/t_mdset.c b/testpar/t_mdset.c index 7077081..39cfbbc 100644 --- a/testpar/t_mdset.c +++ b/testpar/t_mdset.c @@ -510,7 +510,7 @@ void big_dataset(void) /* Check that file of the correct size was created */ file_size = h5_get_file_size(filename, fapl); - VRFY((file_size == 2147485792ULL), "File is correct size(~2GB)"); + VRFY((file_size == 2147485696ULL), "File is correct size(~2GB)"); /* * Create >4GB HDF5 file @@ -539,7 +539,7 @@ void big_dataset(void) /* Check that file of the correct size was created */ file_size = h5_get_file_size(filename, fapl); - VRFY((file_size == 4294969440ULL), "File is correct size(~4GB)"); + VRFY((file_size == 4294969344ULL), "File is correct size(~4GB)"); /* * Create >8GB HDF5 file @@ -568,7 +568,7 @@ void big_dataset(void) /* Check that file of the correct size was created */ file_size = h5_get_file_size(filename, fapl); - VRFY((file_size == 8589936736ULL), "File is correct size(~8GB)"); + VRFY((file_size == 8589936640ULL), "File is correct size(~8GB)"); /* Close fapl */ ret = H5Pclose(fapl); diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index c54cb5e..a58452e 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -351,6 +351,11 @@ int main(int argc, char **argv) AddTest("split", test_split_comm_access, NULL, "dataset using split communicators", PARATESTFILE); +#ifdef PB_OUT /* temporary: disable page buffering when parallel */ + AddTest("page_buffer", test_page_buffer_access, NULL, + "page buffer usage in parallel", PARATESTFILE); +#endif + AddTest("props", test_file_properties, NULL, "Coll Metadata file property settings", PARATESTFILE); diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index 9838673..0cf5857 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -246,6 +246,7 @@ void collective_group_write(void); void independent_group_read(void); void test_fapl_mpio_dup(void); void test_split_comm_access(void); +void test_page_buffer_access(void); void dataset_atomicity(void); void dataset_writeInd(void); void dataset_writeAll(void); diff --git a/tools/src/h5dump/h5dump_ddl.c b/tools/src/h5dump/h5dump_ddl.c index 182d570..c78f325 100644 --- a/tools/src/h5dump/h5dump_ddl.c +++ b/tools/src/h5dump/h5dump_ddl.c @@ -1186,9 +1186,11 @@ dump_fcpl(hid_t fid) hsize_t userblock; /* userblock size retrieved from FCPL */ size_t off_size; /* size of offsets in the file */ size_t len_size; /* size of lengths in the file */ - H5F_file_space_type_t fs_strategy; /* file space strategy */ - hsize_t fs_threshold; /* free-space section threshold */ - H5F_info2_t finfo; /* file information */ + H5F_fspace_strategy_t fs_strategy; /* file space strategy */ + hbool_t fs_persist; /* Persisting free-space or not */ + hsize_t fs_threshold; /* free-space section threshold */ + hsize_t fsp_size; /* file space page size */ + H5F_info2_t finfo; /* file information */ #ifdef SHOW_FILE_DRIVER hid_t fapl; /* file access property list ID */ hid_t fdriver; /* file driver */ @@ -1204,7 +1206,8 @@ dump_fcpl(hid_t fid) H5Pget_sizes(fcpl,&off_size,&len_size); H5Pget_sym_k(fcpl,&sym_ik,&sym_lk); H5Pget_istore_k(fcpl,&istore_ik); - H5Pget_file_space(fcpl, &fs_strategy, &fs_threshold); + H5Pget_file_space_strategy(fcpl, &fs_strategy, &fs_persist, &fs_threshold); + H5Pget_file_space_page_size(fcpl, &fsp_size); H5Pclose(fcpl); #ifdef SHOW_FILE_DRIVER fapl=h5_fileaccess(); @@ -1265,18 +1268,22 @@ dump_fcpl(hid_t fid) PRINTSTREAM(rawoutstream, "%s %u\n","ISTORE_K", istore_ik); indentation(dump_indent + COL); - if(fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_ALL_PERSIST"); - } else if(fs_strategy == H5F_FILE_SPACE_ALL) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_ALL"); - } else if(fs_strategy == H5F_FILE_SPACE_AGGR_VFD) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_AGGR_VFD"); - } else if(fs_strategy == H5F_FILE_SPACE_VFD) { - PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FILE_SPACE_VFD"); + if(fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_FSM_AGGR"); + } else if(fs_strategy == H5F_FSPACE_STRATEGY_PAGE) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_PAGE"); + } else if(fs_strategy == H5F_FSPACE_STRATEGY_AGGR) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_AGGR"); + } else if(fs_strategy == H5F_FSPACE_STRATEGY_NONE) { + PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "H5F_FSPACE_STRATEGY_NONE"); } else PRINTSTREAM(rawoutstream, "%s %s\n", "FILE_SPACE_STRATEGY", "Unknown strategy"); indentation(dump_indent + COL); - PRINTSTREAM(rawoutstream, "%s %Hu\n","FREE_SPACE_THRESHOLD", fs_threshold); + PRINTSTREAM(rawoutstream, "%s %s\n","FREE_SPACE_PERSIST", fs_persist ? "TRUE" : "FALSE"); + indentation(dump_indent + COL); + PRINTSTREAM(rawoutstream, "%s %Hu\n","FREE_SPACE_SECTION_THRESHOLD", fs_threshold); + indentation(dump_indent + COL); + PRINTSTREAM(rawoutstream, "%s %Hu\n","FILE_SPACE_PAGE_SIZE", fsp_size); /*------------------------------------------------------------------------- * USER_BLOCK diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index c79db1c..c09dee6 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -78,8 +78,7 @@ int h5repack(const char* infile, const char* outfile, pack_opt_t *options) { *------------------------------------------------------------------------- */ int -h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, - H5F_file_space_type_t strategy, hsize_t threshold) +h5repack_init(pack_opt_t *options, int verbose, hbool_t latest) { int k, n; @@ -96,9 +95,6 @@ h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, options->filter_g[n].cd_values[k] = 0; } - options->fs_strategy = strategy; - options->fs_threshold = threshold; - return (options_table_init(&(options->op_tbl))); } diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h index d2ab923..5ad0b80 100644 --- a/tools/src/h5repack/h5repack.h +++ b/tools/src/h5repack/h5repack.h @@ -30,6 +30,13 @@ #define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */ #define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */ +/* File space default information */ +#define FS_PAGESIZE_DEF 4096 +#define FS_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR +#define FS_PERSIST_DEF FALSE +#define FS_THRESHOLD_DEF 1 + + /*------------------------------------------------------------------------- * data structures for command line options *------------------------------------------------------------------------- @@ -115,8 +122,10 @@ typedef struct { hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ hsize_t threshold; /* alignment threshold for H5Pset_alignment */ hsize_t alignment; /* alignment for H5Pset_alignment */ - H5F_file_space_type_t fs_strategy; /* File space handling strategy */ - hsize_t fs_threshold; /* Free space section threshold */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ + int fs_persist; /* Free space section threshold */ + long fs_threshold; /* Free space section threshold */ + long long fs_pagesize; /* File space page size */ } pack_opt_t; @@ -138,8 +147,7 @@ extern "C" { int h5repack(const char* infile, const char* outfile, pack_opt_t *options); int h5repack_addfilter(const char* str, pack_opt_t *options); int h5repack_addlayout(const char* str, pack_opt_t *options); -int h5repack_init(pack_opt_t *options, int verbose, hbool_t latest, - H5F_file_space_type_t strategy, hsize_t threshold); +int h5repack_init(pack_opt_t *options, int verbose, hbool_t latest); int h5repack_end(pack_opt_t *options); int h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options); int h5repack_cmp_pl(const char *fname1, const char *fname2); diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 1519053..45d52bf 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -96,6 +96,14 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) hsize_t ub_size = 0; /* size of user block */ hid_t fcpl = H5P_DEFAULT; /* file creation property list ID */ hid_t fapl = H5P_DEFAULT; /* file access property list ID */ + H5F_fspace_strategy_t set_strategy; /* Strategy to be set in outupt file */ + hbool_t set_persist; /* Persist free-space status to be set in output file */ + hsize_t set_threshold; /* Free-space section threshold to be set in output file */ + hsize_t set_pagesize; /* File space page size to be set in output file */ + H5F_fspace_strategy_t in_strategy; /* Strategy from input file */ + hbool_t in_persist; /* Persist free-space status from input file */ + hsize_t in_threshold; /* Free-space section threshold from input file */ + hsize_t in_pagesize; /* File space page size from input file */ /*------------------------------------------------------------------------- * open input file @@ -106,7 +114,7 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } - /* get user block size and file space strategy/threshold */ + /* get user block size and file space strategy/persist/threshold */ { hid_t fcpl_in; /* file creation property list ID for input file */ @@ -120,19 +128,17 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } - if (!options->fs_strategy) { - if (H5Pget_file_space(fcpl_in, &options->fs_strategy, NULL) < 0) { - error_msg("failed to retrieve file space strategy\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); - } + /* If the -S option is not set, get "strategy" from the input file */ + if(H5Pget_file_space_strategy(fcpl_in, &in_strategy, &in_persist, &in_threshold) < 0) { + error_msg("failed to retrieve file space strategy\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } - if (!options->fs_threshold) { - if (H5Pget_file_space(fcpl_in, NULL, &options->fs_threshold) < 0) { - error_msg("failed to retrieve file space threshold\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); - } - } + /* If the -G option is not set, get "pagesize" from the input file */ + if(H5Pget_file_space_page_size(fcpl_in, &in_pagesize) < 0) { + error_msg("failed to retrieve file space threshold\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } if (H5Pclose(fcpl_in) < 0) { error_msg("failed to close property list\n"); @@ -291,9 +297,9 @@ print_user_block(fnamein, fidin); } /*------------------------------------------------------------------------- - * set free-space strategy options - *------------------------------------------------------------------------- - */ + * Set file space information + *------------------------------------------------------------------------- + */ /* either use the FCPL already created or create a new one */ if (fcpl == H5P_DEFAULT) { @@ -304,12 +310,44 @@ print_user_block(fnamein, fidin); } } - /* set file space strategy and free space threshold */ - if (H5Pset_file_space(fcpl, options->fs_strategy, options->fs_threshold) < 0) { - error_msg("failed to set file space strategy & threshold \n"); + /* Set file space info to those from input file */ + set_strategy = in_strategy; + set_persist = in_persist; + set_threshold = in_threshold; + set_pagesize = in_pagesize; + + if(options->fs_strategy == (H5F_fspace_strategy_t)-1) /* A default strategy is specified by user */ + set_strategy = FS_STRATEGY_DEF; + else if(options->fs_strategy != (H5F_fspace_strategy_t)0) /* Set strategy as specified by user */ + set_strategy = options->fs_strategy; + + if(options->fs_persist == -1) /* A default "persist" is specified by user */ + set_persist = FS_PERSIST_DEF; + else if(options->fs_persist != 0) /* Set "persist" as specified by user */ + set_persist = (hbool_t)options->fs_persist; + + if(options->fs_threshold == -1) /* A "0" threshold is specified by user */ + set_threshold = (hsize_t)0; + else if(options->fs_threshold != 0) /* Set threshold as specified by user */ + set_threshold = (hsize_t)options->fs_threshold; + + /* Set file space information as specified */ + if(H5Pset_file_space_strategy(fcpl, set_strategy, set_persist, set_threshold) < 0) { + error_msg("failed to set file space strategy\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); } + if(options->fs_pagesize == -1) /* A "0" file space page size is specified by user */ + set_pagesize = (hsize_t)0; + else if(options->fs_pagesize != 0) /* Set file space page size as specified by user */ + set_pagesize = (hsize_t)options->fs_pagesize; + + if(set_pagesize != FS_PAGESIZE_DEF) /* Set non-default file space page size as specified */ + if(H5Pset_file_space_page_size(fcpl, set_pagesize) < 0) { + error_msg("failed to set file space page size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed"); + } + /*------------------------------------------------------------------------- * create the output file *------------------------------------------------------------------------- diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index 1807056..a8b4cfe 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -33,7 +33,7 @@ const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:S:T:E"; +static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:S:P:T:G:E"; static struct long_options l_opts[] = { { "help", no_arg, 'h' }, { "version", no_arg, 'V' }, @@ -55,7 +55,9 @@ static struct long_options l_opts[] = { { "infile", require_arg, 'i' }, /* -i for backward compability */ { "outfile", require_arg, 'o' }, /* -o for backward compability */ { "fs_strategy", require_arg, 'S' }, + { "fs_persist", require_arg, 'P' }, { "fs_threshold", require_arg, 'T' }, + { "fs_pagesize", require_arg, 'G' }, { "enable-error-stack", no_arg, 'E' }, { NULL, 0, '\0' } }; @@ -92,8 +94,10 @@ static void usage(const char *prog) { PRINTVALSTREAM(rawoutstream, " -a A, --alignment=A Alignment value for H5Pset_alignment\n"); PRINTVALSTREAM(rawoutstream, " -f FILT, --filter=FILT Filter type\n"); PRINTVALSTREAM(rawoutstream, " -l LAYT, --layout=LAYT Layout type\n"); - PRINTVALSTREAM(rawoutstream, " -S FS_STRGY, --fs_strategy=FS_STRGY File space management strategy\n"); - PRINTVALSTREAM(rawoutstream, " -T FS_THRD, --fs_threshold=FS_THRD Free-space section threshold\n"); + PRINTVALSTREAM(rawoutstream, " -S FS_STRATEGY, --fs_strategy=FS_STRATEGY File space management strategy for H5Pset_file_space_strategy\n"); + PRINTVALSTREAM(rawoutstream, " -P FS_PERSIST, --fs_persist=FS_PERSIST Persisting or not persisting free-space for H5Pset_file_space_strategy\n"); + PRINTVALSTREAM(rawoutstream, " -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy\n"); + PRINTVALSTREAM(rawoutstream, " -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size\n"); PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " M - is an integer greater than 1, size of dataset in bytes (default is 0) \n"); PRINTVALSTREAM(rawoutstream, " E - is a filename.\n"); @@ -109,18 +113,27 @@ static void usage(const char *prog) { PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they\n"); PRINTVALSTREAM(rawoutstream, " occur.\n"); PRINTVALSTREAM(rawoutstream, "\n"); - PRINTVALSTREAM(rawoutstream, " FS_STRGY is the file space management strategy to use for the output file.\n"); - PRINTVALSTREAM(rawoutstream, " It is a string as listed below:\n"); - PRINTVALSTREAM(rawoutstream, " ALL_PERSIST - Use persistent free-space managers, aggregators and virtual file driver\n"); - PRINTVALSTREAM(rawoutstream, " for file space allocation\n"); - PRINTVALSTREAM(rawoutstream, " ALL - Use non-persistent free-space managers, aggregators and virtual file driver\n"); - PRINTVALSTREAM(rawoutstream, " for file space allocation\n"); - PRINTVALSTREAM(rawoutstream, " AGGR_VFD - Use aggregators and virtual file driver for file space allocation\n"); - PRINTVALSTREAM(rawoutstream, " VFD - Use virtual file driver for file space allocation\n"); + PRINTVALSTREAM(rawoutstream, " FS_STRATEGY is a string indicating the file space strategy used:\n"); + PRINTVALSTREAM(rawoutstream, " FSM_AGGR:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are free-space managers, aggregators and virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " PAGE:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are free-space managers with embedded paged aggregation and virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " AGGR:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are aggregators and virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " NONE:\n"); + PRINTVALSTREAM(rawoutstream, " The mechanisms used in managing file space are virtual file driver.\n"); + PRINTVALSTREAM(rawoutstream, " The default strategy when not set is FSM_AGGR without persisting free-space.\n"); PRINTVALSTREAM(rawoutstream, "\n"); - PRINTVALSTREAM(rawoutstream, " FS_THRD is the free-space section threshold to use for the output file.\n"); - PRINTVALSTREAM(rawoutstream, " It is the minimum size (in bytes) of free-space sections to be tracked\n"); - PRINTVALSTREAM(rawoutstream, " by the the library's free-space managers.\n"); + PRINTVALSTREAM(rawoutstream, " FS_PERSIST is 1 to persisting free-space or 0 to not persisting free-space.\n"); + PRINTVALSTREAM(rawoutstream, " The default when not set is not persisting free-space.\n"); + PRINTVALSTREAM(rawoutstream, " The value is ignored for AGGR and NONE strategies.\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " FS_THRESHOLD is the minimum size (in bytes) of free-space sections to be tracked by the library.\n"); + PRINTVALSTREAM(rawoutstream, " The default when not set is 1.\n"); + PRINTVALSTREAM(rawoutstream, " The value is ignored for AGGR and NONE strategies.\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, " FS_PAGESIZE is the size (in bytes) >=512 that is used by the library when the file space strategy PAGE is used.\n"); + PRINTVALSTREAM(rawoutstream, " The default when not set is 4096.\n"); PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " FILT - is a string with the format:\n"); PRINTVALSTREAM(rawoutstream, "\n"); @@ -521,25 +534,45 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options) char strategy[MAX_NC_NAME]; HDstrcpy(strategy, opt_arg); - if(!HDstrcmp(strategy, "ALL_PERSIST")) - options->fs_strategy = H5F_FILE_SPACE_ALL_PERSIST; - else if(!HDstrcmp(strategy, "ALL")) - options->fs_strategy = H5F_FILE_SPACE_ALL; - else if(!HDstrcmp(strategy, "AGGR_VFD")) - options->fs_strategy = H5F_FILE_SPACE_AGGR_VFD; - else if(!HDstrcmp(strategy, "VFD")) - options->fs_strategy = H5F_FILE_SPACE_VFD; + if(!HDstrcmp(strategy, "FSM_AGGR")) + options->fs_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + else if(!HDstrcmp(strategy, "PAGE")) + options->fs_strategy = H5F_FSPACE_STRATEGY_PAGE; + else if(!HDstrcmp(strategy, "AGGR")) + options->fs_strategy = H5F_FSPACE_STRATEGY_AGGR; + else if(!HDstrcmp(strategy, "NONE")) + options->fs_strategy = H5F_FSPACE_STRATEGY_NONE; else { error_msg("invalid file space management strategy\n", opt_arg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; } + if(options->fs_strategy == (H5F_fspace_strategy_t)0) + /* To distinguish the "specified" zero value */ + options->fs_strategy = (H5F_fspace_strategy_t)-1; } break; + case 'P': + options->fs_persist = HDatoi( opt_arg ); + if(options->fs_persist == 0) + /* To distinguish the "specified" zero value */ + options->fs_persist = -1; + break; + case 'T': - options->fs_threshold = (hsize_t) HDatol( opt_arg ); + options->fs_threshold = HDatol( opt_arg ); + if(options->fs_threshold == 0) + /* To distinguish the "specified" zero value */ + options->fs_threshold = -1; + break; + + case 'G': + options->fs_pagesize = HDatoll( opt_arg ); + if(options->fs_pagesize == 0) + /* To distinguish the "specified" zero value */ + options->fs_pagesize = -1; break; case 'E': @@ -611,7 +644,7 @@ int main(int argc, const char **argv) } /* initialize options */ - h5repack_init(&options, 0, FALSE, H5F_FILE_SPACE_DEFAULT, (hsize_t) 0); + h5repack_init(&options, 0, FALSE); if (parse_command_line(argc, argv, &options) < 0) goto done; diff --git a/tools/src/h5repack/h5repack_verify.c b/tools/src/h5repack/h5repack_verify.c index 58dea03..1022d7a 100644 --- a/tools/src/h5repack/h5repack_verify.c +++ b/tools/src/h5repack/h5repack_verify.c @@ -58,8 +58,10 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options int ok = 1; hid_t fcpl_in = -1; /* file creation property for input file */ hid_t fcpl_out = -1; /* file creation property for output file */ - H5F_file_space_type_t in_strat, out_strat; /* file space handling strategy for in/output file */ - hsize_t in_thresh, out_thresh; /* free space section threshold for in/output file */ + H5F_fspace_strategy_t in_strategy, out_strategy; /* file space handling strategy for in/output file */ + hbool_t in_persist, out_persist; /* free-space persist status for in/output file */ + hsize_t in_threshold, out_threshold; /* free-space section threshold for in/output file */ + hsize_t in_pagesize, out_pagesize; /* file space page size for input/output file */ /* open the output file */ if((fidout = H5Fopen(out_fname, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0 ) @@ -196,8 +198,7 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options } /*------------------------------------------------------------------------- - * Verify that file space strategy and free space threshold - * are set as expected + * Verify that file space info are set as expected *------------------------------------------------------------------------- */ @@ -211,12 +212,18 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); } - /* Get file space management info for input file */ - if(H5Pget_file_space(fcpl_in, &in_strat, &in_thresh) < 0) { + /* Get file space info for input file */ + if(H5Pget_file_space_strategy(fcpl_in, &in_strategy, &in_persist, &in_threshold) < 0) { error_msg("failed to retrieve file space strategy & threshold\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_file_space failed"); } + /* Get file space page size for input file */ + if(H5Pget_file_space_page_size(fcpl_in, &in_pagesize) < 0) { + error_msg("failed to retrieve file space page size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); + } + /* Output file is already opened */ /* Get file creation property list for output file */ if((fcpl_out = H5Fget_create_plist(fidout)) < 0) { @@ -224,45 +231,84 @@ h5repack_verify(const char *in_fname, const char *out_fname, pack_opt_t *options HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); } - /* Get file space management info for output file */ - if(H5Pget_file_space(fcpl_out, &out_strat, &out_thresh) < 0) { + /* Get file space info for output file */ + if(H5Pget_file_space_strategy(fcpl_out, &out_strategy, &out_persist, &out_threshold) < 0) { error_msg("failed to retrieve file space strategy & threshold\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_file_space failed"); } + /* Get file space page size for output file */ + if(H5Pget_file_space_page_size(fcpl_out, &out_pagesize) < 0) { + error_msg("failed to retrieve file space page size\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fget_create_plist failed"); + } + /* - * If the strategy option is not set, - * file space handling strategy should be the same for both - * input & output files. - * If the strategy option is set, - * the output file's file space handling strategy should be the same - * as what is set via the strategy option + * If -S option is set, the file space handling strategy should be set as specified. + * If -S option is not set, the file space handling strategy should be + * the same as the input file's strategy. */ - if(!options->fs_strategy && out_strat != in_strat) { - error_msg("file space strategy not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); - + if(options->fs_strategy) { + if(out_strategy != (options->fs_strategy == (-1) ? 0 : options->fs_strategy)) { + error_msg("file space strategy not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { + if(out_strategy != in_strategy) { + error_msg("file space strategy not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } - else if(options->fs_strategy && out_strat!= options->fs_strategy) { - error_msg("file space strategy not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + + /* + * If the -P option is set, the free-space persist status should be set as specified. + * If the -P option is not set, the free-space persist status should be + * the same as the input file's free-space persist status + */ + if(options->fs_persist) { + if(out_persist != (hbool_t)(options->fs_persist == (-1) ? FALSE : options->fs_persist)) { + error_msg("free-space persist status is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { + if(out_persist != in_persist) { + error_msg("free-space persist status is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } /* - * If the threshold option is not set, - * the free space section threshold should be the same for both - * input & output files. - * If the threshold option is set, - * the output file's free space section threshold should be the same - * as what is set via the threshold option. + * If the -T option is set, the threshold size should be set as specified. + * If the -T option is not set, the threshold should be the same as the + * input file's threshold size. */ - if(!options->fs_threshold && out_thresh != in_thresh) { - error_msg("free space threshold not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "free space threshold not set as unexpected"); + if(options->fs_threshold) { + if(out_threshold != (hsize_t)(options->fs_threshold == (-1) ? 0 : options->fs_threshold)) { + error_msg("threshold is not set as unexpectec\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { + if(out_threshold != in_threshold) { + error_msg("threshold is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } - else if(options->fs_threshold && out_thresh != options->fs_threshold) { - error_msg("free space threshold not set as unexpected\n"); - HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "free space threshold not set as unexpected"); + + /* + * If the -G option is set, the file space page size should be set as specified. + * If the -G option is not set, the file space page size should be + * the same as the input file's file space page size. + */ + if(options->fs_pagesize) { + if(out_pagesize != (hsize_t)(options->fs_pagesize == (-1) ? 0 : options->fs_pagesize)) { + error_msg("file space page size is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } + } else { /* "-G" is not set */ + if(out_pagesize != in_pagesize) { + error_msg("file space page size is not set as unexpected\n"); + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file space strategy not set as unexpected"); + } } /* Closing */ diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index a331014..f99bab0 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -40,11 +40,11 @@ /* File space management strategies: see H5Fpublic.h for declarations */ const char *FS_STRATEGY_NAME[] = { + "H5F_FSPACE_STRATEGY_FSM_AGGR", + "H5F_FSPACE_STRATEGY_PAGE", + "H5F_FSPACE_STRATEGY_AGGR", + "H5F_FSPACE_STRATEGY_NONE", "unknown", - "H5F_FILE_SPACE_ALL_PERSIST", - "H5F_FILE_SPACE_ALL", - "H5F_FILE_SPACE_AGGR_VFD", - "H5F_FILE_SPACE_VFD", NULL }; @@ -106,10 +106,12 @@ typedef struct iter_t { hsize_t super_size; /* superblock size */ hsize_t super_ext_size; /* superblock extension size */ hsize_t ublk_size; /* user block size (if exists) */ - H5F_file_space_type_t fs_strategy; /* File space management strategy */ + H5F_fspace_strategy_t fs_strategy; /* File space management strategy */ + hbool_t fs_persist; /* Free-space persist or not */ hsize_t fs_threshold; /* Free-space section threshold */ - hsize_t free_space; /* amount of freespace in the file */ - hsize_t free_hdr; /* size of free space manager metadata in the file */ + hsize_t fsp_size; /* File space page size */ + hsize_t free_space; /* Amount of freespace in the file */ + hsize_t free_hdr; /* Size of free space manager metadata in the file */ unsigned long num_small_sects[SIZE_SMALL_SECTS]; /* Size of small free-space sections */ unsigned sect_nbins; /* Number of bins for free-space section sizes */ unsigned long *sect_bins; /* Pointer to array of bins for free-space section sizes */ @@ -1501,6 +1503,7 @@ print_freespace_info(const iter_t *iter) unsigned long total; /* Total count for various statistics */ unsigned u; /* Local index variable */ + HDfprintf(stdout, "Free-space persist: %s\n", iter->fs_persist ? "TRUE" : "FALSE"); HDfprintf(stdout, "Free-space section threshold: %Hu bytes\n", iter->fs_threshold); printf("Small size free-space sections (< %u bytes):\n", (unsigned)SIZE_SMALL_SECTS); total = 0; @@ -1551,6 +1554,7 @@ print_storage_summary(const iter_t *iter) double percent = 0.0f; HDfprintf(stdout, "File space management strategy: %s\n", FS_STRATEGY_NAME[iter->fs_strategy]); + HDfprintf(stdout, "File space page size: %Hu bytes\n", iter->fsp_size); printf("Summary of file space information:\n"); total_meta = iter->super_size + iter->super_ext_size + iter->ublk_size + @@ -1772,9 +1776,12 @@ main(int argc, const char *argv[]) if(H5Pget_userblock(fcpl, &iter.ublk_size) < 0) warn_msg("Unable to retrieve userblock size\n"); - if(H5Pget_file_space(fcpl, &iter.fs_strategy, &iter.fs_threshold) < 0) + if(H5Pget_file_space_strategy(fcpl, &iter.fs_strategy, &iter.fs_persist, &iter.fs_threshold) < 0) warn_msg("Unable to retrieve file space information\n"); - HDassert(iter.fs_strategy != 0 && iter.fs_strategy < H5F_FILE_SPACE_NTYPES); + HDassert(iter.fs_strategy >= 0 && iter.fs_strategy < H5F_FSPACE_STRATEGY_NTYPES); + + if(H5Pget_file_space_page_size(fcpl, &iter.fsp_size) < 0) + warn_msg("Unable to retrieve file space page size\n"); /* get information for free-space sections */ if(freespace_stats(fid, &iter) < 0) diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c index 481c6db..b6c6ea3 100644 --- a/tools/src/misc/h5clear.c +++ b/tools/src/misc/h5clear.c @@ -14,11 +14,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: - * - * Purpose: + * Purpose: A tool to clear the status_flags field from the file's superblock via -s option. + * A tool to remove cache image from the file via -m option. + * */ - #include "hdf5.h" #include "H5private.h" #include "h5tools.h" @@ -30,59 +29,171 @@ /* Make this private property (defined in H5Fprivate.h) available to h5clear. */ #define H5F_ACS_CLEAR_STATUS_FLAGS_NAME "clear_status_flags" +static char *fname_g = NULL; +static hbool_t clear_status_flags = FALSE; +static hbool_t remove_cache_image = FALSE; + +/* + * Command-line options: The user can specify short or long-named + * parameters. + */ +static const char *s_opts = "hVsm"; +static struct long_options l_opts[] = { + { "help", no_arg, 'h' }, + { "hel", no_arg, 'h'}, + { "he", no_arg, 'h'}, + { "version", no_arg, 'V' }, + { "version", no_arg, 'V' }, + { "versio", no_arg, 'V' }, + { "versi", no_arg, 'V' }, + { "vers", no_arg, 'V' }, + { "status", no_arg, 's' }, + { "statu", no_arg, 's' }, + { "stat", no_arg, 's' }, + { "sta", no_arg, 's' }, + { "st", no_arg, 's' }, + { "image", no_arg, 'm' }, + { "imag", no_arg, 'm' }, + { "ima", no_arg, 'm' }, + { "im", no_arg, 'm' }, + { NULL, 0, '\0' } +}; + + + /*------------------------------------------------------------------------- - * Function: leave + * Function: usage * - * Purpose: Close the tools library and exit + * Purpose: Prints a usage message * - * Return: Does not return + * Return: void * *------------------------------------------------------------------------- */ -static void -leave(int ret) +static void usage(const char *prog) { - h5tools_close(); - HDexit(ret); -} /* leave() */ + HDfprintf(stdout, "usage: %s [OPTIONS] file_name\n", prog); + HDfprintf(stdout, " OPTIONS\n"); + HDfprintf(stdout, " -h, --help Print a usage message and exit\n"); + HDfprintf(stdout, " -V, --version Print version number and exit\n"); + HDfprintf(stdout, " -s, --status Clear the status_flags field in the file's superblock\n"); + HDfprintf(stdout, " -m, --image Remove the metadata cache image from the file\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "Examples of use:\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "h5clear -s file_name\n"); + HDfprintf(stdout, " Clear the status_flags field in the superblock of the HDF5 file <file_name>.\n"); + HDfprintf(stdout, "\n"); + HDfprintf(stdout, "h5clear -m file_name\n"); + HDfprintf(stdout, " Remove the metadata cache image from the HDF5 file <file_name>.\n"); +} /* usage() */ + /*------------------------------------------------------------------------- - * Function: usage + * Function: parse_command_line * - * Purpose: Prints a usage message + * Purpose: Parses command line and sets up global variable to control output + * + * Return: Success: 0 * - * Return: void + * Failure: -1 * *------------------------------------------------------------------------- */ -static void -usage(void) +static int +parse_command_line(int argc, const char **argv) { - HDfprintf(stdout, "usage: h5clear filename\n"); + int opt; -} /* usage() */ + /* no arguments */ + if (argc == 1) { + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto error; + } + + /* parse command line options */ + while((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + switch((char)opt) { + case 'h': + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_SUCCESS); + goto done; + + case 'V': + print_version(h5tools_getprogname()); + h5tools_setstatus(EXIT_SUCCESS); + goto done; + + case 's': + clear_status_flags = TRUE; + break; + + case 'm': + remove_cache_image = TRUE; + break; + + default: + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto error; + } /* end switch */ + } /* end while */ + + /* check for file name to be processed */ + if(argc <= opt_ind) { + error_msg("missing file name\n"); + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto error; + } /* end if */ + + fname_g = HDstrdup(argv[opt_ind]); + +done: + return(0); + +error: + return -1; +} + +/*------------------------------------------------------------------------- + * Function: leave + * + * Purpose: Close the tools library and exit + * + * Return: Does not return + * + *------------------------------------------------------------------------- + */ +static void +leave(int ret) +{ + h5tools_close(); + HDexit(ret); +} /* leave() */ /*------------------------------------------------------------------------- * Function: main * - * Purpose: - * - * Return: Success: - * Failure: + * Purpose: To clear the status_flags field in the file's superblock (-s option). + * To remove the cache image from the file (-m option). * - * Programmer: + * Return: Success: 0 + * Failure: 1 * *------------------------------------------------------------------------- */ int -main (int argc, char *argv[]) +main (int argc, const char *argv[]) { - char *fname; /* File name */ - hbool_t clear = TRUE; /* To clear the status_flags in the file's superblock */ - hid_t fapl = -1; /* File access property list */ + char *fname = NULL; /* File name */ + hid_t fapl = -1; /* File access property list */ hid_t fid = -1; /* File ID */ + haddr_t image_addr; + hsize_t image_len; h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); @@ -93,46 +204,80 @@ main (int argc, char *argv[]) /* initialize h5tools lib */ h5tools_init(); - /* Check for the # of arguments */ - if(argc != 2) { - usage(); - leave(EXIT_FAILURE); + /* Parse command line options */ + if(parse_command_line(argc, argv) < 0) + goto done; + + if(fname_g == NULL) + goto done; + + if(!clear_status_flags && !remove_cache_image) { + usage(h5tools_getprogname()); + h5tools_setstatus(EXIT_FAILURE); + goto done; } + /* Duplicate the file name */ + fname = HDstrdup(fname_g); + /* Get a copy of the file access property list */ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { error_msg("H5Pcreate\n"); - leave(EXIT_FAILURE); - } + h5tools_setstatus(EXIT_FAILURE); + goto done; + } - /* Set to clear the status_flags in the file's superblock */ - /* This is a private property used by h5clear only */ - if(H5Pset(fapl, H5F_ACS_CLEAR_STATUS_FLAGS_NAME, &clear) < 0) { - error_msg("H5Pset\n"); - leave(EXIT_FAILURE); + /* -s option */ + if(clear_status_flags) { + /* Set to clear the status_flags in the file's superblock */ + /* This is a private property used by h5clear only */ + if(H5Pset(fapl, H5F_ACS_CLEAR_STATUS_FLAGS_NAME, &clear_status_flags) < 0) { + error_msg("H5Pset\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } } - /* Duplicate the file name */ - fname = HDstrdup(argv[opt_ind]); - if((fid = h5tools_fopen(fname, H5F_ACC_RDWR, fapl, NULL, NULL, (size_t)0)) < 0) { error_msg("h5tools_fopen\n"); - HDfree(fname); - leave(EXIT_FAILURE); + h5tools_setstatus(EXIT_FAILURE); + goto done; } - HDfree(fname); - /* Close the file */ - if(H5Fclose(fid) < 0) { - error_msg("H5Fclose\n"); - leave(EXIT_FAILURE); - } + /* -m option */ + if(remove_cache_image) { + H5AC_cache_image_config_t config; - /* CLose the property list */ - if(H5Pclose(fapl) < 0) { - error_msg("H5Pclose\n"); - leave(EXIT_FAILURE); - } + /* Retrieve cache image config */ + if((fapl = H5Fget_access_plist(fid)) < 0) { + error_msg("H5Fget_access_plist\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + config.version = H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION; + if(H5Pget_mdc_image_config(fapl, &config) < 0) { + error_msg("H5Pget_mdc_image_config\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + + /* Check for image */ + if(!config.generate_image) + warn_msg("No cache image in the file\n"); + } + + h5tools_setstatus(EXIT_SUCCESS); + +done: + if(fname) + HDfree(fname); + if(fname_g) + HDfree(fname_g); + + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Fclose(fid); + } H5E_END_TRY leave(h5tools_getstatus()); } /* main() */ diff --git a/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls b/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls index 0134714..af499a5 100644 --- a/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls +++ b/tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls @@ -8,7 +8,7 @@ Opened "./testfiles/h5copy_extlinks_src.out.h5" with sec2 driver. Storage: <details removed for portability> Type: 32-bit little-endian integer /copy1_group Group - Location: 1:4696 + Location: 1:4600 Links: 1 /copy1_group/extlink_datatype External Link {h5copy_extlinks_trg.h5//datatype} /copy1_group/extlink_dset External Link {h5copy_extlinks_trg.h5//simple} @@ -16,24 +16,24 @@ Opened "./testfiles/h5copy_extlinks_src.out.h5" with sec2 driver. /copy1_group/extlink_notyet1 External Link {h5copy_extlinks_trg.h5//notyet} /copy1_group/extlink_notyet2 External Link {notyet_file.h5//notyet} /copy2_dset Dataset {6/6} - Location: 1:4216 + Location: 1:4120 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /copy2_group Group - Location: 1:5128 + Location: 1:5032 Links: 1 /copy2_group/extlink_datatype Type - Location: 1:6328 + Location: 1:6232 Links: 1 - Type: shared-1:6328 32-bit little-endian integer + Type: shared-1:6232 32-bit little-endian integer /copy2_group/extlink_dset Dataset {6/6} - Location: 1:5496 + Location: 1:5400 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /copy2_group/extlink_grp Group - Location: 1:6288 + Location: 1:6192 Links: 1 /copy2_group/extlink_notyet1 External Link {h5copy_extlinks_trg.h5//notyet} /copy2_group/extlink_notyet2 External Link {notyet_file.h5//notyet} diff --git a/tools/test/h5copy/testfiles/h5copy_ref.out.ls b/tools/test/h5copy/testfiles/h5copy_ref.out.ls index d685af2..f5b31a7 100644 --- a/tools/test/h5copy/testfiles/h5copy_ref.out.ls +++ b/tools/test/h5copy/testfiles/h5copy_ref.out.ls @@ -11,21 +11,21 @@ Opened "./testfiles/h5copy_ref.out.h5" with sec2 driver. Storage: <details removed for portability> Type: 32-bit little-endian integer /COPY/Dset2 Dataset {3/3, 16/16} - Location: 1:1960 + Location: 1:4096 Links: 3 Storage: <details removed for portability> Type: 8-bit integer /COPY/Dset_OBJREF Dataset {2/2} - Location: 1:5184 + Location: 1:5224 Links: 1 Storage: <details removed for portability> Type: object reference /COPY/Dset_REGREF Dataset {2/2} - Location: 1:5304 + Location: 1:5344 Links: 1 Storage: <details removed for portability> Type: dataset region reference /COPY/Group Group - Location: 1:2096 + Location: 1:1960 Links: 3 -/~obj_pointed_by_2096 Group, same as /COPY/Group +/~obj_pointed_by_1960 Group, same as /COPY/Group diff --git a/tools/test/h5copy/testfiles/h5copytst.h5 b/tools/test/h5copy/testfiles/h5copytst.h5 Binary files differindex 1d1cbf1..474c3da 100644 --- a/tools/test/h5copy/testfiles/h5copytst.h5 +++ b/tools/test/h5copy/testfiles/h5copytst.h5 diff --git a/tools/test/h5copy/testfiles/h5copytst.out.ls b/tools/test/h5copy/testfiles/h5copytst.out.ls index 4044aaf..629915e 100644 --- a/tools/test/h5copy/testfiles/h5copytst.out.ls +++ b/tools/test/h5copy/testfiles/h5copytst.out.ls @@ -3,57 +3,57 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. Location: 1:96 Links: 1 /A Group - Location: 1:84304 + Location: 1:84208 Links: 1 /A/B1 Group - Location: 1:85008 + Location: 1:84912 Links: 1 /A/B1/simple Dataset {6/6} - Location: 1:84176 + Location: 1:84080 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /A/B2 Group - Location: 1:88544 + Location: 1:88448 Links: 1 /A/B2/simple2 Dataset {6/6} - Location: 1:88416 + Location: 1:88320 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /C Group - Location: 1:91752 + Location: 1:91656 Links: 1 /C/D Group - Location: 1:92456 + Location: 1:92360 Links: 1 /C/D/simple Dataset {6/6} - Location: 1:91624 + Location: 1:91528 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /E Group - Location: 1:106368 + Location: 1:106272 Links: 1 /E/F Group - Location: 1:107072 + Location: 1:106976 Links: 1 /E/F/grp_dsets Group - Location: 1:94568 + Location: 1:94472 Links: 1 /E/F/grp_dsets/chunk Dataset {6/6} - Location: 1:98752 + Location: 1:98656 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /E/F/grp_dsets/compact Dataset {6/6} - Location: 1:99208 + Location: 1:99112 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /E/F/grp_dsets/compound Dataset {2/2} - Location: 1:99344 + Location: 1:99248 Links: 1 Storage: <details removed for portability> Type: struct { @@ -61,60 +61,60 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /E/F/grp_dsets/compressed Dataset {6/6} - Location: 1:101656 + Location: 1:101560 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /E/F/grp_dsets/named_vl Dataset {2/2} - Location: 1:105920 + Location: 1:105824 Links: 1 Storage: <details removed for portability> - Type: shared-1:106048 variable length of + Type: shared-1:105952 variable length of 32-bit little-endian integer /E/F/grp_dsets/nested_vl Dataset {2/2} - Location: 1:106096 + Location: 1:106000 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /E/F/grp_dsets/simple Dataset {6/6} - Location: 1:106240 + Location: 1:106144 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /E/F/grp_dsets/vl Type - Location: 1:106048 + Location: 1:105952 Links: 2 - Type: shared-1:106048 variable length of + Type: shared-1:105952 variable length of 32-bit little-endian integer /G Group - Location: 1:122016 + Location: 1:121920 Links: 1 /G/H Group - Location: 1:122720 + Location: 1:122624 Links: 1 /G/H/grp_nested Group - Location: 1:109096 + Location: 1:109000 Links: 1 /G/H/grp_nested/grp_dsets Group - Location: 1:109888 + Location: 1:109792 Links: 1 /G/H/grp_nested/grp_dsets/chunk Dataset {6/6} - Location: 1:114072 + Location: 1:113976 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/compact Dataset {6/6} - Location: 1:114528 + Location: 1:114432 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/compound Dataset {2/2} - Location: 1:114664 + Location: 1:114568 Links: 1 Storage: <details removed for portability> Type: struct { @@ -122,48 +122,48 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /G/H/grp_nested/grp_dsets/compressed Dataset {6/6} - Location: 1:116976 + Location: 1:116880 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/named_vl Dataset {2/2} - Location: 1:121240 + Location: 1:121144 Links: 1 Storage: <details removed for portability> - Type: shared-1:121368 variable length of + Type: shared-1:121272 variable length of 32-bit little-endian integer /G/H/grp_nested/grp_dsets/nested_vl Dataset {2/2} - Location: 1:121416 + Location: 1:121320 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /G/H/grp_nested/grp_dsets/simple Dataset {6/6} - Location: 1:121560 + Location: 1:121464 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/vl Type - Location: 1:121368 + Location: 1:121272 Links: 2 - Type: shared-1:121368 variable length of + Type: shared-1:121272 variable length of 32-bit little-endian integer /chunk Dataset {6/6} - Location: 1:6312 + Location: 1:6216 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /compact Dataset {6/6} - Location: 1:6440 + Location: 1:6344 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /compound Dataset {2/2} - Location: 1:8624 + Location: 1:8528 Links: 1 Storage: <details removed for portability> Type: struct { @@ -171,28 +171,28 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /compressed Dataset {6/6} - Location: 1:12984 + Location: 1:12888 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_dsets Group - Location: 1:28128 + Location: 1:28032 Links: 1 /grp_dsets/chunk Dataset {6/6} - Location: 1:32312 + Location: 1:32216 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/compact Dataset {6/6} - Location: 1:32768 + Location: 1:32672 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/compound Dataset {2/2} - Location: 1:32904 + Location: 1:32808 Links: 1 Storage: <details removed for portability> Type: struct { @@ -200,62 +200,62 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_dsets/compressed Dataset {6/6} - Location: 1:35216 + Location: 1:35120 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_dsets/named_vl Dataset {2/2} - Location: 1:39480 + Location: 1:39384 Links: 1 Storage: <details removed for portability> - Type: shared-1:39608 variable length of + Type: shared-1:39512 variable length of 32-bit little-endian integer /grp_dsets/nested_vl Dataset {2/2} - Location: 1:39656 + Location: 1:39560 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_dsets/simple Dataset {6/6} - Location: 1:39800 + Location: 1:39704 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/simple_group Dataset {6/6} - Location: 1:55912 + Location: 1:55816 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_dsets/vl Type - Location: 1:39608 + Location: 1:39512 Links: 2 - Type: shared-1:39608 variable length of + Type: shared-1:39512 variable length of 32-bit little-endian integer /grp_empty Group - Location: 1:27336 + Location: 1:27240 Links: 1 /grp_nested Group - Location: 1:40592 + Location: 1:40496 Links: 1 /grp_nested/grp_dsets Group - Location: 1:41384 + Location: 1:41288 Links: 1 /grp_nested/grp_dsets/chunk Dataset {6/6} - Location: 1:45568 + Location: 1:45472 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_nested/grp_dsets/compact Dataset {6/6} - Location: 1:46024 + Location: 1:45928 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_nested/grp_dsets/compound Dataset {2/2} - Location: 1:46160 + Location: 1:46064 Links: 1 Storage: <details removed for portability> Type: struct { @@ -263,51 +263,51 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_nested/grp_dsets/compressed Dataset {6/6} - Location: 1:48472 + Location: 1:48376 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_nested/grp_dsets/named_vl Dataset {2/2} - Location: 1:52736 + Location: 1:52640 Links: 1 Storage: <details removed for portability> - Type: shared-1:52864 variable length of + Type: shared-1:52768 variable length of 32-bit little-endian integer /grp_nested/grp_dsets/nested_vl Dataset {2/2} - Location: 1:52912 + Location: 1:52816 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_nested/grp_dsets/simple Dataset {6/6} - Location: 1:53056 + Location: 1:52960 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_nested/grp_dsets/vl Type - Location: 1:52864 + Location: 1:52768 Links: 2 - Type: shared-1:52864 variable length of + Type: shared-1:52768 variable length of 32-bit little-endian integer /grp_rename Group - Location: 1:57120 + Location: 1:57024 Links: 1 /grp_rename/chunk Dataset {6/6} - Location: 1:61304 + Location: 1:61208 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/compact Dataset {6/6} - Location: 1:61760 + Location: 1:61664 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/compound Dataset {2/2} - Location: 1:61896 + Location: 1:61800 Links: 1 Storage: <details removed for portability> Type: struct { @@ -315,28 +315,28 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_rename/compressed Dataset {6/6} - Location: 1:64208 + Location: 1:64112 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_rename/grp_dsets Group - Location: 1:70000 + Location: 1:69904 Links: 1 /grp_rename/grp_dsets/chunk Dataset {6/6} - Location: 1:74184 + Location: 1:74088 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/grp_dsets/compact Dataset {6/6} - Location: 1:74640 + Location: 1:74544 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/grp_dsets/compound Dataset {2/2} - Location: 1:74776 + Location: 1:74680 Links: 1 Storage: <details removed for portability> Type: struct { @@ -344,73 +344,73 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_rename/grp_dsets/compressed Dataset {6/6} - Location: 1:77088 + Location: 1:76992 Links: 1 Chunks: {2} 8 bytes Storage: <details removed for portability> Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_rename/grp_dsets/named_vl Dataset {2/2} - Location: 1:81352 + Location: 1:81256 Links: 1 Storage: <details removed for portability> - Type: shared-1:81480 variable length of + Type: shared-1:81384 variable length of 32-bit little-endian integer /grp_rename/grp_dsets/nested_vl Dataset {2/2} - Location: 1:81528 + Location: 1:81432 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_rename/grp_dsets/simple Dataset {6/6} - Location: 1:81672 + Location: 1:81576 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/grp_dsets/vl Type - Location: 1:81480 + Location: 1:81384 Links: 2 - Type: shared-1:81480 variable length of + Type: shared-1:81384 variable length of 32-bit little-endian integer /grp_rename/named_vl Dataset {2/2} - Location: 1:68472 + Location: 1:68376 Links: 1 Storage: <details removed for portability> - Type: shared-1:68600 variable length of + Type: shared-1:68504 variable length of 32-bit little-endian integer /grp_rename/nested_vl Dataset {2/2} - Location: 1:68648 + Location: 1:68552 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /grp_rename/simple Dataset {6/6} - Location: 1:68792 + Location: 1:68696 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer /grp_rename/vl Type - Location: 1:68600 + Location: 1:68504 Links: 2 - Type: shared-1:68600 variable length of + Type: shared-1:68504 variable length of 32-bit little-endian integer /named_vl Dataset {2/2} - Location: 1:17280 + Location: 1:17184 Links: 1 Storage: <details removed for portability> - Type: shared-1:17408 variable length of + Type: shared-1:17312 variable length of 32-bit little-endian integer /nested_vl Dataset {2/2} - Location: 1:21760 + Location: 1:21664 Links: 1 Storage: <details removed for portability> Type: variable length of variable length of 32-bit little-endian integer /rename Dataset {2/2} - Location: 1:26128 + Location: 1:26032 Links: 1 Storage: <details removed for portability> Type: struct { @@ -423,7 +423,7 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. Storage: <details removed for portability> Type: 32-bit little-endian integer /simple_top Dataset {6/6} - Location: 1:23952 + Location: 1:23856 Links: 1 Storage: <details removed for portability> Type: 32-bit little-endian integer diff --git a/tools/test/h5diff/testfiles/h5diff_idx.txt b/tools/test/h5diff/testfiles/h5diff_idx.txt deleted file mode 100644 index 754d3ea..0000000 --- a/tools/test/h5diff/testfiles/h5diff_idx.txt +++ /dev/null @@ -1,14 +0,0 @@ - -file1 file2 ---------------------------------------- - x x / - x x /dset - x x /dset_filter - -group : </> and </> -0 differences found -dataset: </dset> and </dset> -0 differences found -dataset: </dset_filter> and </dset_filter> -0 differences found -EXIT CODE: 0 diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index bf9a698..afcf405 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -289,8 +289,9 @@ typedef struct s1_t { #define F64_DIM1 (F64_ARRAY_BUF_LEN / sizeof(int) + 1) /* File 65 macros */ -#define STRATEGY H5F_FILE_SPACE_AGGR_VFD /* File space handling strategy */ -#define THRESHOLD10 10 /* Free space section threshold */ +#define STRATEGY H5F_FSPACE_STRATEGY_NONE /* File space handling strategy */ +#define THRESHOLD10 10 /* Free-space section threshold */ +#define FSPACE_PAGE_SIZE 8192 /* File space page size */ /* "FILE66" macros and for FILE69 */ #define F66_XDIM 8 @@ -7042,8 +7043,9 @@ gent_extlinks(void) /*------------------------------------------------------------------------- * Function: gent_fs_strategy_threshold * - * Purpose: Generate a file with non-default file space strategy and - * non-default free-space section threshold. + * Purpose: Generate a file with non-default file space strategy, + * non-default free-space section threshold, + * non-default file space page size. *------------------------------------------------------------------------- */ static void @@ -7056,7 +7058,8 @@ gent_fs_strategy_threshold(void) fcpl = H5Pcreate(H5P_FILE_CREATE); /* Set file space information */ - H5Pset_file_space(fcpl, STRATEGY, (hsize_t)THRESHOLD10); + H5Pset_file_space_strategy(fcpl, STRATEGY, TRUE, (hsize_t)THRESHOLD10); + H5Pset_file_space_page_size(fcpl, (hsize_t)FSPACE_PAGE_SIZE); /* Create the file with the specified strategy and threshold */ fid = H5Fcreate(FILE65, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT); diff --git a/tools/test/h5format_convert/h5fc_gentest.c b/tools/test/h5format_convert/h5fc_gentest.c index 9ef8e6e..aea84f8 100644 --- a/tools/test/h5format_convert/h5fc_gentest.c +++ b/tools/test/h5format_convert/h5fc_gentest.c @@ -568,7 +568,7 @@ gen_ext(const char *fname, unsigned new_format, unsigned what) H5Pset_shared_mesg_nindexes(fcpl, 4); break; case 2: - H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0); + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1); break; case 3: H5Pset_istore_k(fcpl, ISTORE_IK); @@ -576,16 +576,16 @@ gen_ext(const char *fname, unsigned new_format, unsigned what) break; case 4: H5Pset_istore_k(fcpl, ISTORE_IK); - H5Pset_file_space(fcpl, H5F_FILE_SPACE_DEFAULT, (hsize_t)2); + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1); break; case 5: H5Pset_shared_mesg_nindexes(fcpl, 4); - H5Pset_file_space(fcpl, H5F_FILE_SPACE_VFD, (hsize_t)0); + H5Pset_file_space_page_size(fcpl, (hsize_t)512); break; case 6: H5Pset_istore_k(fcpl, ISTORE_IK); H5Pset_shared_mesg_nindexes(fcpl, 4); - H5Pset_file_space(fcpl, H5F_FILE_SPACE_AGGR_VFD, (hsize_t)0); + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1); break; default: break; diff --git a/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 b/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 Binary files differindex ac7dbd3..6f92057 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 b/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 Binary files differindex a10e8a4..d1aecc2 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_err_level.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 Binary files differindex 68ba2ac..5f2a01a 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 Binary files differindex 1a58089..df86faf 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 Binary files differindex 26e9b25..e41f51a 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 Binary files differindex e5c5e25..af2cb15 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 Binary files differindex 0e3eca7..5e99ddf 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 Binary files differindex cb15f03..8996bf0 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl +++ b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 Binary files differindex d46cef4..df1db97 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 b/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 Binary files differindex defbcb3..c498ead 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 diff --git a/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 b/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 Binary files differindex 58a340d..2f0f063 100644 --- a/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 +++ b/tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 Binary files differindex 3cbc7f4..b71b31e 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl index d1768c8..d9cc0b7 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 Binary files differindex a2c9187..6a8e5eb 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 Binary files differindex fdf4f33..ebc6919 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 Binary files differindex 6bf0a2f..a4178b3 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 Binary files differindex c0c7ecc..b5cd60a 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl index dae9284..db00a99 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 Binary files differindex 055cabf..8e63726 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl index 8ec4656..4be6d90 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 64 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 Binary files differindex f4caaf4..d581e3c 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 diff --git a/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 b/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 Binary files differindex d0bf344..0e02ca1 100644 --- a/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 +++ b/tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 diff --git a/tools/test/h5repack/h5repacktst.c b/tools/test/h5repack/h5repacktst.c index abdda08..69e576d 100644 --- a/tools/test/h5repack/h5repacktst.c +++ b/tools/test/h5repack/h5repacktst.c @@ -79,6 +79,20 @@ #define FNAME18 "h5repack_layout2.h5" +/* Files for testing file space paging */ +#define FSPACE_OUT "h5repack_fspace_OUT.h5" /* The output file */ +#define NELMTS(X) (sizeof(X)/sizeof(X[0])) /* # of elements */ +const char *H5REPACK_FSPACE_FNAMES[] = { + "h5repack_latest.h5", /* 0 */ + "h5repack_default.h5", /* 1 */ + "h5repack_page_persist.h5", /* 2 */ + "h5repack_fsm_aggr_persist.h5", /* 3 */ + "h5repack_page_threshold.h5", /* 4 */ + "h5repack_fsm_aggr_threshold.h5", /* 5 */ + "h5repack_aggr.h5", /* 6 */ + "h5repack_none.h5" /* 7 */ +}; + #define FNAME_UB "ublock.bin" /* obj and region references */ @@ -169,8 +183,10 @@ int main (void) { pack_opt_t pack_options; diff_opt_t diff_options; - hsize_t fs_size = 0; /* free space section threshold */ - H5F_file_space_type_t fs_type = H5F_FILE_SPACE_DEFAULT; /* file space handling strategy */ + + unsigned j; /* Local index variable for testing file space */ + const char *fname; /* File name for testing file space */ + h5_stat_t file_stat; h5_stat_size_t fsize1, fsize2; /* file sizes */ #if defined (H5_HAVE_FILTER_SZIP) @@ -203,8 +219,161 @@ int main (void) * 2) use the h5diff function to compare the input and output file *------------------------------------------------------------------------- */ + /*------------------------------------------------------------------------- + * Testing file space info setting + *------------------------------------------------------------------------- + */ + TESTING(" files with file space info setting--no options (-S, -P, -T, -G) are set"); + j = 0; /* #0 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + TESTING(" files with file space info setting--all options -S, -P, -T, -G are set"); + ++j; /* #1 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_NONE; + pack_options.fs_persist = -1; /* "FALSE" is set via -P 0 */ + pack_options.fs_threshold = 1; + pack_options.fs_pagesize = 8192; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + + TESTING(" files with file space info setting--options -S and -T are set"); + ++j; /* #2 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + pack_options.fs_strategy = -1; /* "FSM_AGGR" specified via -S FSM_AGGR */ + pack_options.fs_threshold = -1; /* "0" specified via -T 0 */ + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + + TESTING(" files with file space info setting-- options -S and -P are set & -L"); + ++j; /* #3 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_PAGE; /* "PAGE" specified via -S */ + pack_options.fs_persist = TRUE; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -P and -T are set & -L"); + ++j; /* #4 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_persist = -1; /* "FALSE" is set via -P 0 */ + pack_options.fs_threshold = 2; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -S and -G are set & -L"); + ++j; /* #5 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_PAGE; + pack_options.fs_pagesize = 8192; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -S, -P, -T, -G are set"); + ++j; /* #6 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, FALSE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_NONE; + pack_options.fs_persist = -1; /* "FALSE" is set via -P 0 */ + pack_options.fs_threshold = 1; + pack_options.fs_pagesize = 8192; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" files with file space info setting-- options -S, -T, -G are set & -L"); + ++j; /* #7 */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + fname = H5REPACK_FSPACE_FNAMES[j]; + if(h5repack_init(&pack_options, 0, TRUE) < 0) + GOERROR; + pack_options.fs_strategy = H5F_FSPACE_STRATEGY_AGGR; + pack_options.fs_threshold = 1; + pack_options.fs_pagesize = 4096; + if(h5repack(fname, FSPACE_OUT, &pack_options) < 0) + GOERROR; + if(h5diff(fname, FSPACE_OUT, NULL, NULL, &diff_options) > 0) + GOERROR; + if(h5repack_verify(fname, FSPACE_OUT, &pack_options)<=0) + GOERROR; + if(h5repack_end(&pack_options) < 0) + GOERROR; + PASSED(); + /*------------------------------------------------------------------------- * file with fill values *------------------------------------------------------------------------- @@ -212,8 +381,7 @@ int main (void) TESTING(" copy of datasets (fill values)"); - /* fs_type = 0; fs_size = 0 */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME0,FNAME0OUT,&pack_options) < 0) GOERROR; @@ -233,7 +401,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (all datatypes)"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME1,FNAME1OUT,&pack_options) < 0) GOERROR; @@ -253,7 +421,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (attributes)"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME2,FNAME2OUT,&pack_options) < 0) GOERROR; @@ -272,7 +440,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (hardlinks)"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME3,FNAME3OUT,&pack_options) < 0) GOERROR; @@ -292,7 +460,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of allocation early file"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME5,FNAME5OUT,&pack_options) < 0) GOERROR; @@ -322,7 +490,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset1:GZIP=9",&pack_options) < 0) GOERROR; @@ -349,7 +517,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, TRUE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, TRUE) < 0) GOERROR; if (h5repack_addfilter("dset1:GZIP=9",&pack_options) < 0) GOERROR; @@ -377,7 +545,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -415,7 +583,7 @@ int main (void) */ if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset2:SZIP=8,EC",&pack_options) < 0) GOERROR; @@ -447,7 +615,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -476,7 +644,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset1:SHUF",&pack_options) < 0) GOERROR; @@ -500,8 +668,7 @@ int main (void) TESTING(" addding shuffle filter to all"); - /* fs_type = H5F_FILE_SPACE_ALL_PERSIST; fs_size = 1 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("SHUF",&pack_options) < 0) GOERROR; @@ -526,7 +693,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset1:FLET",&pack_options) < 0) GOERROR; @@ -551,7 +718,7 @@ int main (void) TESTING(" adding checksum filter to all"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("FLET",&pack_options) < 0) GOERROR; @@ -576,7 +743,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options) < 0) GOERROR; @@ -616,7 +783,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -638,7 +805,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, TRUE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, TRUE) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -658,7 +825,7 @@ int main (void) */ TESTING(" adding layout chunked to all"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -679,7 +846,7 @@ int main (void) * test an individual object option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:CONTI",&pack_options) < 0) GOERROR; @@ -700,7 +867,7 @@ int main (void) * test all objects option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -717,7 +884,7 @@ int main (void) * do the same test for a file with filters (chunked) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -739,7 +906,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset1:COMPA",&pack_options) < 0) GOERROR; @@ -760,8 +927,7 @@ int main (void) *------------------------------------------------------------------------- */ - /* fs_type = H5F_FILE_SPACE_ALL; fs_size = 2 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("COMPA",&pack_options) < 0) GOERROR; @@ -783,7 +949,7 @@ int main (void) * layout compact to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CONTI",&pack_options) < 0) GOERROR; @@ -803,7 +969,7 @@ int main (void) * layout compact to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CHUNK=2x5",&pack_options) < 0) GOERROR; @@ -823,7 +989,7 @@ int main (void) * layout compact to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_compact:COMPA",&pack_options) < 0) GOERROR; @@ -842,7 +1008,7 @@ int main (void) * layout contiguous to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:COMPA",&pack_options) < 0) GOERROR; @@ -861,7 +1027,7 @@ int main (void) * layout contiguous to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CHUNK=3x6",&pack_options) < 0) GOERROR; @@ -881,7 +1047,7 @@ int main (void) * layout contiguous to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CONTI",&pack_options) < 0) GOERROR; @@ -900,7 +1066,7 @@ int main (void) * layout chunked to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:COMPA",&pack_options) < 0) GOERROR; @@ -920,7 +1086,7 @@ int main (void) * layout chunked to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CONTI",&pack_options) < 0) GOERROR; @@ -939,7 +1105,7 @@ int main (void) * layout chunked to chunked conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CHUNK=18x13",&pack_options) < 0) GOERROR; @@ -969,8 +1135,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - /* fs_type = H5F_FILE_SPACE_AGGR_VFD; fs_size = 3 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME7,FNAME7OUT,&pack_options) < 0) GOERROR; @@ -995,7 +1160,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_szip:NONE",&pack_options) < 0) GOERROR; @@ -1020,7 +1185,7 @@ int main (void) TESTING(" copy of deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME8,FNAME8OUT,&pack_options) < 0) GOERROR; @@ -1040,7 +1205,7 @@ int main (void) TESTING(" removing deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0, FALSE, fs_type, ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:NONE",&pack_options) < 0) GOERROR; @@ -1062,7 +1227,7 @@ int main (void) TESTING(" copy of shuffle filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME9,FNAME9OUT,&pack_options) < 0) GOERROR; @@ -1078,7 +1243,7 @@ int main (void) TESTING(" removing shuffle filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_shuffle:NONE",&pack_options) < 0) GOERROR; @@ -1096,7 +1261,7 @@ int main (void) TESTING(" copy of fletcher filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME10,FNAME10OUT,&pack_options) < 0) GOERROR; @@ -1112,7 +1277,7 @@ int main (void) TESTING(" removing fletcher filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_fletcher32:NONE",&pack_options) < 0) GOERROR; @@ -1130,7 +1295,7 @@ int main (void) TESTING(" copy of nbit filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME12,FNAME12OUT,&pack_options) < 0) GOERROR; @@ -1146,7 +1311,7 @@ int main (void) TESTING(" removing nbit filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_nbit:NONE",&pack_options) < 0) GOERROR; @@ -1164,7 +1329,7 @@ int main (void) TESTING(" adding nbit filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_int31:NBIT",&pack_options) < 0) GOERROR; @@ -1182,7 +1347,7 @@ int main (void) TESTING(" copy of scaleoffset filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME13,FNAME13OUT,&pack_options) < 0) GOERROR; @@ -1198,7 +1363,7 @@ int main (void) TESTING(" removing scaleoffset filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_scaleoffset:NONE",&pack_options) < 0) GOERROR; @@ -1216,7 +1381,7 @@ int main (void) TESTING(" adding scaleoffset filter"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_none:SOFF=31,IN",&pack_options) < 0) GOERROR; @@ -1248,8 +1413,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) if (szip_can_encode) { - /* fs_type = H5F_FILE_SPACE_VFD; fs_size = 4 */ - if (h5repack_init (&pack_options, 0, FALSE, H5_INC_ENUM(H5F_file_space_type_t, fs_type), ++fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -1275,7 +1439,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options) < 0) GOERROR; @@ -1306,7 +1470,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("NONE",&pack_options) < 0) GOERROR; @@ -1330,7 +1494,7 @@ int main (void) */ TESTING(" big file"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME14,FNAME14OUT,&pack_options) < 0) GOERROR; @@ -1347,7 +1511,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" external datasets"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack(FNAME15,FNAME15OUT,&pack_options) < 0) GOERROR; @@ -1364,7 +1528,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" file with userblock"); - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; if(h5repack(FNAME16, FNAME16OUT, &pack_options) < 0) GOERROR; @@ -1383,7 +1547,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" latest file format options"); - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; pack_options.latest=1; pack_options.grp_compact=10; @@ -1413,7 +1577,7 @@ int main (void) #if defined (H5_HAVE_FILTER_DEFLATE) - if (h5repack_init (&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if (h5repack_init (&pack_options, 0, FALSE) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -1442,7 +1606,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; /* add the options for a user block size and user block filename */ @@ -1475,7 +1639,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; /* add the options for aligment */ @@ -1529,7 +1693,7 @@ int main (void) */ TESTING(" file with committed datatypes"); - if(h5repack_init(&pack_options, 0, FALSE, fs_type, fs_size) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; if(h5repack(FNAME17, FNAME17OUT, &pack_options) < 0) @@ -1556,7 +1720,7 @@ int main (void) /* First run without metadata option. No need to verify the correctness */ /* since this has been verified by earlier tests. Just record the file */ /* size of the output file. */ - if(h5repack_init(&pack_options, 0, FALSE, H5F_FILE_SPACE_DEFAULT, (hsize_t)0) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; if(h5repack(FNAME4, FNAME4OUT, &pack_options) < 0) GOERROR; @@ -1567,7 +1731,7 @@ int main (void) GOERROR; /* run it again with metadata option */ - if(h5repack_init(&pack_options, 0, FALSE, H5F_FILE_SPACE_DEFAULT, (hsize_t)0) < 0) + if(h5repack_init(&pack_options, 0, FALSE) < 0) GOERROR; pack_options.meta_block_size = 8192; if(h5repack(FNAME4, FNAME4OUT, &pack_options) < 0) @@ -1625,6 +1789,9 @@ static int make_testfiles(void) { hid_t fid; + hid_t fcpl; /* File creation property list */ + hid_t fapl; /* File access property list */ + unsigned j; /* Local index variable */ /*------------------------------------------------------------------------- * create a file for general copy test @@ -1865,6 +2032,173 @@ int make_testfiles(void) if(H5Fclose(fid) < 0) return -1; + /*------------------------------------------------------------------------- + * create 8 files with combinations ??? + *------------------------------------------------------------------------- */ + + /* Create file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return -1; + + /* Set to use latest library format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + return -1; + + /* + * #0 -- h5repack_latest.h5 + * default: strategy=FSM_AGGR, persist=FALSE, threshold=1 + * default: inpage=4096 + */ + j = 0; + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[j], H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + + /* + * #1 -- h5repack_default.h5 + * default: strategy=FSM_AGGR, persist=FALSE, threshold=1 + * default: inpage=4096 + */ + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + + /* + * #2 -- h5repack_page_persist.h5 + * Setting: + * strategy=PAGE, persist=TRUE, threshold=1 + * inpage=512 + * latest format + */ + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)512) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + + /* + * #3 -- h5repack_fsm_aggr_persist.h5 + * Setting: + * strategy=FSM_AGGR, persist=TRUE, threshold=1 + * default: inpage=4096 + */ + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + + /* + * #4 -- h5repack_page_threshold.h5 + * Setting: + * strategy=PAGE, persist=FALSE, threshold=3 + * inpage=8192 + * latest format + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)3) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)8192) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + /* + * #5 -- h5repack_fsm_aggr_threshold.h5 + * Setting: + * strategy=FSM_AGGR, persist=FALSE, threshold=3 + * inpage=4096 + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, FALSE, (hsize_t)3) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)FS_PAGESIZE_DEF) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + /* + * #6 -- h5repack_aggr.h5 + * Setting: + * strategy=AGGR, persist=FALSE, threshold=1 + * latest format + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_AGGR, FALSE, (hsize_t)1) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, fapl)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + /* + * #7 -- h5repack_none.h5 + * Setting: + * strategy=NONE, persist=FALSE, threshold=1 + * inpage=8192 + */ + + /* Create file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + return -1; + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_NONE, FALSE, (hsize_t)1) < 0) + return -1; + if(H5Pset_file_space_page_size(fcpl, (hsize_t)8192) < 0) + return -1; + HDassert(j < NELMTS(H5REPACK_FSPACE_FNAMES)); + if((fid = H5Fcreate(H5REPACK_FSPACE_FNAMES[++j], H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + return -1; + if(H5Fclose(fid) < 0) + return -1; + if(H5Pclose(fcpl) < 0) + return -1; + + if(H5Pclose(fapl) < 0) + return -1; + return 0; out: diff --git a/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl b/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl index c499b35..54bf26c 100644 --- a/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl +++ b/tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl @@ -5,7 +5,7 @@ DATASET "vds_dset" { STORAGE_LAYOUT { CONTIGUOUS SIZE 576 - OFFSET 2144 + OFFSET 2048 } FILTERS { NONE diff --git a/tools/test/h5repack/testfiles/h5repack-help.txt b/tools/test/h5repack/testfiles/h5repack-help.txt index 049e27d..69b1644 100644 --- a/tools/test/h5repack/testfiles/h5repack-help.txt +++ b/tools/test/h5repack/testfiles/h5repack-help.txt @@ -19,8 +19,10 @@ usage: h5repack [OPTIONS] file1 file2 -a A, --alignment=A Alignment value for H5Pset_alignment -f FILT, --filter=FILT Filter type -l LAYT, --layout=LAYT Layout type - -S FS_STRGY, --fs_strategy=FS_STRGY File space management strategy - -T FS_THRD, --fs_threshold=FS_THRD Free-space section threshold + -S FS_STRATEGY, --fs_strategy=FS_STRATEGY File space management strategy for H5Pset_file_space_strategy + -P FS_PERSIST, --fs_persist=FS_PERSIST Persisting or not persisting free-space for H5Pset_file_space_strategy + -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD Free-space section threshold for H5Pset_file_space_strategy + -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE File space page size for H5Pset_file_space_page_size M - is an integer greater than 1, size of dataset in bytes (default is 0) E - is a filename. @@ -36,18 +38,27 @@ usage: h5repack [OPTIONS] file1 file2 --enable-error-stack Prints messages from the HDF5 error stack as they occur. - FS_STRGY is the file space management strategy to use for the output file. - It is a string as listed below: - ALL_PERSIST - Use persistent free-space managers, aggregators and virtual file driver - for file space allocation - ALL - Use non-persistent free-space managers, aggregators and virtual file driver - for file space allocation - AGGR_VFD - Use aggregators and virtual file driver for file space allocation - VFD - Use virtual file driver for file space allocation - - FS_THRD is the free-space section threshold to use for the output file. - It is the minimum size (in bytes) of free-space sections to be tracked - by the the library's free-space managers. + FS_STRATEGY is a string indicating the file space strategy used: + FSM_AGGR: + The mechanisms used in managing file space are free-space managers, aggregators and virtual file driver. + PAGE: + The mechanisms used in managing file space are free-space managers with embedded paged aggregation and virtual file driver. + AGGR: + The mechanisms used in managing file space are aggregators and virtual file driver. + NONE: + The mechanisms used in managing file space are virtual file driver. + The default strategy when not set is FSM_AGGR without persisting free-space. + + FS_PERSIST is 1 to persisting free-space or 0 to not persisting free-space. + The default when not set is not persisting free-space. + The value is ignored for AGGR and NONE strategies. + + FS_THRESHOLD is the minimum size (in bytes) of free-space sections to be tracked by the library. + The default when not set is 1. + The value is ignored for AGGR and NONE strategies. + + FS_PAGESIZE is the size (in bytes) >=512 that is used by the library when the file space strategy PAGE is used. + The default when not set is 4096. FILT - is a string with the format: diff --git a/tools/test/h5stat/CMakeTests.cmake b/tools/test/h5stat/CMakeTests.cmake index f6735d5..ea602ba 100644 --- a/tools/test/h5stat/CMakeTests.cmake +++ b/tools/test/h5stat/CMakeTests.cmake @@ -25,7 +25,6 @@ h5stat_newgrat.ddl h5stat_newgrat-UG.ddl h5stat_newgrat-UA.ddl - h5stat_idx.ddl h5stat_err1_links.ddl h5stat_links1.ddl h5stat_links2.ddl @@ -46,7 +45,6 @@ h5stat_filters.h5 h5stat_tsohm.h5 h5stat_newgrat.h5 - h5stat_idx.h5 h5stat_threshold.h5 ) @@ -196,8 +194,6 @@ ADD_H5_TEST (h5stat_newgrat 0 h5stat_newgrat.h5) ADD_H5_TEST (h5stat_newgrat-UG 0 -G h5stat_newgrat.h5) ADD_H5_TEST (h5stat_newgrat-UA 0 -A h5stat_newgrat.h5) -# h5stat_idx.h5 is generated by h5stat_gentest.c - ADD_H5_TEST (h5stat_idx 0 h5stat_idx.h5) # # Tests for -l (--links) option on h5stat_threshold.h5: # -l 0 (incorrect threshold value) diff --git a/tools/test/h5stat/h5stat_gentest.c b/tools/test/h5stat/h5stat_gentest.c index b1ab168..804f418 100644 --- a/tools/test/h5stat/h5stat_gentest.c +++ b/tools/test/h5stat/h5stat_gentest.c @@ -77,7 +77,7 @@ gen_newgrat_file(const char *fname) goto error; /* Set file space handling strategy */ - if(H5Pset_file_space(fcpl, H5F_FILE_SPACE_ALL_PERSIST, (hsize_t)0) < 0) + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, 1, (hsize_t)1) < 0) goto error; /* Create file */ diff --git a/tools/test/h5stat/testfiles/h5stat_filters.ddl b/tools/test/h5stat/testfiles/h5stat_filters.ddl index 1a4fd72..9f9e146 100644 --- a/tools/test/h5stat/testfiles/h5stat_filters.ddl +++ b/tools/test/h5stat/testfiles/h5stat_filters.ddl @@ -82,12 +82,14 @@ Small # of attributes (objects with 1 to 10 attributes): Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 37312 bytes Raw data: 8659 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_idx.ddl b/tools/test/h5stat/testfiles/h5stat_idx.ddl index b26f1a4..1b6ae0c 100644 --- a/tools/test/h5stat/testfiles/h5stat_idx.ddl +++ b/tools/test/h5stat/testfiles/h5stat_idx.ddl @@ -79,15 +79,17 @@ Small # of attributes (objects with 1 to 10 attributes): Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 965 bytes Raw data: 110 bytes Amount/Percent of tracked free space: 0 bytes/0.0% - Unaccounted space: 1131 bytes -Total space: 2206 bytes + Unaccounted space: 1083 bytes +Total space: 2158 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_idx.h5 b/tools/test/h5stat/testfiles/h5stat_idx.h5 Binary files differindex 303d1f8..83ebcdb 100644 --- a/tools/test/h5stat/testfiles/h5stat_idx.h5 +++ b/tools/test/h5stat/testfiles/h5stat_idx.h5 diff --git a/tools/test/h5stat/testfiles/h5stat_links2.ddl b/tools/test/h5stat/testfiles/h5stat_links2.ddl index 4622884..9fc82cd 100644 --- a/tools/test/h5stat/testfiles/h5stat_links2.ddl +++ b/tools/test/h5stat/testfiles/h5stat_links2.ddl @@ -91,12 +91,14 @@ Attribute bins: # of objects with 10 - 99 attributes: 3 Total # of objects with attributes: 5 Max. # of attributes to objects: 25 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 16128 bytes Raw data: 0 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_newgrat.ddl b/tools/test/h5stat/testfiles/h5stat_newgrat.ddl index e305f58..130fe2f 100644 --- a/tools/test/h5stat/testfiles/h5stat_newgrat.ddl +++ b/tools/test/h5stat/testfiles/h5stat_newgrat.ddl @@ -9,7 +9,7 @@ File information Max. # of objects in group: 35001 File space information for file metadata (in bytes): Superblock: 48 - Superblock extension: 119 + Superblock extension: 186 User block: 0 Object headers: (total/unused) Groups: 5145147/3220092 @@ -78,6 +78,7 @@ Attribute bins: # of objects with 100 - 999 attributes: 1 Total # of objects with attributes: 1 Max. # of attributes to objects: 100 +Free-space persist: TRUE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): # of sections of size 1: 1 @@ -86,10 +87,11 @@ Free-space section bins: # of sections of size 1 - 9: 1 # of sections of size 10 - 99: 4 Total # of sections: 5 -File space management strategy: H5F_FILE_SPACE_ALL_PERSIST +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: - File metadata: 6362036 bytes + File metadata: 6362103 bytes Raw data: 0 bytes Amount/Percent of tracked free space: 132 bytes/0.0% Unaccounted space: 0 bytes -Total space: 6362168 bytes +Total space: 6362235 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_newgrat.h5 b/tools/test/h5stat/testfiles/h5stat_newgrat.h5 Binary files differindex c919b71..0d68e79 100644 --- a/tools/test/h5stat/testfiles/h5stat_newgrat.h5 +++ b/tools/test/h5stat/testfiles/h5stat_newgrat.h5 diff --git a/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl b/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl index fb5568d..af53776 100644 --- a/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl +++ b/tools/test/h5stat/testfiles/h5stat_numattrs1.ddl @@ -8,7 +8,8 @@ Attribute bins: # of objects with 10 - 99 attributes: 3 Total # of objects with attributes: 5 Max. # of attributes to objects: 25 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 16128 bytes Raw data: 0 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl b/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl index ccb23c1..638781b 100644 --- a/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl +++ b/tools/test/h5stat/testfiles/h5stat_numattrs2.ddl @@ -91,12 +91,14 @@ Attribute bins: # of objects with 10 - 99 attributes: 3 Total # of objects with attributes: 5 Max. # of attributes to objects: 25 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 16128 bytes Raw data: 0 bytes diff --git a/tools/test/h5stat/testfiles/h5stat_tsohm.ddl b/tools/test/h5stat/testfiles/h5stat_tsohm.ddl index 4cf33fc..9369950 100644 --- a/tools/test/h5stat/testfiles/h5stat_tsohm.ddl +++ b/tools/test/h5stat/testfiles/h5stat_tsohm.ddl @@ -76,12 +76,14 @@ Small # of attributes (objects with 1 to 10 attributes): Attribute bins: Total # of objects with attributes: 0 Max. # of attributes to objects: 0 +Free-space persist: FALSE Free-space section threshold: 1 bytes Small size free-space sections (< 10 bytes): Total # of small size sections: 0 Free-space section bins: Total # of sections: 0 -File space management strategy: H5F_FILE_SPACE_ALL +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes Summary of file space information: File metadata: 3850 bytes Raw data: 0 bytes diff --git a/tools/test/misc/CMakeLists.txt b/tools/test/misc/CMakeLists.txt index 029d7a9..97a6f0c 100644 --- a/tools/test/misc/CMakeLists.txt +++ b/tools/test/misc/CMakeLists.txt @@ -39,4 +39,6 @@ INCLUDE_DIRECTORIES (${HDF5_TOOLS_DIR}/lib) target_link_libraries (clear_open_chk ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) set_target_properties (clear_open_chk PROPERTIES FOLDER tools) - include (CMakeTests.cmake) + include (CMakeTestsRepart.cmake) + include (CMakeTestsClear.cmake) + include (CMakeTestsMkgrp.cmake) diff --git a/tools/test/misc/CMakeTests.cmake b/tools/test/misc/CMakeTests.cmake deleted file mode 100644 index 9ecdeae..0000000 --- a/tools/test/misc/CMakeTests.cmake +++ /dev/null @@ -1,340 +0,0 @@ - -############################################################################## -############################################################################## -### T E S T I N G ### -############################################################################## -############################################################################## - - # -------------------------------------------------------------------- - # Copy all the HDF5 files from the source directory into the test directory - # -------------------------------------------------------------------- - set (HDF5_REFERENCE_TEST_FILES - family_file00000.h5 - family_file00001.h5 - family_file00002.h5 - family_file00003.h5 - family_file00004.h5 - family_file00005.h5 - family_file00006.h5 - family_file00007.h5 - family_file00008.h5 - family_file00009.h5 - family_file00010.h5 - family_file00011.h5 - family_file00012.h5 - family_file00013.h5 - family_file00014.h5 - family_file00015.h5 - family_file00016.h5 - family_file00017.h5 - ) - - foreach (h5_file ${HDF5_REFERENCE_TEST_FILES}) - HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "h5repart_files") - endforeach () - add_custom_target(h5repart_files ALL COMMENT "Copying files needed by h5repart tests" DEPENDS ${h5repart_files_list}) - - set (HDF5_MKGRP_TEST_FILES - #h5mkgrp_help.txt - #h5mkgrp_version - h5mkgrp_single.ls - h5mkgrp_single_v.ls - h5mkgrp_single_p.ls - h5mkgrp_single_l.ls - h5mkgrp_several.ls - h5mkgrp_several_v.ls - h5mkgrp_several_p.ls - h5mkgrp_several_l.ls - h5mkgrp_nested_p.ls - h5mkgrp_nested_lp.ls - h5mkgrp_nested_mult_p.ls - h5mkgrp_nested_mult_lp.ls - ) - - # make test dir - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - - foreach (h5_mkgrp_file ${HDF5_MKGRP_TEST_FILES}) - HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_mkgrp_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5_mkgrp_file}" "h5mkgrp_files") - endforeach () - - HDFTEST_COPY_FILE("${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_help.txt" "${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_help.txt" "h5mkgrp_files") - add_custom_target(h5mkgrp_files ALL COMMENT "Copying files needed by h5mkgrp tests" DEPENDS ${h5mkgrp_files_list}) - - configure_file (${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_version.txt.in ${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_version.txt @ONLY) - -############################################################################## -############################################################################## -### T H E T E S T S M A C R O S ### -############################################################################## -############################################################################## - - macro (ADD_H5_TEST resultfile resultcode resultoption) - if (NOT HDF5_ENABLE_USING_MEMCHECKER) - add_test ( - NAME H5MKGRP-${resultfile}-clear-objects - COMMAND ${CMAKE_COMMAND} - -E remove - ${resultfile}.h5 - ) - set_tests_properties (H5MKGRP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - endif () - - add_test ( - NAME H5MKGRP-${resultfile} - COMMAND $<TARGET_FILE:h5mkgrp> ${resultoption} ${resultfile}.h5 ${ARGN} - ) - set_tests_properties (H5MKGRP-${resultfile} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - if (HDF5_ENABLE_USING_MEMCHECKER) - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS ${last_test}) - endif () - else () - set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS H5MKGRP-${resultfile}-clear-objects) - add_test ( - NAME H5MKGRP-${resultfile}-h5ls - COMMAND "${CMAKE_COMMAND}" - -D "TEST_PROGRAM=$<TARGET_FILE:h5ls>" - -D "TEST_ARGS:STRING=-v;-r;${resultfile}.h5" - -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" - -D "TEST_OUTPUT=${resultfile}.out" - -D "TEST_EXPECT=${resultcode}" - -D "TEST_MASK_MOD=true" - -D "TEST_REFERENCE=${resultfile}.ls" - -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" - ) - set_tests_properties (H5MKGRP-${resultfile}-h5ls PROPERTIES DEPENDS H5MKGRP-${resultfile}) - endif () - endmacro () - - macro (ADD_H5_CMP resultfile resultcode) - if (HDF5_ENABLE_USING_MEMCHECKER) - add_test (NAME H5MKGRP_CMP-${resultfile} COMMAND $<TARGET_FILE:h5mkgrp> ${ARGN}) - else () - add_test ( - NAME H5MKGRP_CMP-${resultfile}-clear-objects - COMMAND ${CMAKE_COMMAND} - -E remove - ${resultfile}.h5 - ) - set_tests_properties (H5MKGRP_CMP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - add_test ( - NAME H5MKGRP_CMP-${resultfile} - COMMAND "${CMAKE_COMMAND}" - -D "TEST_PROGRAM=$<TARGET_FILE:h5mkgrp>" - -D "TEST_ARGS:STRING=${ARGN}" - -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" - -D "TEST_OUTPUT=${resultfile}.out" - -D "TEST_EXPECT=${resultcode}" - -D "TEST_REFERENCE=${resultfile}.txt" - -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" - ) - set_tests_properties (H5MKGRP_CMP-${resultfile} PROPERTIES DEPENDS H5MKGRP_CMP-${resultfile}-clear-objects) - endif () - endmacro () - -############################################################################## -############################################################################## -### T H E T E S T S ### -############################################################################## -############################################################################## - - ###################### H5REPART ######################### - # Remove any output file left over from previous test run - add_test ( - NAME H5REPART-clearall-objects - COMMAND ${CMAKE_COMMAND} - -E remove - fst_family00000.h5 - scd_family00000.h5 - scd_family00001.h5 - scd_family00002.h5 - scd_family00003.h5 - family_to_sec2.h5 - ) - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5REPART-clearall-objects PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "H5REPART-clearall-objects") - - # repartition family member size to 20,000 bytes. - add_test (NAME H5REPART-h5repart_20K COMMAND $<TARGET_FILE:h5repart> -m 20000 family_file%05d.h5 fst_family%05d.h5) - set_tests_properties (H5REPART-h5repart_20K PROPERTIES DEPENDS H5REPART-clearall-objects) - - # repartition family member size to 5 KB. - add_test (NAME H5REPART-h5repart_5K COMMAND $<TARGET_FILE:h5repart> -m 5k family_file%05d.h5 scd_family%05d.h5) - set_tests_properties (H5REPART-h5repart_5K PROPERTIES DEPENDS H5REPART-clearall-objects) - - # convert family file to sec2 file of 20,000 bytes - add_test (NAME H5REPART-h5repart_sec2 COMMAND $<TARGET_FILE:h5repart> -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5) - set_tests_properties (H5REPART-h5repart_sec2 PROPERTIES DEPENDS H5REPART-clearall-objects) - - # test the output files repartitioned above. - add_test (NAME H5REPART-h5repart_test COMMAND $<TARGET_FILE:h5repart_test>) - set_tests_properties (H5REPART-h5repart_test PROPERTIES DEPENDS "H5REPART-clearall-objects;H5REPART-h5repart_20K;H5REPART-h5repart_5K;H5REPART-h5repart_sec2") - - set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} - h5repart_test - ) - - ###################### H5CLEAR ######################### - # Remove any output file left over from previous test run - add_test ( - NAME H5CLEAR-clearall-objects - COMMAND ${CMAKE_COMMAND} - -E remove - h5clear_log_v3.h5 - h5clear_sec2_v0.h5 - h5clear_sec2_v2.h5 - h5clear_sec2_v3.h5 - latest_h5clear_log_v3.h5 - latest_h5clear_sec2_v3.h5 - ) - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5CLEAR-clearall-objects PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "H5CLEAR-clearall-objects") - - # create the output files to be used. - add_test (NAME H5CLEAR-h5clear_gentest COMMAND $<TARGET_FILE:h5clear_gentest>) - set_tests_properties (H5CLEAR-h5clear_gentest PROPERTIES DEPENDS "H5CLEAR-clearall-objects") - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-sec2_v3_F COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-sec2_v3 COMMAND $<TARGET_FILE:h5clear> h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-h5clear-sec2_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-sec2_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-sec2_v3 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-sec2_v3) - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-log_v3_F COMMAND $<TARGET_FILE:clear_open_chk> h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-log_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-log_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-log_v3 COMMAND $<TARGET_FILE:h5clear> h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-h5clear-log_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-log_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-log_v3 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-log_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-log_v3) - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-latest_sec2_v3_F COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_sec2_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-latest_sec2_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-latest_sec2_v3 COMMAND $<TARGET_FILE:h5clear> latest_h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-h5clear-latest_sec2_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-latest_sec2_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-latest_sec2_v3 COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_sec2_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_sec2_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-latest_sec2_v3) - - # Initial file open fails - add_test (NAME H5CLEAR-clear_open_chk-latest_log_v3_F COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_log_v3_F PROPERTIES WILL_FAIL "true") - set_tests_properties (H5CLEAR-clear_open_chk-latest_log_v3_F PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-latest_log_v3 COMMAND $<TARGET_FILE:h5clear> latest_h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-h5clear-latest_log_v3 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-latest_log_v3_F) - add_test (NAME H5CLEAR-clear_open_chk-latest_log_v3 COMMAND $<TARGET_FILE:clear_open_chk> latest_h5clear_log_v3.h5) - set_tests_properties (H5CLEAR-clear_open_chk-latest_log_v3 PROPERTIES DEPENDS H5CLEAR-h5clear-latest_log_v3) - - # - # File open succeeds because the library does not check status_flags for file with < v3 superblock - add_test (NAME H5CLEAR-clear_open_chk-sec2_v0_P COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v0.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v0_P PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-sec2_v0 COMMAND $<TARGET_FILE:h5clear> h5clear_sec2_v0.h5) - set_tests_properties (H5CLEAR-h5clear-sec2_v0 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-sec2_v0_P) - add_test (NAME H5CLEAR-clear_open_chk-sec2_v0 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v0.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v0 PROPERTIES DEPENDS H5CLEAR-h5clear-sec2_v0) - - # - # File open succeeds because the library does not check status_flags for file with < v3 superblock - add_test (NAME H5CLEAR-clear_open_chk-sec2_v2_P COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v2.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v2_P PROPERTIES DEPENDS H5CLEAR-h5clear_gentest) - # After "h5clear" the file, the subsequent file open succeeds - add_test (NAME H5CLEAR-h5clear-sec2_v2 COMMAND $<TARGET_FILE:h5clear> h5clear_sec2_v2.h5) - set_tests_properties (H5CLEAR-h5clear-sec2_v2 PROPERTIES DEPENDS H5CLEAR-clear_open_chk-sec2_v2_P) - add_test (NAME H5CLEAR-clear_open_chk-sec2_v2 COMMAND $<TARGET_FILE:clear_open_chk> h5clear_sec2_v2.h5) - set_tests_properties (H5CLEAR-clear_open_chk-sec2_v2 PROPERTIES DEPENDS H5CLEAR-h5clear-sec2_v2) - - set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} - h5clear_gentest - ) - - ###################### H5MKGRP ######################### - if (HDF5_ENABLE_USING_MEMCHECKER) - add_test ( - NAME H5MKGRP-clearall-objects - COMMAND ${CMAKE_COMMAND} - -E remove - h5mkgrp_help.out - h5mkgrp_help.out.err - h5mkgrp_version.out - h5mkgrp_version.out.err - h5mkgrp_single.h5 - h5mkgrp_single.out - h5mkgrp_single.out.err - h5mkgrp_single_v.h5 - h5mkgrp_single_v.out - h5mkgrp_single_v.out.err - h5mkgrp_single_p.h5 - h5mkgrp_single_p.out - h5mkgrp_single_p.out.err - h5mkgrp_single_l.h5 - h5mkgrp_single_l.out - h5mkgrp_single_l.out.err - h5mkgrp_several.h5 - h5mkgrp_several.out - h5mkgrp_several.out.err - h5mkgrp_several_v.h5 - h5mkgrp_several_v.out - h5mkgrp_several_v.out.err - h5mkgrp_several_p.h5 - h5mkgrp_several_p.out - h5mkgrp_several_p.out.err - h5mkgrp_several_l.h5 - h5mkgrp_several_l.out - h5mkgrp_several_l.out.err - h5mkgrp_nested_p.h5 - h5mkgrp_nested_p.out - h5mkgrp_nested_p.out.err - h5mkgrp_nested_lp.h5 - h5mkgrp_nested_lp.out - h5mkgrp_nested_lp.out.err - h5mkgrp_nested_mult_p.h5 - h5mkgrp_nested_mult_p.out - h5mkgrp_nested_mult_p.out.err - h5mkgrp_nested_mult_lp.h5 - h5mkgrp_nested_mult_lp.out - h5mkgrp_nested_mult_lp.out.err - ) - set_tests_properties (H5MKGRP-clearall-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - if (NOT "${last_test}" STREQUAL "") - set_tests_properties (H5MKGRP-clearall-objects PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "H5MKGRP-clearall-objects") - endif () - - # Check that help & version is displayed properly - ADD_H5_CMP (h5mkgrp_help 0 "-h") - ADD_H5_CMP (h5mkgrp_version 0 "-V") - - # Create single group at root level - ADD_H5_TEST (h5mkgrp_single 0 "" single) - ADD_H5_TEST (h5mkgrp_single_v 0 "-v" single) - ADD_H5_TEST (h5mkgrp_single_p 0 "-p" single) - ADD_H5_TEST (h5mkgrp_single_l 0 "-l" latest) - - # Create several groups at root level - ADD_H5_TEST (h5mkgrp_several 0 "" one two) - ADD_H5_TEST (h5mkgrp_several_v 0 "-v" one two) - ADD_H5_TEST (h5mkgrp_several_p 0 "-p" one two) - ADD_H5_TEST (h5mkgrp_several_l 0 "-l" one two) - - # Create various nested groups - ADD_H5_TEST (h5mkgrp_nested_p 0 "-p" /one/two) - ADD_H5_TEST (h5mkgrp_nested_lp 0 "-lp" /one/two) - ADD_H5_TEST (h5mkgrp_nested_mult_p 0 "-p" /one/two /three/four) - ADD_H5_TEST (h5mkgrp_nested_mult_lp 0 "-lp" /one/two /three/four) diff --git a/tools/test/misc/CMakeTestsClear.cmake b/tools/test/misc/CMakeTestsClear.cmake new file mode 100644 index 0000000..a92d523 --- /dev/null +++ b/tools/test/misc/CMakeTestsClear.cmake @@ -0,0 +1,197 @@ + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (HDF5_REFERENCE_TEST_FILES + h5clear_usage.ddl + h5clear_open_fail.ddl + h5clear_missing_file.ddl + h5clear_no_mdc_image.ddl + orig_h5clear_sec2_v0.h5 + orig_h5clear_sec2_v2.h5 + orig_h5clear_sec2_v3.h5 + mod_h5clear_mdc_image.h5 + ) + + foreach (h5_file ${HDF5_REFERENCE_TEST_FILES}) + HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5_file}" "h5clear_files") + endforeach () + # make second copy of mod_h5clear_mdc_image.h5 + HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/mod_h5clear_mdc_image.h5" "${PROJECT_BINARY_DIR}/testfiles/mod_h5clear_mdc_image2.h5" "h5clear_files") + add_custom_target(h5clear_files ALL COMMENT "Copying files needed by h5clear tests" DEPENDS ${h5clear_files_list}) + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_CMP testname resultfile resultcode) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5CLEAR_CMP-${testname}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + ${testname}.out + ${testname}.out.err + ) + add_test ( + NAME H5CLEAR_CMP-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5clear>" + -D "TEST_ARGS:STRING=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5CLEAR_CMP-${testname} PROPERTIES DEPENDS H5CLEAR_CMP-${testname}-clear-objects) + endif () + endmacro () + + macro (ADD_H5_RETTEST testname resultcode) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5CLEAR_RET-${testname} + COMMAND $<TARGET_FILE:h5clear> ${ARGN} + ) + set_tests_properties (H5CLEAR_RET-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set_tests_properties (H5CLEAR_RET-${testname} PROPERTIES WILL_FAIL "${resultcode}") + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR_RET-${testname} PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5CLEAR_RET-${testname}") + endif () + endmacro () + + macro (ADD_H5_TEST testname resultcode) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + # Initial file open fails OR + # File open succeeds because the library does not check status_flags for file with < v3 superblock + add_test (NAME H5CLEAR-clear_open_chk-${testname}_${resultcode} COMMAND $<TARGET_FILE:clear_open_chk> ${testname}.h5) + set_tests_properties (H5CLEAR-clear_open_chk-${testname}_${resultcode} PROPERTIES WILL_FAIL "${resultcode}") + set_tests_properties (H5CLEAR-clear_open_chk-${testname}_${resultcode} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR-clear_open_chk-${testname}_${resultcode} PROPERTIES DEPENDS ${last_test}) + endif () + # After "h5clear" the file, the subsequent file open succeeds + add_test (NAME H5CLEAR-h5clear-${testname} COMMAND $<TARGET_FILE:h5clear> -s ${testname}.h5) + set_tests_properties (H5CLEAR-h5clear-${testname} PROPERTIES DEPENDS H5CLEAR-clear_open_chk-${testname}_${resultcode}) + set_tests_properties (H5CLEAR-h5clear-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + add_test (NAME H5CLEAR-clear_open_chk-${testname} COMMAND $<TARGET_FILE:clear_open_chk> ${testname}.h5) + set_tests_properties (H5CLEAR-clear_open_chk-${testname} PROPERTIES DEPENDS H5CLEAR-h5clear-${testname}) + set_tests_properties (H5CLEAR-clear_open_chk-${testname} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set (last_test "H5CLEAR-clear_open_chk-${testname}") + endif () + endmacro () + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## +# +# +# +# The following are tests to verify the status_flags field is cleared properly: + # Remove any output file left over from previous test run + add_test ( + NAME H5CLEAR-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + h5clear_log_v3.h5 + h5clear_mdc_image.h5 + h5clear_sec2_v0.h5 + h5clear_sec2_v2.h5 + h5clear_sec2_v3.h5 + latest_h5clear_log_v3.h5 + latest_h5clear_sec2_v3.h5 + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5CLEAR-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5CLEAR-clearall-objects") + + # create the output files to be used. + add_test (NAME H5CLEAR-h5clear_gentest COMMAND $<TARGET_FILE:h5clear_gentest>) + set_tests_properties (H5CLEAR-h5clear_gentest PROPERTIES DEPENDS "H5CLEAR-clearall-objects") + set_tests_properties (H5CLEAR-h5clear_gentest PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + set (last_test "H5CLEAR-h5clear_gentest") + +# +# +# +# The following are tests to verify the expected output from h5clear +# "h5clear -h" +# "h5clear" (no options, no file) +# "h5clear junk.h5" (no options, nonexisting file) +# "h5clear orig_h5clear_sec2_v3.h5" (no options, existing file) +# "h5clear -m" (valid 1 option, no file) +# "h5clear -s junk.h5" (valid 1 option, nonexisting file) +# "h5clear -m -s junk.h5" (valid 2 options, no file) +# "h5clear -m -s junk.h5" (valid 2 options, nonexisting file) +# "h5clear -m orig_h5clear_sec2_v2.h5" (valid 1 option, existing file, no cache image) +# "h5clear -s -m orig_h5clear_sec2_v0.h5" (valid 2 options, existing file, no cache image) + ADD_H5_CMP (h5clear_usage_h h5clear_usage 0 "-h") + ADD_H5_CMP (h5clear_usage h5clear_usage 1 "") + ADD_H5_CMP (h5clear_usage_junk h5clear_usage 1 "" junk.h5) + ADD_H5_CMP (h5clear_usage_none h5clear_usage 1 "" orig_h5clear_sec2_v3.h5) + ADD_H5_CMP (h5clear_missing_file_m h5clear_missing_file 1 "-m") + ADD_H5_CMP (h5clear_open_fail_s h5clear_open_fail 1 "-s" junk.h5) + ADD_H5_CMP (h5clear_missing_file_ms h5clear_missing_file 1 "-m" "-s") + ADD_H5_CMP (h5clear_open_fail_ms h5clear_open_fail 1 "-m" "-s" junk.h5) + ADD_H5_CMP (h5clear_no_mdc_image_m h5clear_no_mdc_image 0 "-m" orig_h5clear_sec2_v2.h5) + ADD_H5_CMP (h5clear_no_mdc_image_ms h5clear_no_mdc_image 0 "-m" "-s" orig_h5clear_sec2_v0.h5) +# +# +# +# The following are tests to verify the expected exit code from h5clear: +# "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) +# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear -k" (invalid 1 option, no file, fail exit code) +# "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) +# "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) +# "h5clear -m -k" (valid/invalid 2 options, nofile, fail exit code) +# "h5clear -l -m" (invalid/valid 2 options, nofile, fail exit code) +# "h5clear -m -k junk.h5" (valid/invalid 2 options, nonexisting file, fail exit code) +# "h5clear -l -m junk.h5" (invalid/valid 2 options, nonexisting file, fail exit code) +# "h5clear -m -k h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) +# "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) + ADD_H5_RETTEST (h5clear_mdc_image "false" "-m" h5clear_mdc_image.h5) + ADD_H5_RETTEST (h5clear_vers "false" "--vers") + ADD_H5_RETTEST (h5clear_k "true" "-k") + ADD_H5_RETTEST (h5clear_k_junk "true" "-k" junk.h5) + ADD_H5_RETTEST (h5clear_l_sec2 "true" "-l" h5clear_sec2_v2.h5) + ADD_H5_RETTEST (h5clear_mk "true" "-m" "-k") + ADD_H5_RETTEST (h5clear_lm "true" "-l" "-m") + ADD_H5_RETTEST (h5clear_ml_junk "true" "-m" "-l" junk.h5) + ADD_H5_RETTEST (h5clear_lm_junk "true" "-l" "-m" junk.h5) + ADD_H5_RETTEST (h5clear_ml_sec2 "true" "-m" "-l" h5clear_sec2_v2.h5) + ADD_H5_RETTEST (h5clear_lm_sec2 "true" "-l" "-m" h5clear_sec2_v2.h5) +# +# +# +# h5clear_mdc_image.h5 already has cache image removed earlier, verify the expected warning from h5clear: + ADD_H5_CMP (h5clear_mdc_image_m h5clear_no_mdc_image 0 "-m" mod_h5clear_mdc_image.h5) + ADD_H5_CMP (h5clear_mdc_image_sm h5clear_no_mdc_image 0 "-m" "-s" mod_h5clear_mdc_image2.h5) +# +# +# +# The following are tests to verify the status_flags field is cleared properly: + ADD_H5_TEST (h5clear_sec2_v3 "true") + ADD_H5_TEST (h5clear_log_v3 "true") + ADD_H5_TEST (latest_h5clear_sec2_v3 "true") + ADD_H5_TEST (latest_h5clear_log_v3 "true") + ADD_H5_TEST (h5clear_sec2_v0 "false") + ADD_H5_TEST (h5clear_sec2_v2 "false") + + set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} + h5clear_gentest + ) diff --git a/tools/test/misc/CMakeTestsMkgrp.cmake b/tools/test/misc/CMakeTestsMkgrp.cmake new file mode 100644 index 0000000..2a54a0e --- /dev/null +++ b/tools/test/misc/CMakeTestsMkgrp.cmake @@ -0,0 +1,188 @@ + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (HDF5_MKGRP_TEST_FILES + #h5mkgrp_help.txt + #h5mkgrp_version + h5mkgrp_single.ls + h5mkgrp_single_v.ls + h5mkgrp_single_p.ls + h5mkgrp_single_l.ls + h5mkgrp_several.ls + h5mkgrp_several_v.ls + h5mkgrp_several_p.ls + h5mkgrp_several_l.ls + h5mkgrp_nested_p.ls + h5mkgrp_nested_lp.ls + h5mkgrp_nested_mult_p.ls + h5mkgrp_nested_mult_lp.ls + ) + + # make test dir + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + + foreach (h5_mkgrp_file ${HDF5_MKGRP_TEST_FILES}) + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_mkgrp_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5_mkgrp_file}" "h5mkgrp_files") + endforeach () + + HDFTEST_COPY_FILE("${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_help.txt" "${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_help.txt" "h5mkgrp_files") + add_custom_target(h5mkgrp_files ALL COMMENT "Copying files needed by h5mkgrp tests" DEPENDS ${h5mkgrp_files_list}) + + configure_file (${HDF5_TOOLS_TEST_MISC_SOURCE_DIR}/testfiles/h5mkgrp_version.txt.in ${PROJECT_BINARY_DIR}/testfiles/h5mkgrp_version.txt @ONLY) + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_TEST resultfile resultcode resultoption) + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5MKGRP-${resultfile}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + ${resultfile}.h5 + ) + set_tests_properties (H5MKGRP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + endif () + + add_test ( + NAME H5MKGRP-${resultfile} + COMMAND $<TARGET_FILE:h5mkgrp> ${resultoption} ${resultfile}.h5 ${ARGN} + ) + set_tests_properties (H5MKGRP-${resultfile} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (HDF5_ENABLE_USING_MEMCHECKER) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS ${last_test}) + endif () + else (HDF5_ENABLE_USING_MEMCHECKER) + set_tests_properties (H5MKGRP-${resultfile} PROPERTIES DEPENDS H5MKGRP-${resultfile}-clear-objects) + add_test ( + NAME H5MKGRP-${resultfile}-h5ls + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5ls>" + -D "TEST_ARGS:STRING=-v;-r;${resultfile}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_MASK_MOD=true" + -D "TEST_REFERENCE=${resultfile}.ls" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5MKGRP-${resultfile}-h5ls PROPERTIES DEPENDS H5MKGRP-${resultfile}) + endif () + endmacro () + + macro (ADD_H5_CMP resultfile resultcode) + if (HDF5_ENABLE_USING_MEMCHECKER) + add_test (NAME H5MKGRP_CMP-${resultfile} COMMAND $<TARGET_FILE:h5mkgrp> ${ARGN}) + else () + add_test ( + NAME H5MKGRP_CMP-${resultfile}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + ${resultfile}.h5 + ) + set_tests_properties (H5MKGRP_CMP-${resultfile}-clear-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + add_test ( + NAME H5MKGRP_CMP-${resultfile} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5mkgrp>" + -D "TEST_ARGS:STRING=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.txt" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5MKGRP_CMP-${resultfile} PROPERTIES DEPENDS H5MKGRP_CMP-${resultfile}-clear-objects) + endif () + endmacro () + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + if (HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5MKGRP-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + h5mkgrp_help.out + h5mkgrp_help.out.err + h5mkgrp_version.out + h5mkgrp_version.out.err + h5mkgrp_single.h5 + h5mkgrp_single.out + h5mkgrp_single.out.err + h5mkgrp_single_v.h5 + h5mkgrp_single_v.out + h5mkgrp_single_v.out.err + h5mkgrp_single_p.h5 + h5mkgrp_single_p.out + h5mkgrp_single_p.out.err + h5mkgrp_single_l.h5 + h5mkgrp_single_l.out + h5mkgrp_single_l.out.err + h5mkgrp_several.h5 + h5mkgrp_several.out + h5mkgrp_several.out.err + h5mkgrp_several_v.h5 + h5mkgrp_several_v.out + h5mkgrp_several_v.out.err + h5mkgrp_several_p.h5 + h5mkgrp_several_p.out + h5mkgrp_several_p.out.err + h5mkgrp_several_l.h5 + h5mkgrp_several_l.out + h5mkgrp_several_l.out.err + h5mkgrp_nested_p.h5 + h5mkgrp_nested_p.out + h5mkgrp_nested_p.out.err + h5mkgrp_nested_lp.h5 + h5mkgrp_nested_lp.out + h5mkgrp_nested_lp.out.err + h5mkgrp_nested_mult_p.h5 + h5mkgrp_nested_mult_p.out + h5mkgrp_nested_mult_p.out.err + h5mkgrp_nested_mult_lp.h5 + h5mkgrp_nested_mult_lp.out + h5mkgrp_nested_mult_lp.out.err + ) + set_tests_properties (H5MKGRP-clearall-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5MKGRP-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5MKGRP-clearall-objects") + endif () + + # Check that help & version is displayed properly + ADD_H5_CMP (h5mkgrp_help 0 "-h") + ADD_H5_CMP (h5mkgrp_version 0 "-V") + + # Create single group at root level + ADD_H5_TEST (h5mkgrp_single 0 "" single) + ADD_H5_TEST (h5mkgrp_single_v 0 "-v" single) + ADD_H5_TEST (h5mkgrp_single_p 0 "-p" single) + ADD_H5_TEST (h5mkgrp_single_l 0 "-l" latest) + + # Create several groups at root level + ADD_H5_TEST (h5mkgrp_several 0 "" one two) + ADD_H5_TEST (h5mkgrp_several_v 0 "-v" one two) + ADD_H5_TEST (h5mkgrp_several_p 0 "-p" one two) + ADD_H5_TEST (h5mkgrp_several_l 0 "-l" one two) + + # Create various nested groups + ADD_H5_TEST (h5mkgrp_nested_p 0 "-p" /one/two) + ADD_H5_TEST (h5mkgrp_nested_lp 0 "-lp" /one/two) + ADD_H5_TEST (h5mkgrp_nested_mult_p 0 "-p" /one/two /three/four) + ADD_H5_TEST (h5mkgrp_nested_mult_lp 0 "-lp" /one/two /three/four) diff --git a/tools/test/misc/CMakeTestsRepart.cmake b/tools/test/misc/CMakeTestsRepart.cmake new file mode 100644 index 0000000..a1549ec --- /dev/null +++ b/tools/test/misc/CMakeTestsRepart.cmake @@ -0,0 +1,85 @@ + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (HDF5_REFERENCE_TEST_FILES + family_file00000.h5 + family_file00001.h5 + family_file00002.h5 + family_file00003.h5 + family_file00004.h5 + family_file00005.h5 + family_file00006.h5 + family_file00007.h5 + family_file00008.h5 + family_file00009.h5 + family_file00010.h5 + family_file00011.h5 + family_file00012.h5 + family_file00013.h5 + family_file00014.h5 + family_file00015.h5 + family_file00016.h5 + family_file00017.h5 + ) + + foreach (h5_file ${HDF5_REFERENCE_TEST_FILES}) + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "h5repart_files") + endforeach () + add_custom_target(h5repart_files ALL COMMENT "Copying files needed by h5repart tests" DEPENDS ${h5repart_files_list}) + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + + # Remove any output file left over from previous test run + add_test ( + NAME H5REPART-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + fst_family00000.h5 + scd_family00000.h5 + scd_family00001.h5 + scd_family00002.h5 + scd_family00003.h5 + family_to_sec2.h5 + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5REPART-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5REPART-clearall-objects") + + # repartition family member size to 20,000 bytes. + add_test (NAME H5REPART-h5repart_20K COMMAND $<TARGET_FILE:h5repart> -m 20000 family_file%05d.h5 fst_family%05d.h5) + set_tests_properties (H5REPART-h5repart_20K PROPERTIES DEPENDS H5REPART-clearall-objects) + + # repartition family member size to 5 KB. + add_test (NAME H5REPART-h5repart_5K COMMAND $<TARGET_FILE:h5repart> -m 5k family_file%05d.h5 scd_family%05d.h5) + set_tests_properties (H5REPART-h5repart_5K PROPERTIES DEPENDS H5REPART-clearall-objects) + + # convert family file to sec2 file of 20,000 bytes + add_test (NAME H5REPART-h5repart_sec2 COMMAND $<TARGET_FILE:h5repart> -m 20000 -family_to_sec2 family_file%05d.h5 family_to_sec2.h5) + set_tests_properties (H5REPART-h5repart_sec2 PROPERTIES DEPENDS H5REPART-clearall-objects) + + # test the output files repartitioned above. + add_test (NAME H5REPART-h5repart_test COMMAND $<TARGET_FILE:h5repart_test>) + set_tests_properties (H5REPART-h5repart_test PROPERTIES DEPENDS "H5REPART-clearall-objects;H5REPART-h5repart_20K;H5REPART-h5repart_5K;H5REPART-h5repart_sec2") + + set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} + h5repart_test + ) diff --git a/tools/test/misc/h5clear_gentest.c b/tools/test/misc/h5clear_gentest.c index 0f20c35..f8b6a70 100644 --- a/tools/test/misc/h5clear_gentest.c +++ b/tools/test/misc/h5clear_gentest.c @@ -25,6 +25,103 @@ const char *FILENAME[] = { #define KB 1024U +#define CACHE_IMAGE_FILE "h5clear_mdc_image.h5" +#define DSET "DSET" + +/*------------------------------------------------------------------------- + * Function: gen_cache_image_file + * + * Purpose: To create a file with cache image feature enabled. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Vailin Choi; March 2017 + * + *------------------------------------------------------------------------- + */ +static int +gen_cache_image_file(const char *fname) +{ + hid_t fid = -1; /* File ID */ + hid_t did = -1, sid = -1; /* Dataset ID, dataspace ID */ + hid_t fapl = -1; /* File access property list */ + hid_t dcpl = -1; /* Dataset creation property list */ + hsize_t dims[2]; /* Dimension sizes */ + hsize_t chunks[2]; /* Chunked dimension sizes */ + int buf[50][100]; /* Buffer for data to write */ + int i, j; /* Local index variables */ + H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */ + { H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, + TRUE, FALSE, + H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; + + /* Create a copy of file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + goto error; + + /* Enable latest format in fapl */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + goto error; + + /* Enable metadata cache image in fapl */ + if(H5Pset_mdc_image_config(fapl, &cache_image_config) < 0) + goto error; + + /* Create the file */ + if((fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + goto error; + + /* Create dataspace */ + dims[0] = 50; + dims[1] = 100; + if((sid = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + + /* Initialize buffer for writing to dataset */ + for(i = 0; i < 50; i++) + for(j = 0; j < 100; j++) + buf[i][j] = i * j; + + /* Set up to create a chunked dataset */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + chunks[0] = 5; + chunks[1] = 10; + if(H5Pset_chunk(dcpl, 2, chunks) < 0) + goto error; + if((did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Write to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + goto error; + + /* Closing */ + if(H5Dclose(did) < 0) + goto error; + if(H5Pclose(dcpl) < 0) + goto error; + if(H5Pclose(fapl) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Fclose(fid) < 0) + goto error; + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(sid); + H5Dclose(did); + H5Fclose(fid); + H5Pclose(fapl); + H5Pclose(dcpl); + } H5E_END_TRY; + return 1; +} + /*------------------------------------------------------------------------- * Function: main * @@ -60,6 +157,14 @@ main(void) char fname[512]; /* File name */ unsigned new_format; /* To use latest library format or not */ + /* Generate a file with cache image feature enabled */ + if(gen_cache_image_file(CACHE_IMAGE_FILE) < 0) + goto error; + + /* + * Generate files with invalid status_flags + */ + /* Create a copy of the file access property list */ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; diff --git a/tools/test/misc/testfiles/h5clear_missing_file.ddl b/tools/test/misc/testfiles/h5clear_missing_file.ddl new file mode 100644 index 0000000..1e5150c --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_missing_file.ddl @@ -0,0 +1,15 @@ +usage: h5clear [OPTIONS] file_name + OPTIONS + -h, --help Print a usage message and exit + -V, --version Print version number and exit + -s, --status Clear the status_flags field in the file's superblock + -m, --image Remove the metadata cache image from the file + +Examples of use: + +h5clear -s file_name + Clear the status_flags field in the superblock of the HDF5 file <file_name>. + +h5clear -m file_name + Remove the metadata cache image from the HDF5 file <file_name>. +h5clear error: missing file name diff --git a/tools/test/misc/testfiles/h5clear_no_mdc_image.ddl b/tools/test/misc/testfiles/h5clear_no_mdc_image.ddl new file mode 100644 index 0000000..f5acd71 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_no_mdc_image.ddl @@ -0,0 +1 @@ +h5clear warning: No cache image in the file diff --git a/tools/test/misc/testfiles/h5clear_open_fail.ddl b/tools/test/misc/testfiles/h5clear_open_fail.ddl new file mode 100644 index 0000000..895ecd4 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_open_fail.ddl @@ -0,0 +1 @@ +h5clear error: h5tools_fopen diff --git a/tools/test/misc/testfiles/h5clear_usage.ddl b/tools/test/misc/testfiles/h5clear_usage.ddl new file mode 100644 index 0000000..a399ae7 --- /dev/null +++ b/tools/test/misc/testfiles/h5clear_usage.ddl @@ -0,0 +1,14 @@ +usage: h5clear [OPTIONS] file_name + OPTIONS + -h, --help Print a usage message and exit + -V, --version Print version number and exit + -s, --status Clear the status_flags field in the file's superblock + -m, --image Remove the metadata cache image from the file + +Examples of use: + +h5clear -s file_name + Clear the status_flags field in the superblock of the HDF5 file <file_name>. + +h5clear -m file_name + Remove the metadata cache image from the HDF5 file <file_name>. diff --git a/tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 b/tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 Binary files differnew file mode 100644 index 0000000..310134b --- /dev/null +++ b/tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 diff --git a/tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 b/tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 Binary files differnew file mode 100644 index 0000000..571fd46 --- /dev/null +++ b/tools/test/misc/testfiles/orig_h5clear_sec2_v0.h5 diff --git a/tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 b/tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 Binary files differnew file mode 100644 index 0000000..92833e6 --- /dev/null +++ b/tools/test/misc/testfiles/orig_h5clear_sec2_v2.h5 diff --git a/tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 b/tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 Binary files differnew file mode 100644 index 0000000..9f1e6ce --- /dev/null +++ b/tools/test/misc/testfiles/orig_h5clear_sec2_v3.h5 diff --git a/tools/test/misc/testh5clear.sh.in b/tools/test/misc/testh5clear.sh.in index a6836d4..71d5cad 100644 --- a/tools/test/misc/testh5clear.sh.in +++ b/tools/test/misc/testh5clear.sh.in @@ -20,11 +20,19 @@ TESTNAME=h5clear EXIT_SUCCESS=0 EXIT_FAILURE=1 -H5CLEAR=../../src/misc/h5clear -H5CLEAR_BIN=`pwd`/$H5CLEAR # The path of the tool binary +H5CLEAR=../../src/misc/h5clear # The tool name +H5CLEAR_BIN=`pwd`/$H5CLEAR # The path of the tool binary -OPENCHK=clear_open_chk # Try opening the test file -OPENCHK_BIN=`pwd`/$OPENCHK # The path to the binary +OPENCHK=clear_open_chk # Program to check whether the file can be opened +OPENCHK_BIN=`pwd`/$OPENCHK # The path to the binary + +RM='rm -rf' +CMP='cmp -s' +DIFF='diff -c' +CP='cp' +DIRNAME='dirname' +LS='ls' +AWK='awk' SUCCEED=0 FAIL=1 @@ -32,6 +40,83 @@ FAIL=1 nerrors=0 verbose=yes +# source dirs +SRC_TOOLS="$srcdir/../.." + +SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" +# testfiles source dirs for tools +SRC_H5CLEAR_TESTFILES="$SRC_TOOLS/test/misc/testfiles" + +TESTDIR=./testh5clear +test -d $TESTDIR || mkdir -p $TESTDIR + +###################################################################### +# test files +# -------------------------------------------------------------------- +# All the test files copy from source directory to test directory +# NOTE: Keep this framework to add/remove test files. +# Any test files from other tools can be used in this framework. +# This list are also used for checking exist. +# Comment '#' without space can be used. +# -------------------------------------------------------------------- + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES=" +$SRC_H5CLEAR_TESTFILES/h5clear_usage.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_open_fail.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_missing_file.ddl +$SRC_H5CLEAR_TESTFILES/h5clear_no_mdc_image.ddl +$SRC_H5CLEAR_TESTFILES/orig_h5clear_sec2_v0.h5 +$SRC_H5CLEAR_TESTFILES/orig_h5clear_sec2_v2.h5 +$SRC_H5CLEAR_TESTFILES/orig_h5clear_sec2_v3.h5 +$SRC_H5CLEAR_TESTFILES/mod_h5clear_mdc_image.h5 +" + +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + # skip cp if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=`$DIRNAME $tstfile` + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $CP -f $tstfile $TESTDIR + if [ $? -ne 0 ]; then + echo "Error: FAILED to copy $tstfile ." + + # Comment out this to CREATE expected file + exit $EXIT_FAILURE + fi + fi + fi + done +} + +CLEAN_TESTFILES_AND_TESTDIR() +{ + # skip rm if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=$SRC_H5CLEAR_TESTFILES + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $RM $TESTDIR + fi +} + +# +# # Print a line-line message left justified in a field of 70 characters # beginning with the word "Testing". # @@ -40,68 +125,197 @@ TESTING() { echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' } +# $1 is the expected output +# $2 is the output from testing +COMPARE_OUT() { + if $CMP $1 $2; then + echo "PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi +} + +# +# +# +# Run h5clear and verify the expected output +# $1: the test file name +# $2: option 1 passed to the tool +# $3: option 2 passed to the tool +# $4: the expected output +TOOLTEST_OUT() { + fname=$1 + option1=$2 + option2=$3 + expected=$4 + # Prepare expected and actual output + expect="$TESTDIR/$expected" + actual="$TESTDIR/`basename $expected .ddl`.out" + actual_err="$TESTDIR/`basename $expected .ddl`.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + + # Run test. + TESTING $H5CLEAR $option1 $option2 $fname + ( + $RUNSERIAL $H5CLEAR_BIN $option1 $option2 $fname + ) >$actual 2>$actual_err + cp $actual $actual_sav + cp $actual_err $actual_err_sav + cat $actual_err >> $actual + + # Compare output + COMPARE_OUT $expect $actual + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + $RM $actual $actual_err $actual_sav $actual_err_sav + fi +} -# (1)Use "clear_open_chk" to check if the file open succeeds or fails # $1 is the filename to open -# $2 is the expected return from "clear_open_chk" -# (2) Use "h5clear" to clear the status_flags in the test file -# (3) Open the test file via "clear_open_chk" -TOOLTEST() { +# $2 is the expected return from the open/check program +OPEN_CHK() { fname=$1 expected=$2 - # (1) $OPENCHK_BIN $fname 2>/dev/null actual=$? if test $actual -ne $expected; then - echo "Unexpected return from $OPENCHK" - nerrors=`expr $nerrors + 1` + echo "Unexpected return from $OPENCHK" + nerrors=`expr $nerrors + 1` fi - # (2) - TESTING $H5CLEAR $1 +} + +# Run h5clear and verify the expected exit code +# $1 is the filename to open +# $2 is option 1 +# $3 is option 2 +# $4 is the expected exit code from the tool +TOOLTEST() { fname=$1 - # Use "h5clear" to clear the status_flags in the test file - $RUNSERIAL $H5CLEAR_BIN $fname - if test $? -ne $SUCCEED; then - echo ".....$H5CLEAR: should succeed" - nerrors=`expr $nerrors + 1` - else - # (3) Open the test file via "clear_open_chk" - $OPENCHK_BIN $fname - if test $? -ne $SUCCEED; then - echo "......$OPENCHK: should succeed" + option1=$2 + option2=$3 + ret_expected=$4 + # + TESTING $H5CLEAR $option1 $option2 $fname + fname=$1 + # h5clear $option1 $option2 $fname + $RUNSERIAL $H5CLEAR_BIN $option1 $option2 $fname 2>/dev/null 1>/dev/null + if test $? -ne $ret_expected; then + echo "*FAILED*" nerrors=`expr $nerrors + 1` else echo "PASSED" fi - fi } ############################################################################## ############################################################################## -### T H E T E S T S ### +### T H E T E S T S ### ############################################################################## ############################################################################## +# prepare for testing +COPY_TESTFILES_TO_TESTDIR +# +# +# +# The following are tests to verify the expected output from h5clear +# "h5clear -h" +# "h5clear" (no options, no file) +# "h5clear junk.h5" (no options, nonexisting file) +# "h5clear orig_h5clear_sec2_v3.h5" (no options, existing file) +# "h5clear -m" (valid 1 option, no file) +# "h5clear -s junk.h5" (valid 1 option, nonexisting file) +# "h5clear -m -s junk.h5" (valid 2 options, no file) +# "h5clear -m -s junk.h5" (valid 2 options, nonexisting file) +# "h5clear -m orig_h5clear_sec2_v2.h5" (valid 1 option, existing file, no cache image) +# "h5clear -s -m orig_h5clear_sec2_v0.h5" (valid 2 options, existing file, no cache image) +TOOLTEST_OUT "" -h "" h5clear_usage.ddl +TOOLTEST_OUT "" "" "" h5clear_usage.ddl +TOOLTEST_OUT junk.h5 "" "" h5clear_usage.ddl +TOOLTEST_OUT $TESTDIR/orig_h5clear_sec2_v3.h5 "" "" h5clear_usage.ddl +TOOLTEST_OUT "" -m "" h5clear_missing_file.ddl +TOOLTEST_OUT junk.h5 -s "" h5clear_open_fail.ddl +TOOLTEST_OUT "" -m -s h5clear_missing_file.ddl +TOOLTEST_OUT junk.h5 -m -s h5clear_open_fail.ddl +TOOLTEST_OUT $TESTDIR/orig_h5clear_sec2_v2.h5 -m "" h5clear_no_mdc_image.ddl +TOOLTEST_OUT $TESTDIR/orig_h5clear_sec2_v0.h5 -s -m h5clear_no_mdc_image.ddl # -# Initial file open fails -# After "h5clear" the file, the subsequent file open succeeds -TOOLTEST h5clear_sec2_v3.h5 $FAIL # -TOOLTEST h5clear_log_v3.h5 $FAIL # -TOOLTEST latest_h5clear_sec2_v3.h5 $FAIL +# The following are tests to verify the expected exit code from h5clear: +# "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) +# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear -k" (invalid 1 option, no file, fail exit code) +# "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) +# "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) +# "h5clear -m -k" (valid/invalid 2 options, nofile, fail exit code) +# "h5clear -l -m" (invalid/valid 2 options, nofile, fail exit code) +# "h5clear -m -k junk.h5" (valid/invalid 2 options, nonexisting file, fail exit code) +# "h5clear -l -m junk.h5" (invalid/valid 2 options, nonexisting file, fail exit code) +# "h5clear -m -k h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) +# "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) +TOOLTEST h5clear_mdc_image.h5 -m "" $SUCCEED +TOOLTEST "" --vers "" $SUCCEED +TOOLTEST "" -k "" $FAIL +TOOLTEST junk.h5 -k "" $FAIL +TOOLTEST h5clear_sec2_v2.h5 -l "" $FAIL +TOOLTEST "" -m -k $FAIL +TOOLTEST "" -l -m $FAIL +TOOLTEST junk.h5 -m -l $FAIL +TOOLTEST junk.h5 -l -m $FAIL +TOOLTEST h5clear_sec2_v0.h5 -m -l $FAIL +TOOLTEST h5clear_sec2_v0.h5 -l -m $FAIL # -TOOLTEST latest_h5clear_log_v3.h5 $FAIL # # -# File open succeeds because the library does not check status_flags for file with < v3 superblock -TOOLTEST h5clear_sec2_v0.h5 $SUCCEED -TOOLTEST h5clear_sec2_v2.h5 $SUCCEED +# h5clear_mdc_image.h5 already has cache image removed earlier, verify the expected warning from h5clear: +TOOLTEST_OUT $TESTDIR/mod_h5clear_mdc_image.h5 -m "" h5clear_no_mdc_image.ddl +TOOLTEST_OUT $TESTDIR/mod_h5clear_mdc_image.h5 -s -m h5clear_no_mdc_image.ddl +# +# +# +# The following are tests to verify the status_flags field is cleared properly: +OPEN_CHK h5clear_sec2_v3.h5 $FAIL +TOOLTEST h5clear_sec2_v3.h5 -s "" $SUCCEED +OPEN_CHK h5clear_sec2_v3.h5 $SUCCEED +# +OPEN_CHK h5clear_log_v3.h5 $FAIL +TOOLTEST h5clear_log_v3.h5 -s "" $SUCCEED +OPEN_CHK h5clear_log_v3.h5 $SUCCEED +# +OPEN_CHK latest_h5clear_sec2_v3.h5 $FAIL +TOOLTEST latest_h5clear_sec2_v3.h5 -s "" $SUCCEED +OPEN_CHK latest_h5clear_sec2_v3.h5 $SUCCEED +# +OPEN_CHK latest_h5clear_log_v3.h5 $FAIL +TOOLTEST latest_h5clear_log_v3.h5 -s "" $SUCCEED +OPEN_CHK latest_h5clear_log_v3.h5 $SUCCEED +# +# File open succeeds for the following 2 test files because the +# library does not check status_flags for files with < v3 superblock: +OPEN_CHK h5clear_sec2_v0.h5 $SUCCEED +TOOLTEST h5clear_sec2_v0.h5 -s "" $SUCCEED +OPEN_CHK h5clear_sec2_v0.h5 $SUCCEED +# +OPEN_CHK h5clear_sec2_v2.h5 $SUCCEED +TOOLTEST h5clear_sec2_v2.h5 -s "" $SUCCEED +OPEN_CHK h5clear_sec2_v2.h5 $SUCCEED +# +# # # Clean up test files if test -z "$HDF5_NOCLEANUP"; then rm -f h5clear_*.h5 latest_h5clear*.h5 fi +# +# +# +# Clean up temporary files/directories +CLEAN_TESTFILES_AND_TESTDIR if test $nerrors -eq 0 ; then echo "All $TESTNAME tests passed." diff --git a/tools/test/perform/sio_engine.c b/tools/test/perform/sio_engine.c index abf1fa6..2a5efb6 100644 --- a/tools/test/perform/sio_engine.c +++ b/tools/test/perform/sio_engine.c @@ -1064,6 +1064,7 @@ done: do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) { int ret_code = SUCCESS; + hid_t fcpl; switch (param->io_type) { case POSIXIO: @@ -1088,9 +1089,17 @@ do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) GOTOERROR(FAIL); } + fcpl = H5Pcreate(H5P_FILE_CREATE); + if(param->page_size) { + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1); + H5Pset_file_space_page_size(fcpl, param->page_size); + if(param->page_buffer_size) + H5Pset_page_buffer_size(fapl, param->page_buffer_size, 0, 0); + } + /* create the parallel file */ if (flags & (SIO_CREATE | SIO_WRITE)) { - fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); } else { fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, fapl); } diff --git a/tools/test/perform/sio_perf.c b/tools/test/perform/sio_perf.c index 5bae832..2201b99 100644 --- a/tools/test/perform/sio_perf.c +++ b/tools/test/perform/sio_perf.c @@ -100,7 +100,7 @@ static const char *progname = "h5perf_serial"; * It seems that only the options that accept additional information * such as dataset size (-e) require the colon next to it. */ -static const char *s_opts = "a:A:B:c:Cd:D:e:F:ghi:Imno:p:P:r:stT:v:wx:X:"; +static const char *s_opts = "a:A:B:b:c:Cd:D:e:F:G:ghi:Imno:p:P:r:stT:v:wx:X:"; static struct long_options l_opts[] = { { "align", require_arg, 'a' }, { "alig", require_arg, 'a' }, @@ -292,7 +292,8 @@ struct options { int h5_extendable; /* Perform the write tests only */ int verify; /* Verify data correctness */ vfdtype vfd; /* File driver */ - + size_t page_buffer_size; + size_t page_size; }; typedef struct _minmax { @@ -401,6 +402,8 @@ run_test_loop(struct options *opts) parms.h5_write_only = opts->h5_write_only; parms.verify = opts->verify; parms.vfd = opts->vfd; + parms.page_buffer_size = opts->page_buffer_size; + parms.page_size = opts->page_size; /* load multidimensional options */ parms.num_bytes = 1; @@ -865,6 +868,16 @@ report_parameters(struct options *opts) recover_size_and_print((long long)opts->buf_size[i], " "); HDfprintf(output, "\n"); + if(opts->page_size) { + HDfprintf(output, "Page Aggregation Enabled. Page size = %ld\n", opts->page_size); + if(opts->page_buffer_size) + HDfprintf(output, "Page Buffering Enabled. Page Buffer size = %ld\n", opts->page_buffer_size); + else + HDfprintf(output, "Page Buffering Disabled\n"); + } + else + HDfprintf(output, "Page Aggregation Disabled\n"); + HDfprintf(output, "Dimension access order="); for (i=0; i<rank; i++) recover_size_and_print((long long)opts->order[i], " "); @@ -941,6 +954,9 @@ parse_command_line(int argc, char *argv[]) cl_opts = (struct options *)HDmalloc(sizeof(struct options)); + cl_opts->page_buffer_size = 0; + cl_opts->page_size = 0; + cl_opts->output_file = NULL; cl_opts->io_types = 0; /* will set default after parsing options */ cl_opts->num_iters = 1; @@ -975,6 +991,12 @@ parse_command_line(int argc, char *argv[]) case 'a': cl_opts->h5_alignment = parse_size_directive(opt_arg); break; + case 'G': + cl_opts->page_size = parse_size_directive(opt_arg); + break; + case 'b': + cl_opts->page_buffer_size = parse_size_directive(opt_arg); + break; case 'A': { const char *end = opt_arg; diff --git a/tools/test/perform/sio_perf.h b/tools/test/perform/sio_perf.h index 7a7d708..5d9358d 100644 --- a/tools/test/perform/sio_perf.h +++ b/tools/test/perform/sio_perf.h @@ -67,6 +67,8 @@ typedef struct parameters_ { int h5_extendable; /* Make HDF5 dataset chunked */ int h5_write_only; /* Perform the write tests only */ int verify; /* Verify data correctness */ + size_t page_size; + size_t page_buffer_size; } parameters; typedef struct results_ { diff --git a/tools/testfiles/file_space.ddl b/tools/testfiles/file_space.ddl index bed5349..6b8aa47 100644 --- a/tools/testfiles/file_space.ddl +++ b/tools/testfiles/file_space.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_AGGR_VFD - FREE_SPACE_THRESHOLD 10 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_NONE + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 8192 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/file_space.h5 b/tools/testfiles/file_space.h5 Binary files differindex 425d0c2..d9e2890 100644 --- a/tools/testfiles/file_space.h5 +++ b/tools/testfiles/file_space.h5 diff --git a/tools/testfiles/tboot1.ddl b/tools/testfiles/tboot1.ddl index 4758948..73ae4e5 100644 --- a/tools/testfiles/tboot1.ddl +++ b/tools/testfiles/tboot1.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tboot2.ddl b/tools/testfiles/tboot2.ddl index d83b92b..504a634 100644 --- a/tools/testfiles/tboot2.ddl +++ b/tools/testfiles/tboot2.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tboot2A.ddl b/tools/testfiles/tboot2A.ddl index d83b92b..504a634 100644 --- a/tools/testfiles/tboot2A.ddl +++ b/tools/testfiles/tboot2A.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tboot2B.ddl b/tools/testfiles/tboot2B.ddl index d83b92b..504a634 100644 --- a/tools/testfiles/tboot2B.ddl +++ b/tools/testfiles/tboot2B.ddl @@ -9,8 +9,10 @@ SUPER_BLOCK { BTREE_RANK 16 BTREE_LEAF 4 ISTORE_K 32 - FILE_SPACE_STRATEGY H5F_FILE_SPACE_ALL - FREE_SPACE_THRESHOLD 1 + FILE_SPACE_STRATEGY H5F_FSPACE_STRATEGY_FSM_AGGR + FREE_SPACE_PERSIST FALSE + FREE_SPACE_SECTION_THRESHOLD 1 + FILE_SPACE_PAGE_SIZE 4096 USER_BLOCK { USERBLOCK_SIZE 0 } diff --git a/tools/testfiles/tdset_idx.ddl b/tools/testfiles/tdset_idx.ddl deleted file mode 100644 index 65d9f44..0000000 --- a/tools/testfiles/tdset_idx.ddl +++ /dev/null @@ -1,61 +0,0 @@ -############################# -Expected output for 'h5dump -p -H tdset_idx.h5' -############################# -HDF5 "tdset_idx.h5" { -GROUP "/" { - DATASET "dset_btree" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 20, 10 ) / ( 200, 100 ) } - STORAGE_LAYOUT { - CHUNKED ( 5, 5 ) - SIZE 800 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE 0 - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_filter" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } - STORAGE_LAYOUT { - CHUNKED ( 5, 5 ) - SIZE 200 (4.000:1 COMPRESSION) - } - FILTERS { - COMPRESSION DEFLATE { LEVEL 9 } - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE 0 - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } - DATASET "dset_fixed" { - DATATYPE H5T_STD_I32LE - DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } - STORAGE_LAYOUT { - CHUNKED ( 5, 5 ) - SIZE 800 - } - FILTERS { - NONE - } - FILLVALUE { - FILL_TIME H5D_FILL_TIME_IFSET - VALUE 0 - } - ALLOCATION_TIME { - H5D_ALLOC_TIME_INCR - } - } -} -} |