From 8c8dad1cc779566fc689a93b0d248299786e4ce7 Mon Sep 17 00:00:00 2001 From: Mike McGreevy Date: Thu, 24 Mar 2011 09:42:23 -0500 Subject: [svn-r20309] Purpose: Interim checkin of 'avoid truncate' feature to development branch. Description: Bringing initial changes in support of 'avoid truncate' feature to a branch off the trunk. While this feature is not yet complete, I didn't want to continue working solely from a local copy. Development will continue on the branch and be merged to the trunk after thorough review and testing. Tested: While I've added and run tests of new code changes thus far, no extensive regression testing has been performed as of yet. This will be done before merging back to trunk. --- configure | 5 +- configure.in | 1 + src/CMakeLists.txt | 1 + src/H5AC.c | 1 + src/H5F.c | 34 ++- src/H5FD.c | 2 +- src/H5FDcore.c | 2 +- src/H5FDdirect.c | 2 +- src/H5FDfamily.c | 2 +- src/H5FDlog.c | 2 +- src/H5FDmpio.c | 20 +- src/H5FDmpiposix.c | 69 ++++- src/H5FDmulti.c | 2 +- src/H5FDsec2.c | 2 +- src/H5FDstdio.c | 2 +- src/H5FDwindows.c | 2 +- src/H5Fpkg.h | 4 + src/H5Fprivate.h | 1 + src/H5Fsuper.c | 41 ++- src/H5Fsuper_cache.c | 194 ++++++++++--- src/H5O.c | 3 +- src/H5Oeoa.c | 319 ++++++++++++++++++++++ src/H5Omessage.c | 65 +++++ src/H5Opkg.h | 7 +- src/H5Oprivate.h | 5 +- src/H5Pfcpl.c | 77 ++++++ src/H5Ppublic.h | 2 + src/Makefile.am | 2 +- src/Makefile.in | 38 +-- test/CMakeLists.txt | 2 + test/Makefile.am | 6 +- test/Makefile.in | 83 +++--- test/trunc_detect.c | 554 ++++++++++++++++++++++++++++++++++++++ testpar/t_file.c | 162 +++++++++++ testpar/testphdf5.c | 3 + testpar/testphdf5.h | 1 + tools/lib/h5tools.h | 1 + tools/misc/CMakeLists.txt | 11 +- tools/misc/Makefile.am | 9 +- tools/misc/Makefile.in | 48 +++- tools/misc/h5extend.c | 264 ++++++++++++++++++ tools/misc/h5extend_gentest.c | 70 +++++ tools/misc/testh5extend.sh.in | 131 +++++++++ tools/testfiles/h5extend_exp1.ddl | 7 + tools/testfiles/h5extend_exp2.ddl | 7 + tools/testfiles/h5extend_exp3.ddl | 6 + tools/testfiles/h5extend_exp4.ddl | 7 + tools/testfiles/h5extend_exp5.ddl | 6 + tools/testfiles/h5extend_tfile.h5 | Bin 0 -> 1392 bytes 49 files changed, 2144 insertions(+), 141 deletions(-) create mode 100644 src/H5Oeoa.c create mode 100644 test/trunc_detect.c create mode 100644 tools/misc/h5extend.c create mode 100644 tools/misc/h5extend_gentest.c create mode 100644 tools/misc/testh5extend.sh.in create mode 100644 tools/testfiles/h5extend_exp1.ddl create mode 100644 tools/testfiles/h5extend_exp2.ddl create mode 100644 tools/testfiles/h5extend_exp3.ddl create mode 100644 tools/testfiles/h5extend_exp4.ddl create mode 100644 tools/testfiles/h5extend_exp5.ddl create mode 100644 tools/testfiles/h5extend_tfile.h5 diff --git a/configure b/configure index 54d7101..0118bf6 100755 --- a/configure +++ b/configure @@ -1,5 +1,5 @@ #! /bin/sh -# From configure.in Id: configure.in 19833 2010-11-19 21:34:29Z songyulu . +# From configure.in Id: configure.in 20135 2011-02-22 01:03:37Z koziol . # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.68 for HDF5 1.9.80. # @@ -29525,7 +29525,7 @@ if test -n "$TESTPARALLEL"; then fi fi -ac_config_files="$ac_config_files src/libhdf5.settings Makefile src/Makefile test/Makefile test/testcheck_version.sh test/testerror.sh test/H5srcdir_str.h test/testlibinfo.sh test/testlinks_env.sh testpar/Makefile testpar/testph5.sh perform/Makefile tools/Makefile tools/h5dump/Makefile tools/h5dump/testh5dump.sh tools/h5dump/testh5dumpxml.sh tools/h5ls/testh5ls.sh tools/h5import/Makefile tools/h5diff/Makefile tools/h5jam/Makefile tools/h5jam/testh5jam.sh tools/h5repack/Makefile tools/h5repack/h5repack.sh tools/h5ls/Makefile tools/h5copy/Makefile tools/lib/Makefile tools/misc/Makefile tools/misc/h5cc tools/misc/testh5repart.sh tools/h5stat/testh5stat.sh tools/h5stat/Makefile examples/Makefile examples/run-c-ex.sh examples/testh5cc.sh c++/Makefile c++/src/Makefile c++/src/h5c++ c++/test/Makefile c++/test/H5srcdir_str.h c++/examples/Makefile c++/examples/run-c++-ex.sh c++/examples/testh5c++.sh fortran/Makefile fortran/src/h5fc fortran/src/Makefile fortran/test/Makefile fortran/testpar/Makefile fortran/examples/Makefile fortran/examples/run-fortran-ex.sh fortran/examples/testh5fc.sh hl/Makefile hl/src/Makefile hl/test/Makefile hl/test/H5srcdir_str.h hl/tools/Makefile hl/tools/gif2h5/Makefile hl/examples/Makefile hl/examples/run-hlc-ex.sh hl/c++/Makefile hl/c++/src/Makefile hl/c++/test/Makefile hl/c++/examples/Makefile hl/c++/examples/run-hlc++-ex.sh hl/fortran/Makefile hl/fortran/src/Makefile hl/fortran/test/Makefile hl/fortran/examples/Makefile hl/fortran/examples/run-hlfortran-ex.sh" +ac_config_files="$ac_config_files src/libhdf5.settings Makefile src/Makefile test/Makefile test/testcheck_version.sh test/testerror.sh test/H5srcdir_str.h test/testlibinfo.sh test/testlinks_env.sh testpar/Makefile testpar/testph5.sh perform/Makefile tools/Makefile tools/h5dump/Makefile tools/h5dump/testh5dump.sh tools/h5dump/testh5dumpxml.sh tools/h5ls/testh5ls.sh tools/h5import/Makefile tools/h5diff/Makefile tools/h5jam/Makefile tools/h5jam/testh5jam.sh tools/h5repack/Makefile tools/h5repack/h5repack.sh tools/h5ls/Makefile tools/h5copy/Makefile tools/lib/Makefile tools/misc/Makefile tools/misc/h5cc tools/misc/testh5extend.sh tools/misc/testh5repart.sh tools/h5stat/testh5stat.sh tools/h5stat/Makefile examples/Makefile examples/run-c-ex.sh examples/testh5cc.sh c++/Makefile c++/src/Makefile c++/src/h5c++ c++/test/Makefile c++/test/H5srcdir_str.h c++/examples/Makefile c++/examples/run-c++-ex.sh c++/examples/testh5c++.sh fortran/Makefile fortran/src/h5fc fortran/src/Makefile fortran/test/Makefile fortran/testpar/Makefile fortran/examples/Makefile fortran/examples/run-fortran-ex.sh fortran/examples/testh5fc.sh hl/Makefile hl/src/Makefile hl/test/Makefile hl/test/H5srcdir_str.h hl/tools/Makefile hl/tools/gif2h5/Makefile hl/examples/Makefile hl/examples/run-hlc-ex.sh hl/c++/Makefile hl/c++/src/Makefile hl/c++/test/Makefile hl/c++/examples/Makefile hl/c++/examples/run-hlc++-ex.sh hl/fortran/Makefile hl/fortran/src/Makefile hl/fortran/test/Makefile hl/fortran/examples/Makefile hl/fortran/examples/run-hlfortran-ex.sh" cat >confcache <<\_ACEOF @@ -30808,6 +30808,7 @@ do "tools/lib/Makefile") CONFIG_FILES="$CONFIG_FILES tools/lib/Makefile" ;; "tools/misc/Makefile") CONFIG_FILES="$CONFIG_FILES tools/misc/Makefile" ;; "tools/misc/h5cc") CONFIG_FILES="$CONFIG_FILES tools/misc/h5cc" ;; + "tools/misc/testh5extend.sh") CONFIG_FILES="$CONFIG_FILES tools/misc/testh5extend.sh" ;; "tools/misc/testh5repart.sh") CONFIG_FILES="$CONFIG_FILES tools/misc/testh5repart.sh" ;; "tools/h5stat/testh5stat.sh") CONFIG_FILES="$CONFIG_FILES tools/h5stat/testh5stat.sh" ;; "tools/h5stat/Makefile") CONFIG_FILES="$CONFIG_FILES tools/h5stat/Makefile" ;; diff --git a/configure.in b/configure.in index 79bc1cf..2c2c1d5 100644 --- a/configure.in +++ b/configure.in @@ -4343,6 +4343,7 @@ AC_CONFIG_FILES([src/libhdf5.settings tools/lib/Makefile tools/misc/Makefile tools/misc/h5cc + tools/misc/testh5extend.sh tools/misc/testh5repart.sh tools/h5stat/testh5stat.sh tools/h5stat/Makefile diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 93370c0..2493fe5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -340,6 +340,7 @@ SET (H5O_SRCS ${HDF5_SRC_DIR}/H5Odrvinfo.c ${HDF5_SRC_DIR}/H5Odtype.c ${HDF5_SRC_DIR}/H5Oefl.c + ${HDF5_SRC_DIR}/H5Oeoa.c ${HDF5_SRC_DIR}/H5Ofill.c ${HDF5_SRC_DIR}/H5Ofsinfo.c ${HDF5_SRC_DIR}/H5Oginfo.c diff --git a/src/H5AC.c b/src/H5AC.c index cfb8a17..77ace4b 100644 --- a/src/H5AC.c +++ b/src/H5AC.c @@ -845,6 +845,7 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id) /* Attempt to flush all entries from rank 0 & Bcast clean list to other ranks */ if(H5AC_flush_entries(f) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.") + #endif /* H5_HAVE_PARALLEL */ /* Flush the cache */ diff --git a/src/H5F.c b/src/H5F.c index 21b9a19..b8e2a60 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -880,6 +880,8 @@ H5F_new(H5F_file_t *shared, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space strategy") if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &f->shared->fs_threshold) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get free-space section threshold") + if(H5P_get(plist, H5F_CRT_AVOID_TRUNCATE_NAME, &f->shared->avoid_truncate) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'avoid truncate' feature") /* Get the FAPL values to cache */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) @@ -1009,6 +1011,22 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) if(H5F_flush(f, dxpl_id, TRUE) < 0) HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache") +#ifdef H5_HAVE_PARALLEL + /* Depending on how entries in the cache were distributed amongst flushing + * processes, the superblock may have been given an EOF value that's not + * accurate. Check for that here, and mark the superblock as dirty if we + * need to re-flush it. Note this only matters if we're avoiding truncation, + * as otherwise we use the EOA value rather than EOF value when writing + * out the superblock. + */ + if (f->shared->avoid_truncate) + if((f->shared->flags & H5F_ACC_RDWR) && flush) + if (f->shared->sblock->eof_in_file != H5FD_get_eof(f->shared->lf)) + if (H5F_super_dirty(f) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTMARKDIRTY, HADDR_UNDEF, + "unable to mark superblock as dirty") +#endif /* H5_HAVE_PARALLEL */ + /* Release the external file cache */ if(f->shared->efc) { if(H5F_efc_destroy(f->shared->efc) < 0) @@ -1085,12 +1103,14 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close property list") - /* Only truncate the file on an orderly close, with write-access */ - if(f->closing && (H5F_ACC_RDWR & H5F_INTENT(f))) { - /* Truncate the file to the current allocated size */ - if(H5FD_truncate(f->shared->lf, dxpl_id, (unsigned)TRUE) < 0) - /* Push error, but keep going*/ - HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed") + if (!f->shared->avoid_truncate) { + /* Only truncate the file on an orderly close, with write-access */ + if(f->closing && (H5F_ACC_RDWR & H5F_INTENT(f))) { + /* Truncate the file to the current allocated size */ + if(H5FD_truncate(f->shared->lf, dxpl_id, (unsigned)TRUE) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed") + } /* end if */ } /* end if */ /* Close the file */ @@ -1301,7 +1321,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, * Read or write the file superblock, depending on whether the file is * empty or not. */ - if(0 == H5FD_get_eof(lf) && (flags & H5F_ACC_RDWR)) { + if(0 == (MAX(H5FD_get_eof(lf),H5FD_get_eoa(lf,H5FD_MEM_SUPER))) && (flags & H5F_ACC_RDWR)) { /* * We've just opened a fresh new file (or truncated one). We need * to create & write the superblock. diff --git a/src/H5FD.c b/src/H5FD.c index d0683e1..2e22a53 100644 --- a/src/H5FD.c +++ b/src/H5FD.c @@ -1612,7 +1612,7 @@ H5FDget_eof(H5FD_t *file) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer") /* The real work */ - if(HADDR_UNDEF == (ret_value = H5FD_get_eof(file))) + if(HADDR_UNDEF == (ret_value = MAX(H5FD_get_eof(file), H5FD_get_eoa(file, H5FD_MEM_SUPER)))) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eof request failed") /* (Note compensating for base address subtraction in internal routine) */ diff --git a/src/H5FDcore.c b/src/H5FDcore.c index b3cc84d..f8e0ccb 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -794,7 +794,7 @@ H5FD_core_get_eof(const H5FD_t *_file) FUNC_ENTER_NOAPI(H5FD_core_get_eof, HADDR_UNDEF) /* Set return value */ - ret_value=MAX(file->eof, file->eoa); + ret_value=file->eof; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 29fd84f..09f78ac 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -854,7 +854,7 @@ H5FD_direct_get_eof(const H5FD_t *_file) FUNC_ENTER_NOAPI(H5FD_direct_get_eof, HADDR_UNDEF) /* Set return value */ - ret_value=MAX(file->eof, file->eoa); + ret_value=file->eof; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index bd09ea4..5b526e0 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -1178,7 +1178,7 @@ H5FD_family_get_eof(const H5FD_t *_file) eof += ((unsigned)i)*file->memb_size; /* Set return value */ - ret_value = MAX(eof, file->eoa); + ret_value = eof; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 96df596..c927773 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -1025,7 +1025,7 @@ H5FD_log_get_eof(const H5FD_t *_file) FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5FD_log_get_eof) - FUNC_LEAVE_NOAPI(MAX(file->eof, file->eoa)) + FUNC_LEAVE_NOAPI(file->eof) } /* end H5FD_log_get_eof() */ diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index b382fb4..8587867 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -60,6 +60,7 @@ typedef struct H5FD_mpio_t { haddr_t eof; /*end-of-file marker */ haddr_t eoa; /*end-of-address marker */ haddr_t last_eoa; /* Last known end-of-address marker */ + haddr_t local_eof; /* Local end-of-file address for each process */ } H5FD_mpio_t; /* Private Prototypes */ @@ -1049,6 +1050,7 @@ H5FD_mpio_open(const char *name, unsigned flags, hid_t fapl_id, /* Set the size of the file (from library's perspective) */ file->eof = H5FD_mpi_MPIOff_to_haddr(size); + file->local_eof = file->eof; /* Set return value */ ret_value=(H5FD_t*)file; @@ -1842,8 +1844,16 @@ H5FD_mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, if(bytes_written != io_size) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") - /* Forget the EOF value (see H5FD_mpio_get_eof()) --rpm 1999-08-06 */ + /* Each process will keep track of its perceived EOF value locally, and + * ultimately we will reduce this value to the maximum amongst all + * processes, but until then keep the actual eof at HADDR_UNDEF just in + * case something bad happens before that point. (rather have a value + * we know is wrong sitting around rather than one that could only + * potentially be wrong.) */ file->eof = HADDR_UNDEF; + if ((bytes_written+addr)>file->local_eof) { + file->local_eof = addr+bytes_written; + } /* end if */ done: #ifdef H5FDmpio_DEBUG @@ -1875,6 +1885,7 @@ H5FD_mpio_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned closing) H5FD_mpio_t *file = (H5FD_mpio_t*)_file; int mpi_code; /* mpi return code */ herr_t ret_value = SUCCEED; + haddr_t max_eof; /* End-of-file value */ FUNC_ENTER_NOAPI(H5FD_mpio_flush, FAIL) @@ -1891,6 +1902,13 @@ H5FD_mpio_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned closing) HMPI_GOTO_ERROR(FAIL, "MPI_File_sync failed", mpi_code) } /* end if */ + /* Find the maximum 'EOF' value amongst all processes' locally tracked copies */ + if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&(file->local_eof), &max_eof, 1, HADDR_AS_MPI_TYPE, MPI_MAX, file->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + + /* Synchronize actual eof amongst all processes with max value reported */ + file->eof = max_eof; + done: #ifdef H5FDmpio_DEBUG if(H5FD_mpio_Debug[(int)'t']) diff --git a/src/H5FDmpiposix.c b/src/H5FDmpiposix.c index 7ca38cc..e7f28b1 100644 --- a/src/H5FDmpiposix.c +++ b/src/H5FDmpiposix.c @@ -92,6 +92,7 @@ typedef struct H5FD_mpiposix_t { haddr_t eof; /*end-of-file marker */ haddr_t eoa; /*end-of-address marker */ haddr_t last_eoa; /* Last known end-of-address marker */ + haddr_t local_eof; /* Local end-of-file address for each process */ haddr_t pos; /* Current file I/O position */ int op; /* Last file I/O operation */ hsize_t naccess; /* Number of (write) accesses to file */ @@ -190,6 +191,7 @@ static herr_t H5FD_mpiposix_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, size_t size, void *buf); static herr_t H5FD_mpiposix_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, const void *buf); +static herr_t H5FD_mpiposix_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned closing); static herr_t H5FD_mpiposix_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static int H5FD_mpiposix_mpi_rank(const H5FD_t *_file); static int H5FD_mpiposix_mpi_size(const H5FD_t *_file); @@ -230,7 +232,7 @@ static const H5FD_class_mpi_t H5FD_mpiposix_g = { H5FD_mpiposix_get_handle, /*get_handle */ H5FD_mpiposix_read, /*read */ H5FD_mpiposix_write, /*write */ - NULL, /*flush */ + H5FD_mpiposix_flush, /*flush */ H5FD_mpiposix_truncate, /*truncate */ NULL, /*lock */ NULL, /*unlock */ @@ -755,6 +757,7 @@ H5FD_mpiposix_open(const char *name, unsigned flags, hid_t fapl_id, /* Set the general file information */ file->fd = fd; file->eof = sb.st_size; + file->local_eof = file->eof; /* for _WIN32 support. _WIN32 'stat' does not have st_blksize and st_blksize is only used for the H5_HAVE_GPFS case */ @@ -1051,7 +1054,7 @@ H5FD_mpiposix_get_eof(const H5FD_t *_file) assert(H5FD_MPIPOSIX==file->pub.driver_id); /* Set return value */ - ret_value=MAX(file->eof,file->eoa); + ret_value=file->eof; done: FUNC_LEAVE_NOAPI(ret_value) @@ -1340,6 +1343,16 @@ H5FD_mpiposix_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, /* Update current last file I/O information */ file->pos = addr; file->op = OP_WRITE; + + /* Each process will keep track of its perceived EOF value locally, and + * ultimately we will reduce this value to the maximum amongst all + * processes, but until then keep the actual eof at HADDR_UNDEF just in + * case something bad happens before that point. (rather have a value + * we know is wrong sitting around rather than one that could only + * potentially be wrong.) */ + file->eof = HADDR_UNDEF; + if (file->pos > file->local_eof) + file->local_eof = file->pos; done: /* Check for error */ @@ -1371,6 +1384,57 @@ done: /*------------------------------------------------------------------------- + * Function: H5FD_mpiposix_flush + * + * Purpose: Makes sure that all data is on disk. This is collective. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Mike McGreevy + * January 30, 1998 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mpiposix_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned closing) +{ + H5FD_mpiposix_t *file = (H5FD_mpiposix_t*)_file; + int mpi_code; /* mpi return code */ + herr_t ret_value = SUCCEED; + haddr_t max_eof; /* End-of-file value */ + + FUNC_ENTER_NOAPI(H5FD_mpiposix_flush, FAIL) + + HDassert(file); + HDassert(H5FD_MPIPOSIX == file->pub.driver_id); + + /* This fuction doesn't do anything but sync the 'EOF' value + * amongst all processes. So, it's entirely misnamed. Yay! + * However, I need a good place (whilst collective) in the file driver + * layer to synchronize the 'EOF' value across processes after all data + * has hit disk. The MPIO file driver's flush callback was a nice + * place to do it but since the MPIPOSIX didn't (previously) have need + * for a flush callback, I created one to do just the sync task. + * In the future, we may want to create another file driver member + * function outside of the _flush call to do this? + */ + + /* Find the maximum 'EOF' value amongst all processes' locally tracked copies */ + if(MPI_SUCCESS != (mpi_code = MPI_Allreduce(&(file->local_eof), &max_eof, 1, HADDR_AS_MPI_TYPE, MPI_MAX, file->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + + /* Synchronize actual eof amongst all processes with max value reported */ + file->eof = max_eof; + +done: + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mpiposix_flush() */ + + +/*------------------------------------------------------------------------- * Function: H5FD_mpiposix_truncate * * Purpose: Makes sure that the true file size is the same (or larger) @@ -1432,6 +1496,7 @@ H5FD_mpiposix_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t UNUSED closi /* Update the 'last' eoa and eof values */ file->last_eoa = file->eoa; file->eof = file->eoa; + file->local_eof = file->eof; /* Reset last file I/O information */ file->pos = HADDR_UNDEF; diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 328b530..5d01b3f 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1664,7 +1664,7 @@ H5FD_multi_get_eof(const H5FD_t *_file) if (tmp_eoa>eoa) eoa = tmp_eoa; } END_MEMBERS; - return MAX(eoa, eof); + return eof; } diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 1153df7..2b91fba 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -624,7 +624,7 @@ H5FD_sec2_get_eof(const H5FD_t *_file) FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5FD_sec2_get_eof) - FUNC_LEAVE_NOAPI(MAX(file->eof, file->eoa)) + FUNC_LEAVE_NOAPI(file->eof) } /* end H5FD_sec2_get_eof() */ diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 9bc4161..900f723 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -710,7 +710,7 @@ H5FD_stdio_get_eof(const H5FD_t *_file) /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); - return(MAX(file->eof, file->eoa)); + return(file->eof); } diff --git a/src/H5FDwindows.c b/src/H5FDwindows.c index 47c7f2d..67f1106 100644 --- a/src/H5FDwindows.c +++ b/src/H5FDwindows.c @@ -666,7 +666,7 @@ H5FD_windows_get_eof(const H5FD_t *_file) FUNC_ENTER_NOAPI(H5FD_windows_get_eof, HADDR_UNDEF) /* Set return value */ - ret_value=MAX(file->eof, file->eoa); + ret_value=file->eof; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 837458e..4dfe9ae 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -189,6 +189,9 @@ typedef struct H5F_super_t { haddr_t driver_addr; /* File driver information block address */ haddr_t root_addr; /* Root group address */ H5G_entry_t *root_ent; /* Root group symbol table entry */ +#ifdef H5_HAVE_PARALLEL + haddr_t eof_in_file; /* Last known EOF stored in file's superblock */ +#endif /* H5_HAVE_PARALLEL */ } H5F_super_t; /* @@ -214,6 +217,7 @@ typedef struct H5F_file_t { unsigned sohm_nindexes; /* Number of shared messages indexes in the table */ unsigned long feature_flags; /* VFL Driver feature Flags */ haddr_t maxaddr; /* Maximum address for file */ + hbool_t avoid_truncate; /* Whether or not to avoid file truncation */ H5AC_t *cache; /* The object cache */ H5AC_cache_config_t diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 22d9acd..1d23f1c 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -358,6 +358,7 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_CRT_SHMSG_BTREE_MIN_NAME "shmsg_btree_min" /* Shared message B-tree minimum size */ #define H5F_CRT_FILE_SPACE_STRATEGY_NAME "file_space_strategy" /* File space handling strategy */ #define H5F_CRT_FREE_SPACE_THRESHOLD_NAME "free_space_threshold" /* Free space section threshold */ +#define H5F_CRT_AVOID_TRUNCATE_NAME "avoid_truncate" /* Avoid truncate */ diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 2510487..e25007f 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -126,7 +126,7 @@ H5F_locate_signature(H5FD_t *file, hid_t dxpl_id) FUNC_ENTER_NOAPI_NOINIT(H5F_locate_signature) /* Find the least N such that 2^N is larger than the file size */ - if(HADDR_UNDEF == (addr = H5FD_get_eof(file)) || HADDR_UNDEF == (eoa = H5FD_get_eoa(file, H5FD_MEM_SUPER))) + if(HADDR_UNDEF == (addr = MAX(H5FD_get_eof(file),H5FD_get_eoa(file,H5FD_MEM_SUPER))) || HADDR_UNDEF == (eoa = H5FD_get_eoa(file, H5FD_MEM_SUPER))) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, HADDR_UNDEF, "unable to obtain EOF/EOA value") for(maxpow = 0; addr; maxpow++) addr >>= 1; @@ -412,6 +412,9 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) sblock->ext_addr = HADDR_UNDEF; sblock->driver_addr = HADDR_UNDEF; sblock->root_addr = HADDR_UNDEF; +#ifdef H5_HAVE_PARALLEL + sblock->eof_in_file = HADDR_UNDEF; +#endif /* H5_HAVE_PARALLEL */ /* Get the shared file creation property list */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(f->shared->fcpl_id))) @@ -437,6 +440,9 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) else if(f->shared->fs_strategy != H5F_FILE_SPACE_STRATEGY_DEF || f->shared->fs_threshold != H5F_FREE_SPACE_THRESHOLD_DEF) super_vers = HDF5_SUPERBLOCK_VERSION_2; + /* Bump superblock version if the 'avoid truncate' feature is enabled */ + else if(f->shared->avoid_truncate) + super_vers = HDF5_SUPERBLOCK_VERSION_2; /* Check for non-default indexed storage B-tree internal 'K' value * and set the version # of the superblock to 1 if it is a non-default * value. @@ -538,6 +544,10 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) HDassert(super_vers >= HDF5_SUPERBLOCK_VERSION_2); need_ext = TRUE; } /* end if */ + /* Files that avoid truncation calls need to store the 'EOA' value in the + superblock extension */ + else if (f->shared->avoid_truncate) + need_ext = TRUE; /* If we're going to use a version of the superblock format which allows * for the superblock extension, check for non-default values to store * in it. @@ -614,20 +624,31 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) } /* end if */ /* Check for non-default free space settings */ - if(f->shared->fs_strategy != H5F_FILE_SPACE_STRATEGY_DEF || + if(f->shared->fs_strategy != H5F_FILE_SPACE_STRATEGY_DEF || f->shared->fs_threshold != H5F_FREE_SPACE_THRESHOLD_DEF) { - H5FD_mem_t type; /* Memory type for iteration */ - H5O_fsinfo_t fsinfo; /* Free space manager info message */ + H5FD_mem_t type; /* Memory type for iteration */ + H5O_fsinfo_t fsinfo; /* Free space manager info message */ - /* Write free-space manager info message to superblock extension object header if needed */ - fsinfo.strategy = f->shared->fs_strategy; - fsinfo.threshold = f->shared->fs_threshold; - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type-1] = HADDR_UNDEF; + /* Write free-space manager info message to superblock extension object header if needed */ + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.threshold = f->shared->fs_threshold; + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + fsinfo.fs_addr[type-1] = HADDR_UNDEF; if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update free-space info header message") - } /* end if */ + } /* end if */ + + /* Check if we need to store the 'EOA' value in the superblock extension */ + if (f->shared->avoid_truncate) { + haddr_t eoa; /* 'EOA' value */ + + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER))) + HGOTO_ERROR(H5E_IO, H5E_CANTINIT, HADDR_UNDEF, "unable to obtain EOA value") + + if(H5O_msg_create(&ext_loc, H5O_EOA_ID, H5O_MSG_FLAG_MARK_IF_UNKNOWN, H5O_UPDATE_TIME, &eoa, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update 'EOA' value header message") + } /* end if */ } /* end if */ done: diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 56f9219..2f1374e 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -119,8 +119,8 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) H5P_genplist_t *c_plist; /* File creation property list */ H5F_file_t *shared; /* shared part of `file' */ H5FD_t *lf; /* file driver part of `shared' */ - haddr_t stored_eoa; /*relative end-of-addr in file */ - haddr_t eof; /*end of file address */ + haddr_t stored_eof; /* stored end-of-file address in file */ + haddr_t eof; /* end of file address */ uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ const size_t fixed_size = H5F_SUPERBLOCK_FIXED_SIZE; /*fixed sizeof superblock */ @@ -277,7 +277,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) /* Remainder of "variable-sized" portion of superblock */ H5F_addr_decode(f, (const uint8_t **)&p, &sblock->base_addr/*out*/); H5F_addr_decode(f, (const uint8_t **)&p, &sblock->ext_addr/*out*/); - H5F_addr_decode(f, (const uint8_t **)&p, &stored_eoa/*out*/); + H5F_addr_decode(f, (const uint8_t **)&p, &stored_eof/*out*/); H5F_addr_decode(f, (const uint8_t **)&p, &sblock->driver_addr/*out*/); /* Allocate space for the root group symbol table entry */ @@ -299,10 +299,10 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) if(!H5F_addr_eq(base_addr, sblock->base_addr)) { /* Check if the superblock moved earlier in the file */ if(H5F_addr_lt(base_addr, sblock->base_addr)) - stored_eoa -= (sblock->base_addr - base_addr); + stored_eof -= (sblock->base_addr - base_addr); else /* The superblock moved later in the file */ - stored_eoa += (base_addr - sblock->base_addr); + stored_eof += (base_addr - sblock->base_addr); /* Adjust base address for offsets of the HDF5 data in the file */ sblock->base_addr = base_addr; @@ -409,7 +409,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) /* Base, superblock extension, end of file & root group object header addresses */ H5F_addr_decode(f, (const uint8_t **)&p, &sblock->base_addr/*out*/); H5F_addr_decode(f, (const uint8_t **)&p, &sblock->ext_addr/*out*/); - H5F_addr_decode(f, (const uint8_t **)&p, &stored_eoa/*out*/); + H5F_addr_decode(f, (const uint8_t **)&p, &stored_eof/*out*/); H5F_addr_decode(f, (const uint8_t **)&p, &sblock->root_addr/*out*/); /* Compute checksum for superblock */ @@ -429,10 +429,10 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) if(!H5F_addr_eq(base_addr, sblock->base_addr)) { /* Check if the superblock moved earlier in the file */ if(H5F_addr_lt(base_addr, sblock->base_addr)) - stored_eoa -= (sblock->base_addr - base_addr); + stored_eof -= (sblock->base_addr - base_addr); else /* The superblock moved later in the file */ - stored_eoa += (base_addr - sblock->base_addr); + stored_eof += (base_addr - sblock->base_addr); /* Adjust base address for offsets of the HDF5 data in the file */ sblock->base_addr = base_addr; @@ -464,19 +464,31 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) * possible is if the first file of a family of files was opened * individually. */ - if(HADDR_UNDEF == (eof = H5FD_get_eof(lf))) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to determine file size") - - /* (Account for the stored EOA being absolute offset -QAK) */ - if((eof + sblock->base_addr) < stored_eoa) + /* if(HADDR_UNDEF == (eof = H5FD_get_eof(lf))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to determine file size") */ + eof = H5FD_get_eof(lf); + /* (Account for the stored EOF being absolute offset -QAK) */ + if((eof + sblock->base_addr) < stored_eof) HGOTO_ERROR(H5E_FILE, H5E_TRUNCATED, NULL, "truncated file") +#ifdef H5_HAVE_PARALLEL + if (f->shared->avoid_truncate) + sblock->eof_in_file = stored_eof - sblock->base_addr; +#endif /* H5_HAVE_PARALLEL */ + /* * Tell the file driver how much address space has already been * allocated so that it knows how to allocate additional memory. + * + * (Account for the stored EOF being absolute offset -NAF) + * + * (We use the EOF value as in most cases this is the same as the EOA value + * at this point in time. In the 'avoid truncate' situation, we will + * later set the EOA value to the value loaded from the superblock extension, + * but we need to set the EOA to something significant prior to this so + * we can read in the superblock extension. -MAM) */ - /* (Account for the stored EOA being absolute offset -NAF) */ - if(H5FD_set_eoa(lf, H5FD_MEM_SUPER, stored_eoa - sblock->base_addr) < 0) + if(H5FD_set_eoa(lf, H5FD_MEM_SUPER, stored_eof - sblock->base_addr) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to set end-of-address marker for file") /* Read the file's superblock extension, if there is one. */ @@ -484,7 +496,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) H5O_loc_t ext_loc; /* "Object location" for superblock extension */ H5O_btreek_t btreek; /* v1 B-tree 'K' value message from superblock extension */ H5O_drvinfo_t drvinfo; /* Driver info message from superblock extension */ - size_t u; /* Local index variable */ + size_t u; /* Local index variable */ htri_t status; /* Status for message existing */ /* Sanity check - superblock extension should only be defined for @@ -495,7 +507,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) /* Check for superblock extension being located "outside" the stored * 'eoa' value, which can occur with the split/multi VFD. */ - if(H5F_addr_gt(sblock->ext_addr, stored_eoa)) { + if(H5F_addr_gt(sblock->ext_addr, stored_eof)) { /* Set the 'eoa' for the object header memory type large enough * to give some room for a reasonably sized superblock extension. * (This is _rather_ a kludge -QAK) @@ -505,8 +517,48 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) } /* end if */ /* Open the superblock extension */ - if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "unable to open file's superblock extension") + if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, NULL, "unable to open file's superblock extension") + + /* Check for the extension having an 'EOA' message */ + if((status = H5O_msg_exists(&ext_loc, H5O_EOA_ID, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to read object header") + if(status) { + haddr_t eoa; + f->shared->avoid_truncate = TRUE; + unsigned mesg_flags; + + /* Set 'avoid truncate' feature in the property list */ + if(H5P_set(c_plist, H5F_CRT_AVOID_TRUNCATE_NAME, &f->shared->avoid_truncate) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, NULL, "unable to set avoid truncate feature") + + /* Retrieve 'EOA' message */ + if(NULL == H5O_msg_read(&ext_loc, H5O_EOA_ID, &eoa, dxpl_id)) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "'EOA' message not present") + + /* Get the 'EOA' messages flags */ + if (H5O_msg_flags(&ext_loc, H5O_EOA_ID, &mesg_flags, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "Cannot retrieve object header message flags") + + /* If this message was previously accessed and unknown, then don't modify the 'EOA', as + * that means a version of the library that didn't understand the 'EOA' message modified + * the file, re-synchronizig the EOA/EOF values. There's no need to set the EOA here at all + * since we set it up above to the EOF value read from the file. Instead, re-write the 'EOA' + * message with the EOF value. (not actually sure we want to do this ... :? )*/ + if (mesg_flags & H5O_MSG_FLAG_WAS_UNKNOWN) { + /* Re-write EOA message with EOF value */ + /* + eoa = stored_eof - sblock->base_addr; + if(H5O_msg_write(&ext_loc, H5O_EOA_ID, H5O_MSG_FLAG_MARK_IF_UNKNOWN, H5O_UPDATE_TIME, &eoa, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message") */ + } /* end if */ + else { + /* Set 'EOA' value in file driver */ + if(H5FD_set_eoa(lf, H5FD_MEM_SUPER, eoa - sblock->base_addr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to set end-of-address marker for file") + + } /* end else */ + } /* end if */ /* Check for the extension having a 'driver info' message */ if((status = H5O_msg_exists(&ext_loc, H5O_DRVINFO_ID, dxpl_id)) < 0) @@ -538,7 +590,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) H5O_msg_reset(H5O_DRVINFO_ID, &drvinfo); } /* end else */ } /* end if */ - + /* Read in the shared OH message information if there is any */ if(H5SM_get_info(&ext_loc, c_plist, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read SOHM table information") @@ -630,6 +682,7 @@ static herr_t H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, H5F_super_t *sblock) { + herr_t status; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT(H5F_sblock_flush) @@ -642,7 +695,7 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, if(sblock->cache_info.is_dirty) { uint8_t buf[H5F_MAX_SUPERBLOCK_SIZE + H5F_MAX_DRVINFOBLOCK_SIZE]; /* Superblock & driver info blockencoding buffer */ uint8_t *p; /* Ptr into encoding buffer */ - haddr_t rel_eoa; /* Relative EOA for file */ + haddr_t rel_eof; /* Relative EOF for file */ size_t superblock_size; /* Size of superblock, in bytes */ size_t driver_size; /* Size of driver info block (bytes)*/ @@ -677,10 +730,22 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, *p++ = 0; /*reserved */ } /* end if */ + /* Encode the base address */ H5F_addr_encode(f, &p, sblock->base_addr); + + /* Encode the address of global free-space index */ H5F_addr_encode(f, &p, sblock->ext_addr); - rel_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER); - H5F_addr_encode(f, &p, (rel_eoa + sblock->base_addr)); + + /* Encode the end-of-file address. Note that at this point in time, + the EOF value itself may not be reflective of the file's size, as + we will eventually truncate the file to match the EOA value. As + such, use the EOA value in its place, knowing that the current EOF + value will ultimately match it. */ + if ((rel_eof = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + H5F_addr_encode(f, &p, (rel_eof + sblock->base_addr)); + + /* Encode the driver informaton block address */ H5F_addr_encode(f, &p, sblock->driver_addr); /* Encode the root group object entry, including the cached stab info */ @@ -731,11 +796,40 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, *p++ = (uint8_t)H5F_SIZEOF_SIZE(f); *p++ = sblock->status_flags; - /* Base, superblock extension & end of file addresses */ + /* Encode the base address */ H5F_addr_encode(f, &p, sblock->base_addr); + + /* Encode the address of the superblock extension */ H5F_addr_encode(f, &p, sblock->ext_addr); - rel_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER); - H5F_addr_encode(f, &p, (rel_eoa + sblock->base_addr)); + + /* Encode the end-of-file address */ + if(f->shared->avoid_truncate) { + /* If we're avoiding truncating the file, then the current + value of the 'EOF' address will reflect the file's size, so + we can use it here directly. Note, however, that if + in a parallel situation, H5FD_get_eof may return failure, as + individual processes only keep local copies of their perceived + eof values. For now, we'll flush the superblock a second time + once we synchronize the eof value amongst all processes, and + this will happen later (see H5F_dest). */ + rel_eof = H5FD_get_eof(f->shared->lf); +#ifndef H5_HAVE_PARALLEL + if (rel_eof == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to determine file size") +#else /* H5_HAVE_PARALLEL */ + sblock->eof_in_file = rel_eof; +#endif /* H5_HAVE_PARALLEL */ + H5F_addr_encode(f, &p, (rel_eof + sblock->base_addr)); + } else { + /* Otherwise, at this point in time, the EOF value itself may + not be reflective of the file's size, since we'll eventually + truncate it to match the EOA value. As such, use the EOA value + in its place, knowing that the current EOF value will + ultimately match it. */ + if ((rel_eof = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + H5F_addr_encode(f, &p, (rel_eof + sblock->base_addr)); + } /* end if */ /* Retrieve information for root group */ if(NULL == (root_oloc = H5G_oloc(f->shared->root_grp))) @@ -767,13 +861,45 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, /* Check for newer version of superblock format & superblock extension */ if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 && H5F_addr_defined(sblock->ext_addr)) { + H5O_loc_t ext_loc; /* "Object location" for superblock extension */ + + /* Open the superblock extension's object header */ + if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") + + /* Check for the 'EOA' message */ + if((status = H5O_msg_exists(&ext_loc, H5O_EOA_ID, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to read object header") + if(status) { + haddr_t eoa = HADDR_UNDEF; + + if ((eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the 'EOA' message is the same as the 'EOF' message, then + older versions of HDF5 can read the file provided they just + ignore this EOA message, so we write it with the 'mark if + unknown' flag. However, if the two values differ, then + older versions will be unable to correctly predict if the file + has been truncated, so we write this message with a 'fail if + unknown' flag'. */ + if (eoa == rel_eof) { + if(H5O_msg_write(&ext_loc, H5O_EOA_ID, H5O_MSG_FLAG_MARK_IF_UNKNOWN, H5O_UPDATE_TIME, &eoa, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message") + } /* end if */ + else { + if(H5O_msg_write(&ext_loc, H5O_EOA_ID, H5O_MSG_FLAG_FAIL_IF_UNKNOWN, H5O_UPDATE_TIME, &eoa, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message") + } /* end else */ + } /* end if */ + /* Check for ignoring the driver info for this file */ if(!H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) { + /* Check for driver info message */ H5_ASSIGN_OVERFLOW(driver_size, H5FD_sb_size(f->shared->lf), hsize_t, size_t); if(driver_size > 0) { H5O_drvinfo_t drvinfo; /* Driver info */ - H5O_loc_t ext_loc; /* "Object location" for superblock extension */ uint8_t dbuf[H5F_MAX_DRVINFOBLOCK_SIZE]; /* Driver info block encoding buffer */ /* Sanity check */ @@ -783,25 +909,25 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, if(H5FD_sb_encode(f->shared->lf, drvinfo.name, dbuf) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to encode driver information") - /* Open the superblock extension's object header */ - if(H5F_super_ext_open(f, sblock->ext_addr, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") - /* Write driver info information to the superblock extension */ drvinfo.len = driver_size; drvinfo.buf = dbuf; if(H5O_msg_write(&ext_loc, H5O_DRVINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &drvinfo, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message") - /* Close the superblock extension object header */ - if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") } /* end if */ + } /* end if */ + + /* Close the superblock extension object header */ + if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") + } /* end if */ /* Reset the dirty flag. */ sblock->cache_info.is_dirty = FALSE; + } /* end if */ if(destroy) diff --git a/src/H5O.c b/src/H5O.c index eb90a70..6f65f5d 100644 --- a/src/H5O.c +++ b/src/H5O.c @@ -126,7 +126,8 @@ const H5O_msg_class_t *const H5O_msg_class_g[] = { H5O_MSG_AINFO, /*0x0015 Attribute information */ H5O_MSG_REFCOUNT, /*0x0016 Object's ref. count */ H5O_MSG_FSINFO, /*0x0017 Free-space manager info message */ - H5O_MSG_UNKNOWN, /*0x0018 Placeholder for unknown message */ + H5O_MSG_EOA, /*0x0018 'EOA' message */ + H5O_MSG_UNKNOWN, /*0x0019 Placeholder for unknown message */ }; /* Declare a free list to manage the H5O_t struct */ diff --git a/src/H5Oeoa.c b/src/H5Oeoa.c new file mode 100644 index 0000000..85ba7ce --- /dev/null +++ b/src/H5Oeoa.c @@ -0,0 +1,319 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: Mike McGreevy + * December 9, 2010 + * + * Purpose: The EOA message. + */ + +#define H5O_PACKAGE /* Suppress error about including H5Opkg */ + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error Handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Opkg.h" /* Object Headers */ + +static void *H5O_eoa_new_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, + unsigned mesg_flags, unsigned *ioflags, const uint8_t *p); + +static herr_t H5O_eoa_new_encode(H5F_t *f, hbool_t disable_shared, uint8_t *p, const void *_mesg); + +static size_t H5O_eoa_new_size(const H5F_t *f, hbool_t disable_shared, const void *_mesg); + +static void *H5O_eoa_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, + unsigned mesg_flags, unsigned *ioflags, const uint8_t *p); + +static herr_t H5O_eoa_encode(H5F_t *f, hbool_t disable_shared, uint8_t *p, const void *_mesg); + +static void *H5O_eoa_copy(const void *_mesg, void *_dest); + +static size_t H5O_eoa_size(const H5F_t *f, hbool_t disable_shared, const void *_mesg); + +static herr_t H5O_eoa_reset(void *_mesg); + +static herr_t H5O_eoa_free(void *_mesg); + +static herr_t H5O_eoa_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE *stream, + int indent, int fwidth); + +/* This message derives from H5O message class */ +const H5O_msg_class_t H5O_MSG_EOA[1] = {{ + H5O_EOA_ID, /* Message ID number */ + "'EOA' value", /* Message name for debugging */ + sizeof(haddr_t), /* Native message size */ + 0, /* Messages are sharable? */ + H5O_eoa_decode, /* Decode message */ + H5O_eoa_encode, /* Encode message */ + H5O_eoa_copy, /* Copy the native value */ + H5O_eoa_size, /* Raw message size */ + H5O_eoa_reset, /* Free internal memory */ + H5O_eoa_free, /* Free method */ + NULL, /* File delete method */ + NULL, /* Link method */ + NULL, /* Set share method */ + NULL, /* Can share method */ + NULL, /* Pre copy native value to file */ + NULL, /* Copy native value to file */ + NULL, /* Post copy native value to file */ + NULL, /* get creation index */ + NULL, /* set creation index */ + H5O_eoa_debug /* Debug the message */ +}}; + +/* Current version of 'EOA' message */ +#define H5O_EOA_VERSION 0 + +/* Declare a free list to manage the haddr_t struct */ +H5FL_DEFINE(haddr_t); + + +/*------------------------------------------------------------------------- + * Function: H5O_eoa_decode + * + * Purpose: Decode an 'EOA' message a return a pointer to a + * new haddr_t value. + * + * Return: Success: PTR to a new message in native struct. + * Failure: NULL + * + * Programmer: Mike McGreevy + * December 9, 2010 + * + *------------------------------------------------------------------------- + */ +static void * +H5O_eoa_decode(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, H5O_t UNUSED *open_oh, + unsigned UNUSED mesg_flags, unsigned UNUSED *ioflags, const uint8_t *p) +{ + haddr_t *mesg = NULL; /* Native message */ + haddr_t tmp_eoa; /* Temporary EOA */ + void *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_eoa_decode); + + /* Check Arguments */ + HDassert(f); + HDassert(p); + + /* Version of the message */ + if(*p++ != H5O_EOA_VERSION) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") + + /* Get the 'EOA' message from the file */ + H5F_addr_decode(f, (const uint8_t **)&p, &tmp_eoa); + + /* The return value */ + if (NULL == (mesg = H5FL_MALLOC(haddr_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); + *mesg = (haddr_t)tmp_eoa; + + /* Set return value */ + ret_value = (void *)mesg; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5O_eoa_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_eoa_encode + * + * Purpose: Encode an 'EOA' message. + * + * Return: Success: Non-Negative + * Failure: Negative + * + * Programmer: Mike McGreevy + * December 9, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_eoa_encode(H5F_t UNUSED *f, hbool_t UNUSED disable_shared, uint8_t *p, const void *_mesg) +{ + const haddr_t *mesg = (const haddr_t *) _mesg; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_eoa_encode); + + /* Check Arguments */ + HDassert(f); + HDassert(p); + HDassert(mesg); + + /* Version */ + *p++ = H5O_EOA_VERSION; + + /* Encode 'EOA' Message */ + H5F_addr_encode(f, &p, *mesg); + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* end H5O_eoa_encode() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_eoa_copy + * + * Purpose: Copies a message from _MESG to _DEST, allocating _DEST if + * necessary. + * + * Return: Success: Ptr to _DEST + * Failure: NULL + * + * Programmer: Mike McGreevy + * December 9, 2010 + * + *------------------------------------------------------------------------- + */ +static void * +H5O_eoa_copy(const void *_mesg, void *_dest) +{ + const haddr_t *mesg = (const haddr_t *) _mesg; + haddr_t *dest = (haddr_t *)_dest; + void *ret_value; + + FUNC_ENTER_NOAPI_NOINIT(H5O_eoa_copy); + + /* Check Arguments */ + HDassert(mesg); + if (!dest && NULL == (dest = H5FL_MALLOC(haddr_t))) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); + + /* Copy */ + *dest = *mesg; + + /* Set return value */ + ret_value = dest; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5O_eoa_copy */ + + +/*------------------------------------------------------------------------- + * Function: H5O_eoa_size + * + * Purpose: Returns the size of the raw message in bytes. This function + * doesn't take into account alignment. + * + * Return: Success: Mesage data size in bytes w/o alignment. + * Failure: 0 + * + * Programmer: Mike McGreevy + * December 9, 2010 + * + *------------------------------------------------------------------------- + */ +static size_t +H5O_eoa_size(const H5F_t UNUSED * f, hbool_t UNUSED disable_shared, const void UNUSED * mesg) +{ + size_t ret_value; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_eoa_size); + + /* Check Arguments */ + HDassert(f); + HDassert(mesg); + + /* Determine Size */ + ret_value = 1 + /* 8-bit Version Number (int) */ + H5F_SIZEOF_ADDR(f); /* EOA Address (haddr_t) */ + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5O_eoa_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_eoa_reset + * + * Purpose: Frees resources within an 'EOA' message, but doesn't free + * the message itself. + * + * Return: Success: Non-Negative + * Failure: Negative + * + * Programmer: Mike McGreevy + * December 9, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_eoa_reset(void UNUSED *_mesg) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_eoa_reset); + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* H5O_eoa_reset */ + + +/*------------------------------------------------------------------------- + * Function: H5O_eoa_free + * + * Purpose: Frees the 'EOA' message + * + * Return: Success: Non-Negative + * Failure: Negative + * + * Programmer: Mike McGreevy + * December 9, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_eoa_free(void *mesg) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_eoa_free) + + HDassert(mesg); + + mesg = H5FL_FREE(haddr_t, mesg); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5O_eoa_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_eoa_debug + * + * Purpose: Prints debugging info for the 'EOA' message. + * + * Return: Success: Non-Negative + * Failure: Negative + * + * Programmer: Mike McGreevy + * December 9, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_eoa_debug(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_mesg, FILE *stream, + int indent, int fwidth) +{ + const haddr_t *mesg = (const haddr_t *)_mesg; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_eoa_debug); +HDassert(0); + + /* Check Arguments */ + HDassert(f); + HDassert(mesg); + HDassert(stream); + HDassert(indent >= 0); + HDassert(fwidth >= 0); + + HDfprintf(stream, "%*s%-*s %ld\n", indent, "", fwidth, "EOA value:", *mesg); + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* H5O_eoa_debug */ diff --git a/src/H5Omessage.c b/src/H5Omessage.c index 0dd4565..c6cbd9e 100644 --- a/src/H5Omessage.c +++ b/src/H5Omessage.c @@ -438,6 +438,71 @@ done: /*------------------------------------------------------------------------- + * Function: H5O_msg_flags + * + * Purpose: Retrieves the flags from a message in an object header. + * + * Return: Success: SUCCEED + * Failure: FAL + * + * Programmer: Mike McGreevy + * February 10, 2011 + *------------------------------------------------------------------------- + */ +herr_t +H5O_msg_flags(const H5O_loc_t *loc, unsigned type_id, unsigned *mesg_flags, hid_t dxpl_id) +{ + H5O_t *oh = NULL; /* Object header to use */ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + unsigned idx; /* Message's index in object header */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_TAG(H5O_msg_flags, dxpl_id, loc->addr, FAIL) + + /* check args */ + HDassert(loc); + HDassert(loc->file); + HDassert(H5F_addr_defined(loc->addr)); + HDassert(type_id < NELMTS(H5O_msg_class_g)); + + /* Get the object header */ + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC_READ))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + type = H5O_msg_class_g[type_id]; + HDassert(type); + + /* Scan through the messages looking for the right one */ + for(idx = 0; idx < oh->nmesgs; idx++) + if(type == oh->mesg[idx].type) + break; + + /* make sure we found the message */ + if(idx == oh->nmesgs) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found") + + /* + * Decode the message if necessary. If the message is shared then retrieve + * native message through the shared interface. + */ + H5O_LOAD_NATIVE(loc->file, dxpl_id, 0, oh, &(oh->mesg[idx]), NULL) + + /* + * The object header caches the native message (along with + * the raw message) so we must copy the native message before + * returning. + */ + *mesg_flags = oh->mesg[idx].flags; + +done: + if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* H5O_msg_flags */ + + +/*------------------------------------------------------------------------- * Function: H5O_msg_read * * Purpose: Reads a message from an object header and returns a pointer diff --git a/src/H5Opkg.h b/src/H5Opkg.h index 54e7233..d0e9cef 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -31,7 +31,7 @@ #define H5O_NMESGS 8 /*initial number of messages */ #define H5O_NCHUNKS 2 /*initial number of chunks */ #define H5O_MIN_SIZE 22 /* Min. obj header data size (must be big enough for a message prefix and a continuation message) */ -#define H5O_MSG_TYPES 25 /* # of types of messages */ +#define H5O_MSG_TYPES 26 /* # of types of messages */ #define H5O_MAX_CRT_ORDER_IDX 65535 /* Max. creation order index value */ /* Versions of object header structure */ @@ -504,7 +504,10 @@ H5_DLLVAR const H5O_msg_class_t H5O_MSG_REFCOUNT[1]; /* Free-space Manager Info message. (0x0017) */ H5_DLLVAR const H5O_msg_class_t H5O_MSG_FSINFO[1]; -/* Placeholder for unknown message. (0x0018) */ +/* 'EOA' message. (0x0018) */ +H5_DLLVAR const H5O_msg_class_t H5O_MSG_EOA[1]; + +/* Placeholder for unknown message. (0x0019) */ H5_DLLVAR const H5O_msg_class_t H5O_MSG_UNKNOWN[1]; diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 56901f3..51e3b0f 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -170,7 +170,8 @@ typedef struct H5O_copy_t { #define H5O_AINFO_ID 0x0015 /* Attribute info message. */ #define H5O_REFCOUNT_ID 0x0016 /* Reference count message. */ #define H5O_FSINFO_ID 0x0017 /* Free-space manager info message. */ -#define H5O_UNKNOWN_ID 0x0018 /* Placeholder message ID for unknown message. */ +#define H5O_EOA_ID 0x0018 /* 'EOA' value info message */ +#define H5O_UNKNOWN_ID 0x0019 /* Placeholder message ID for unknown message. */ /* (this should never exist in a file) */ @@ -663,6 +664,8 @@ H5_DLL herr_t H5O_msg_write_oh(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned type_id, unsigned mesg_flags, unsigned update_flags, void *mesg); H5_DLL void *H5O_msg_read(const H5O_loc_t *loc, unsigned type_id, void *mesg, hid_t dxpl_id); +H5_DLL herr_t H5O_msg_flags(const H5O_loc_t *loc, unsigned type_id, + unsigned *mesg_flags, hid_t dxpl_id); H5_DLL void *H5O_msg_read_oh(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned type_id, void *mesg); H5_DLL herr_t H5O_msg_reset(unsigned type_id, void *native); diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c index a0f2f89..3e0c35b 100644 --- a/src/H5Pfcpl.c +++ b/src/H5Pfcpl.c @@ -80,6 +80,9 @@ #define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF #define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t) #define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF +/* Definitions for avoiding file truncation */ +#define H5F_CRT_AVOID_TRUNCATE_SIZE sizeof(hbool_t) +#define H5F_CRT_AVOID_TRUNCATE_DEF (FALSE) /******************/ @@ -156,6 +159,7 @@ H5P_fcrt_reg_prop(H5P_genclass_t *pclass) unsigned sohm_list_max = H5F_CRT_SHMSG_LIST_MAX_DEF; unsigned sohm_btree_min = H5F_CRT_SHMSG_BTREE_MIN_DEF; unsigned file_space_strategy = H5F_CRT_FILE_SPACE_STRATEGY_DEF; + hbool_t avoid_truncate = H5F_CRT_AVOID_TRUNCATE_DEF; hsize_t free_space_threshold = H5F_CRT_FREE_SPACE_THRESHOLD_DEF; herr_t ret_value = SUCCEED; /* Return value */ @@ -206,6 +210,10 @@ H5P_fcrt_reg_prop(H5P_genclass_t *pclass) /* Register the free space section threshold */ if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, H5F_CRT_FREE_SPACE_THRESHOLD_SIZE, &free_space_threshold, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the avoid truncate property */ + if(H5P_register_real(pclass, H5F_CRT_AVOID_TRUNCATE_NAME, H5F_CRT_AVOID_TRUNCATE_SIZE, &avoid_truncate, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_fcrt_reg_prop() */ @@ -995,3 +1003,72 @@ done: FUNC_LEAVE_API(ret_value) } /* H5Pget_file_space() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_avoid_truncate + * + * Purpose: Gets whether or not the library avoids truncation calls + * by storing an EOA message in the superblock extension. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mike McGreevy + * January 26, 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_avoid_truncate(hid_t fcpl_id, hbool_t *avoid_truncate) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5Pget_avoid_truncate, FAIL) + H5TRACE2("e", "i*b", fcpl_id, avoid_truncate); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(fcpl_id, H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get the avoid truncate value */ + if(H5P_get(plist, H5F_CRT_AVOID_TRUNCATE_NAME, avoid_truncate) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get 'avoid truncate' property") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_avoid_truncate() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_avoid_truncate + * + * Purpose: Sets whether or not the library avoids truncation calls + * by storing an EOA message in the superblock extension. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mike McGreevy + * January 26, 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_avoid_truncate(hid_t fcpl_id, hbool_t avoid_truncate) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5Pset_avoid_truncate, FAIL) + H5TRACE2("e", "ib", fcpl_id, avoid_truncate); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(fcpl_id, H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set the avoid_truncate value */ + if(H5P_set(plist, H5F_CRT_AVOID_TRUNCATE_NAME, &avoid_truncate) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set 'avoid truncate' property") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_avoid_truncate() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 6867b63..5152c5d 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -247,6 +247,8 @@ H5_DLL herr_t H5Pset_shared_mesg_phase_change(hid_t plist_id, unsigned max_list, H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *min_btree); H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); +H5_DLL herr_t H5Pset_avoid_truncate(hid_t fcpl_id, hbool_t avoid_truncate); +H5_DLL herr_t H5Pget_avoid_truncate(hid_t fcpl_id, hbool_t *avoid_truncate); /* File access property list (FAPL) routines */ diff --git a/src/Makefile.am b/src/Makefile.am index ee0a3f5..e0cacf2 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -80,7 +80,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Ocont.c H5Ocopy.c H5Odbg.c H5Odrvinfo.c H5Odtype.c H5Oefl.c \ H5Ofill.c H5Ofsinfo.c H5Oginfo.c \ H5Olayout.c \ - H5Olinfo.c H5Olink.c H5Omessage.c H5Omtime.c \ + H5Olinfo.c H5Olink.c H5Omessage.c H5Omtime.c H5Oeoa.c \ H5Oname.c H5Onull.c H5Opline.c H5Orefcount.c \ H5Osdspace.c H5Oshared.c H5Ostab.c \ H5Oshmesg.c H5Otest.c H5Ounknown.c \ diff --git a/src/Makefile.in b/src/Makefile.in index 51003a0..262d8b5 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -129,23 +129,24 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \ H5Obtreek.lo H5Ocache.lo H5Ochunk.lo H5Ocont.lo H5Ocopy.lo \ H5Odbg.lo H5Odrvinfo.lo H5Odtype.lo H5Oefl.lo H5Ofill.lo \ H5Ofsinfo.lo H5Oginfo.lo H5Olayout.lo H5Olinfo.lo H5Olink.lo \ - H5Omessage.lo H5Omtime.lo H5Oname.lo H5Onull.lo H5Opline.lo \ - H5Orefcount.lo H5Osdspace.lo H5Oshared.lo H5Ostab.lo \ - H5Oshmesg.lo H5Otest.lo H5Ounknown.lo H5P.lo H5Pacpl.lo \ - H5Pdapl.lo H5Pdcpl.lo H5Pdeprec.lo H5Pdxpl.lo H5Pfapl.lo \ - H5Pfcpl.lo H5Pfmpl.lo H5Pgcpl.lo H5Pint.lo H5Plapl.lo \ - H5Plcpl.lo H5Pocpl.lo H5Pocpypl.lo H5Pstrcpl.lo H5Ptest.lo \ - H5R.lo H5Rdeprec.lo H5RC.lo H5RS.lo H5S.lo H5Sall.lo H5Sdbg.lo \ - H5Shyper.lo H5Smpio.lo H5Snone.lo H5Spoint.lo H5Sselect.lo \ - H5Stest.lo H5SL.lo H5SM.lo H5SMbtree2.lo H5SMcache.lo \ - H5SMmessage.lo H5SMtest.lo H5ST.lo H5T.lo H5Tarray.lo \ - H5Tbit.lo H5Tcommit.lo H5Tcompound.lo H5Tconv.lo H5Tcset.lo \ - H5Tdbg.lo H5Tdeprec.lo H5Tenum.lo H5Tfields.lo H5Tfixed.lo \ - H5Tfloat.lo H5Tinit.lo H5Tnative.lo H5Toffset.lo H5Toh.lo \ - H5Topaque.lo H5Torder.lo H5Tpad.lo H5Tprecis.lo H5Tstrpad.lo \ - H5Tvisit.lo H5Tvlen.lo H5TS.lo H5V.lo H5WB.lo H5Z.lo \ - H5Zdeflate.lo H5Zfletcher32.lo H5Znbit.lo H5Zshuffle.lo \ - H5Zszip.lo H5Zscaleoffset.lo H5Ztrans.lo + H5Omessage.lo H5Omtime.lo H5Oeoa.lo H5Oname.lo H5Onull.lo \ + H5Opline.lo H5Orefcount.lo H5Osdspace.lo H5Oshared.lo \ + H5Ostab.lo H5Oshmesg.lo H5Otest.lo H5Ounknown.lo H5P.lo \ + H5Pacpl.lo H5Pdapl.lo H5Pdcpl.lo H5Pdeprec.lo H5Pdxpl.lo \ + H5Pfapl.lo H5Pfcpl.lo H5Pfmpl.lo H5Pgcpl.lo H5Pint.lo \ + H5Plapl.lo H5Plcpl.lo H5Pocpl.lo H5Pocpypl.lo H5Pstrcpl.lo \ + H5Ptest.lo H5R.lo H5Rdeprec.lo H5RC.lo H5RS.lo H5S.lo \ + H5Sall.lo H5Sdbg.lo H5Shyper.lo H5Smpio.lo H5Snone.lo \ + H5Spoint.lo H5Sselect.lo H5Stest.lo H5SL.lo H5SM.lo \ + H5SMbtree2.lo H5SMcache.lo H5SMmessage.lo H5SMtest.lo H5ST.lo \ + H5T.lo H5Tarray.lo H5Tbit.lo H5Tcommit.lo H5Tcompound.lo \ + H5Tconv.lo H5Tcset.lo H5Tdbg.lo H5Tdeprec.lo H5Tenum.lo \ + H5Tfields.lo H5Tfixed.lo H5Tfloat.lo H5Tinit.lo H5Tnative.lo \ + H5Toffset.lo H5Toh.lo H5Topaque.lo H5Torder.lo H5Tpad.lo \ + H5Tprecis.lo H5Tstrpad.lo H5Tvisit.lo H5Tvlen.lo H5TS.lo \ + H5V.lo H5WB.lo H5Z.lo H5Zdeflate.lo H5Zfletcher32.lo \ + H5Znbit.lo H5Zshuffle.lo H5Zszip.lo H5Zscaleoffset.lo \ + H5Ztrans.lo libhdf5_la_OBJECTS = $(am_libhdf5_la_OBJECTS) libhdf5_la_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ @@ -513,7 +514,7 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Ocont.c H5Ocopy.c H5Odbg.c H5Odrvinfo.c H5Odtype.c H5Oefl.c \ H5Ofill.c H5Ofsinfo.c H5Oginfo.c \ H5Olayout.c \ - H5Olinfo.c H5Olink.c H5Omessage.c H5Omtime.c \ + H5Olinfo.c H5Olink.c H5Omessage.c H5Omtime.c H5Oeoa.c \ H5Oname.c H5Onull.c H5Opline.c H5Orefcount.c \ H5Osdspace.c H5Oshared.c H5Ostab.c \ H5Oshmesg.c H5Otest.c H5Ounknown.c \ @@ -842,6 +843,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Odrvinfo.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Odtype.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Oefl.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Oeoa.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Ofill.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Ofsinfo.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Oginfo.Plo@am__quote@ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ca975f9..6128562 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -245,6 +245,7 @@ ADD_TEST ( SET (H5_TESTS accum + trunc_detect lheap ohdr stab @@ -408,6 +409,7 @@ IF (HDF5_TEST_VFD) SET (H5_VFD_TESTS testhdf5 accum + trunc_detect lheap ohdr stab diff --git a/test/Makefile.am b/test/Makefile.am index 6f4a46a..93223c5 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -37,7 +37,7 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) # the library yet. Move them to the end so that their failure do not block # other current library code tests. TEST_PROG= testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ - pool accum hyperslab istore bittests dt_arith \ + pool accum trunc_detect hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ set_extent ttsafe \ @@ -63,7 +63,7 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version testmeta links_ BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \ gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \ gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread \ - gen_sizes_lheap + gen_sizes_lheap gen_eoamessage if BUILD_ALL_CONDITIONAL noinst_PROGRAMS=$(BUILD_ALL_PROGS) @@ -108,7 +108,7 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \ huge_chunks.h5 chunk_cache.h5 big_chunk.h5 chunk_expand.h5 \ copy_dcpl_newfile.h5 extend.h5 istore.h5 extlinks*.h5 frspace.h5 links*.h5 \ - sys_file1 tfile[1-5].h5 th5s[1-3].h5 lheap.h5 fheap.h5 ohdr.h5 \ + sys_file1 tfile[1-5].h5 th5s[1-3].h5 lheap.h5 trunc.h5 fheap.h5 ohdr.h5 \ stab.h5 extern_[1-3].h5 extern_[1-4][ab].raw gheap[0-4].h5 \ dt_arith[1-2] links.h5 links[0-6]*.h5 extlinks[0-15].h5 tmp \ big.data big[0-9][0-9][0-9][0-9][0-9].h5 \ diff --git a/test/Makefile.in b/test/Makefile.in index 98fa754..adb9db8 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -79,17 +79,18 @@ libh5test_la_OBJECTS = $(am_libh5test_la_OBJECTS) am__EXEEXT_1 = testhdf5$(EXEEXT) lheap$(EXEEXT) ohdr$(EXEEXT) \ stab$(EXEEXT) gheap$(EXEEXT) cache$(EXEEXT) cache_api$(EXEEXT) \ cache_tagging$(EXEEXT) pool$(EXEEXT) accum$(EXEEXT) \ - hyperslab$(EXEEXT) istore$(EXEEXT) bittests$(EXEEXT) \ - dt_arith$(EXEEXT) dtypes$(EXEEXT) dsets$(EXEEXT) \ - cmpd_dset$(EXEEXT) filter_fail$(EXEEXT) extend$(EXEEXT) \ - external$(EXEEXT) efc$(EXEEXT) objcopy$(EXEEXT) links$(EXEEXT) \ - unlink$(EXEEXT) big$(EXEEXT) mtime$(EXEEXT) fillval$(EXEEXT) \ - mount$(EXEEXT) flush1$(EXEEXT) flush2$(EXEEXT) \ - app_ref$(EXEEXT) enum$(EXEEXT) set_extent$(EXEEXT) \ - ttsafe$(EXEEXT) getname$(EXEEXT) vfd$(EXEEXT) ntypes$(EXEEXT) \ - dangle$(EXEEXT) dtransform$(EXEEXT) reserved$(EXEEXT) \ - cross_read$(EXEEXT) freespace$(EXEEXT) mf$(EXEEXT) \ - farray$(EXEEXT) earray$(EXEEXT) btree2$(EXEEXT) fheap$(EXEEXT) + trunc_detect$(EXEEXT) hyperslab$(EXEEXT) istore$(EXEEXT) \ + bittests$(EXEEXT) dt_arith$(EXEEXT) dtypes$(EXEEXT) \ + dsets$(EXEEXT) cmpd_dset$(EXEEXT) filter_fail$(EXEEXT) \ + extend$(EXEEXT) external$(EXEEXT) efc$(EXEEXT) \ + objcopy$(EXEEXT) links$(EXEEXT) unlink$(EXEEXT) big$(EXEEXT) \ + mtime$(EXEEXT) fillval$(EXEEXT) mount$(EXEEXT) flush1$(EXEEXT) \ + flush2$(EXEEXT) app_ref$(EXEEXT) enum$(EXEEXT) \ + set_extent$(EXEEXT) ttsafe$(EXEEXT) getname$(EXEEXT) \ + vfd$(EXEEXT) ntypes$(EXEEXT) dangle$(EXEEXT) \ + dtransform$(EXEEXT) reserved$(EXEEXT) cross_read$(EXEEXT) \ + freespace$(EXEEXT) mf$(EXEEXT) farray$(EXEEXT) earray$(EXEEXT) \ + btree2$(EXEEXT) fheap$(EXEEXT) am__EXEEXT_2 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \ gen_cross$(EXEEXT) gen_deflate$(EXEEXT) gen_filters$(EXEEXT) \ gen_new_array$(EXEEXT) gen_new_fill$(EXEEXT) \ @@ -97,7 +98,8 @@ am__EXEEXT_2 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \ gen_new_super$(EXEEXT) gen_noencoder$(EXEEXT) \ gen_nullspace$(EXEEXT) gen_udlinks$(EXEEXT) \ space_overflow$(EXEEXT) gen_filespace$(EXEEXT) \ - gen_specmetaread$(EXEEXT) gen_sizes_lheap$(EXEEXT) + gen_specmetaread$(EXEEXT) gen_sizes_lheap$(EXEEXT) \ + gen_eoamessage$(EXEEXT) PROGRAMS = $(noinst_PROGRAMS) accum_SOURCES = accum.c accum_OBJECTS = accum.$(OBJEXT) @@ -231,6 +233,10 @@ gen_deflate_SOURCES = gen_deflate.c gen_deflate_OBJECTS = gen_deflate.$(OBJEXT) gen_deflate_LDADD = $(LDADD) gen_deflate_DEPENDENCIES = libh5test.la $(LIBHDF5) +gen_eoamessage_SOURCES = gen_eoamessage.c +gen_eoamessage_OBJECTS = gen_eoamessage.$(OBJEXT) +gen_eoamessage_LDADD = $(LDADD) +gen_eoamessage_DEPENDENCIES = libh5test.la $(LIBHDF5) gen_filespace_SOURCES = gen_filespace.c gen_filespace_OBJECTS = gen_filespace.$(OBJEXT) gen_filespace_LDADD = $(LDADD) @@ -371,6 +377,10 @@ testmeta_SOURCES = testmeta.c testmeta_OBJECTS = testmeta.$(OBJEXT) testmeta_LDADD = $(LDADD) testmeta_DEPENDENCIES = libh5test.la $(LIBHDF5) +trunc_detect_SOURCES = trunc_detect.c +trunc_detect_OBJECTS = trunc_detect.$(OBJEXT) +trunc_detect_LDADD = $(LDADD) +trunc_detect_DEPENDENCIES = libh5test.la $(LIBHDF5) am_ttsafe_OBJECTS = ttsafe.$(OBJEXT) ttsafe_dcreate.$(OBJEXT) \ ttsafe_error.$(OBJEXT) ttsafe_cancel.$(OBJEXT) \ ttsafe_acreate.$(OBJEXT) @@ -404,29 +414,30 @@ SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c bittests.c \ earray.c efc.c enum.c err_compat.c error_test.c extend.c \ external.c farray.c fheap.c fillval.c filter_fail.c flush1.c \ flush2.c freespace.c gen_bad_ohdr.c gen_bogus.c gen_cross.c \ - gen_deflate.c gen_filespace.c gen_filters.c gen_new_array.c \ - gen_new_fill.c gen_new_group.c gen_new_mtime.c gen_new_super.c \ - gen_noencoder.c gen_nullspace.c gen_sizes_lheap.c \ - gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \ - istore.c lheap.c links.c links_env.c mf.c mount.c mtime.c \ - ntypes.c objcopy.c ohdr.c pool.c reserved.c set_extent.c \ - space_overflow.c stab.c tcheck_version.c $(testhdf5_SOURCES) \ - testmeta.c $(ttsafe_SOURCES) unlink.c vfd.c + gen_deflate.c gen_eoamessage.c gen_filespace.c gen_filters.c \ + gen_new_array.c gen_new_fill.c gen_new_group.c gen_new_mtime.c \ + gen_new_super.c gen_noencoder.c gen_nullspace.c \ + gen_sizes_lheap.c gen_specmetaread.c gen_udlinks.c getname.c \ + gheap.c hyperslab.c istore.c lheap.c links.c links_env.c mf.c \ + mount.c mtime.c ntypes.c objcopy.c ohdr.c pool.c reserved.c \ + set_extent.c space_overflow.c stab.c tcheck_version.c \ + $(testhdf5_SOURCES) testmeta.c trunc_detect.c \ + $(ttsafe_SOURCES) unlink.c vfd.c DIST_SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c \ bittests.c btree2.c cache.c cache_api.c cache_tagging.c \ cmpd_dset.c cross_read.c dangle.c dsets.c dt_arith.c \ dtransform.c dtypes.c earray.c efc.c enum.c err_compat.c \ error_test.c extend.c external.c farray.c fheap.c fillval.c \ filter_fail.c flush1.c flush2.c freespace.c gen_bad_ohdr.c \ - gen_bogus.c gen_cross.c gen_deflate.c gen_filespace.c \ - gen_filters.c gen_new_array.c gen_new_fill.c gen_new_group.c \ - gen_new_mtime.c gen_new_super.c gen_noencoder.c \ - gen_nullspace.c gen_sizes_lheap.c gen_specmetaread.c \ - gen_udlinks.c getname.c gheap.c hyperslab.c istore.c lheap.c \ - links.c links_env.c mf.c mount.c mtime.c ntypes.c objcopy.c \ - ohdr.c pool.c reserved.c set_extent.c space_overflow.c stab.c \ - tcheck_version.c $(testhdf5_SOURCES) testmeta.c \ - $(ttsafe_SOURCES) unlink.c vfd.c + gen_bogus.c gen_cross.c gen_deflate.c gen_eoamessage.c \ + gen_filespace.c gen_filters.c gen_new_array.c gen_new_fill.c \ + gen_new_group.c gen_new_mtime.c gen_new_super.c \ + gen_noencoder.c gen_nullspace.c gen_sizes_lheap.c \ + gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \ + istore.c lheap.c links.c links_env.c mf.c mount.c mtime.c \ + ntypes.c objcopy.c ohdr.c pool.c reserved.c set_extent.c \ + space_overflow.c stab.c tcheck_version.c $(testhdf5_SOURCES) \ + testmeta.c trunc_detect.c $(ttsafe_SOURCES) unlink.c vfd.c ETAGS = etags CTAGS = ctags am__tty_colors = \ @@ -715,7 +726,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog accum.h5 cmpd_dset.h5 \ huge_chunks.h5 chunk_cache.h5 big_chunk.h5 chunk_expand.h5 \ copy_dcpl_newfile.h5 extend.h5 istore.h5 extlinks*.h5 \ frspace.h5 links*.h5 sys_file1 tfile[1-5].h5 th5s[1-3].h5 \ - lheap.h5 fheap.h5 ohdr.h5 stab.h5 extern_[1-3].h5 \ + lheap.h5 trunc.h5 fheap.h5 ohdr.h5 stab.h5 extern_[1-3].h5 \ extern_[1-4][ab].raw gheap[0-4].h5 dt_arith[1-2] links.h5 \ links[0-6]*.h5 extlinks[0-15].h5 tmp big.data \ big[0-9][0-9][0-9][0-9][0-9].h5 stdio.h5 sec2.h5 \ @@ -747,7 +758,7 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) # the library yet. Move them to the end so that their failure do not block # other current library code tests. TEST_PROG = testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ - pool accum hyperslab istore bittests dt_arith \ + pool accum trunc_detect hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ set_extent ttsafe \ @@ -765,7 +776,7 @@ TEST_PROG = testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ BUILD_ALL_PROGS = gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \ gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \ gen_nullspace gen_udlinks space_overflow gen_filespace gen_specmetaread \ - gen_sizes_lheap + gen_sizes_lheap gen_eoamessage # The libh5test library provides common support code for the tests. @@ -979,6 +990,9 @@ gen_cross$(EXEEXT): $(gen_cross_OBJECTS) $(gen_cross_DEPENDENCIES) gen_deflate$(EXEEXT): $(gen_deflate_OBJECTS) $(gen_deflate_DEPENDENCIES) @rm -f gen_deflate$(EXEEXT) $(LINK) $(gen_deflate_OBJECTS) $(gen_deflate_LDADD) $(LIBS) +gen_eoamessage$(EXEEXT): $(gen_eoamessage_OBJECTS) $(gen_eoamessage_DEPENDENCIES) + @rm -f gen_eoamessage$(EXEEXT) + $(LINK) $(gen_eoamessage_OBJECTS) $(gen_eoamessage_LDADD) $(LIBS) gen_filespace$(EXEEXT): $(gen_filespace_OBJECTS) $(gen_filespace_DEPENDENCIES) @rm -f gen_filespace$(EXEEXT) $(LINK) $(gen_filespace_OBJECTS) $(gen_filespace_LDADD) $(LIBS) @@ -1078,6 +1092,9 @@ testhdf5$(EXEEXT): $(testhdf5_OBJECTS) $(testhdf5_DEPENDENCIES) testmeta$(EXEEXT): $(testmeta_OBJECTS) $(testmeta_DEPENDENCIES) @rm -f testmeta$(EXEEXT) $(LINK) $(testmeta_OBJECTS) $(testmeta_LDADD) $(LIBS) +trunc_detect$(EXEEXT): $(trunc_detect_OBJECTS) $(trunc_detect_DEPENDENCIES) + @rm -f trunc_detect$(EXEEXT) + $(LINK) $(trunc_detect_OBJECTS) $(trunc_detect_LDADD) $(LIBS) ttsafe$(EXEEXT): $(ttsafe_OBJECTS) $(ttsafe_DEPENDENCIES) @rm -f ttsafe$(EXEEXT) $(LINK) $(ttsafe_OBJECTS) $(ttsafe_LDADD) $(LIBS) @@ -1128,6 +1145,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_bogus.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_cross.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_deflate.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_eoamessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_filespace.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_filters.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_new_array.Po@am__quote@ @@ -1179,6 +1197,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tmisc.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/trefer.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/trefstr.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/trunc_detect.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tselect.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tskiplist.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsohm.Po@am__quote@ diff --git a/test/trunc_detect.c b/test/trunc_detect.c new file mode 100644 index 0000000..654450a --- /dev/null +++ b/test/trunc_detect.c @@ -0,0 +1,554 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/********************************************************************* +* +* Test Name: trunc_detect +* +* Purpose: Tests the library's methods of truncation detection. +* +* Developer: Mike McGreevy +* January 31, 2011 +* +*********************************************************************/ + +#define H5F_PACKAGE +#define FILENAME "trunc_detect.h5" +#define MULTIPARTNAME "trunc_detect.h5-s.h5" +#define SPLITPARTNAME "trunc_detect.h5-m.h5" +#define FAMILYPARTNAME "trunc_detect.h500000.h5" + +#include "hdf5.h" +#include "testhdf5.h" +#include "H5srcdir.h" +#include "H5Iprivate.h" +#include "H5FDprivate.h" +#include "H5Fpkg.h" + +/* Test Function Definitions */ +int test_standard_file_truncation(hid_t fapl); +int test_avoid_truncate_1(hid_t fapl); +int test_avoid_truncate_2(hid_t fapl); + + +/*------------------------------------------------------------------------- + * + * Function: main + * + * Purpose: Tests the library's methods of truncation detection. + * + * Return: SUCCEED or FAIL. + * + * Programmer: Mike McGreevy + * January 31, 2011 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + /* Variable Declarations */ + unsigned nerrors = 0; /* Number of errors */ + hid_t fapl = -1; /* file access property list */ + const char * env_h5_drvr; + + /* Get current driver */ + env_h5_drvr = HDgetenv("HDF5_DRIVER"); + + /* Skip tests with multi, split, and family drivers (for now ... ) */ + if ((env_h5_drvr != NULL) && + ((!HDstrcmp(env_h5_drvr,"multi")) || + (!HDstrcmp(env_h5_drvr,"split")) || + (!HDstrcmp(env_h5_drvr,"log")) || + (!HDstrcmp(env_h5_drvr,"family")))) { + + HDfprintf(stdout, "Truncation Detection Test skipped with %s file driver\n", env_h5_drvr); + + } /* end if */ + else { + + /* Open the HDF5 Library */ + H5open(); + + /* Create a fapl */ + fapl = h5_fileaccess(); + + /* Run tests on truncation detection methods */ + HDfprintf(stdout, "Running tests on library's truncation detection methods:\n"); + if (!nerrors) nerrors += test_standard_file_truncation(fapl); + if (!nerrors) nerrors += test_avoid_truncate_1(fapl); + if (!nerrors) nerrors += test_avoid_truncate_2(fapl); + + } /* end else */ + + return(nerrors > 0); +} /* main() */ + + +/*------------------------------------------------------------------------- + * + * Function: test_standard_file_truncation + * + * Purpose: Tests the library's standard file truncation detection + * mechanism (i.e., on file close, the library truncates + * the file to match the EOA value, and on file open then checks + * to see that the file size and stored EOF values are the same). + * + * Return: SUCCEED or FAIL. + * + * Programmer: Mike McGreevy + * January 31, 2011 + * + *------------------------------------------------------------------------- + */ +int test_standard_file_truncation(hid_t fapl) +{ + /* Variables */ + hid_t fid,did,sid = -1; /* Object Descriptors */ + H5F_t * f = NULL; /* Internal File Pointer */ + haddr_t eoa,eof = HADDR_UNDEF; /* End of File/Allocation values */ + char filename[1024]; /* File Name */ + int filesize; + hbool_t v = FALSE; + + /* Testing Message */ + TESTING("truncating file to match EOA during file close"); + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME, fapl, filename, (size_t)1024); + + /* Create a file */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR; + + /* Get internal file pointer */ + if ( NULL == (f = (H5F_t *)H5I_object(fid)) ) TEST_ERROR; + + /* Create dataspace for dataset */ + if ((sid = H5Screate(H5S_SCALAR)) < 0) TEST_ERROR; + + /* Create dataset */ + if ((did = H5Dcreate2(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; + + /* Close dataset, dataspace, and file */ + if (H5Dclose(did) < 0) TEST_ERROR; + if (H5Sclose(sid) < 0) TEST_ERROR; + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Re-open file */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Unlink the dataset */ + if (H5Ldelete(fid, "Dataset", H5P_DEFAULT) < 0) TEST_ERROR; + + /* Determine EOA and EOF values */ + if ((eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + TEST_ERROR; + if ((eof = H5FD_get_eof(f->shared->lf)) == HADDR_UNDEF) TEST_ERROR; + + /* make sure the values are as different */ + if (eoa == eof) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* now get file size. */ + if ((filesize = h5_get_file_size(filename, fapl)) < 0) TEST_ERROR; + + /* File size should now match previous EOA value */ + if (eoa != (unsigned)filesize) TEST_ERROR; + + /* Re-open file */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Determine EOA and EOF values */ + if ((eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + TEST_ERROR; + if ((eof = H5FD_get_eof(f->shared->lf)) == HADDR_UNDEF) TEST_ERROR; + + /* All values should now be in alignment. */ + if (eoa != eof) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Extending the file should be fine. Add a byte to the end of the + file, and make sure HDF5 doesn't care. (it will, however, write + over the byte). */ + if (truncate(filename, (off_t)filesize+1) != 0) TEST_ERROR; + /* Make sure we can open the file without problem */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Manually truncate the file by a byte, rendering it unreadable by HDF5 */ + if (truncate(filename, (off_t)filesize-1) != 0) TEST_ERROR; + + /* Try to re-open file: this should fail, as the file has been truncated */ + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + } H5E_END_TRY; + if (fid >= 0) TEST_ERROR; + + /* Delete test file */ + HDremove(filename); + + /* Return */ + PASSED(); + return SUCCEED; +error: + return FAIL; +} /* test_standard_file_truncation */ + + +/*------------------------------------------------------------------------- + * + * Function: test_avoid_truncate_1 + * + * Purpose: Tests the library's method of stored the EOA in the superblock + * in order to detect file truncation rather than truncating the + * file to match the EOA on file close. + * + * This test verifies that the 'avoid truncate' feature can + * be enabled via setting the latest format flag. It also verifies + * that file truncation is detected correctly when a file is + * closed with its EOA value less than its EOF. (i.e., the file + * is not truncated down in size at file close). + * + * Return: SUCCEED or FAIL. + * + * Programmer: Mike McGreevy + * January 31, 2011 + * + *------------------------------------------------------------------------- + */ +int test_avoid_truncate_1(hid_t fapl) +{ + /* Variables */ + hid_t fcpl,fid,did,sid = -1; /* Object Descriptors */ + H5F_t * f = NULL; /* Internal File Pointer */ + haddr_t eoa,eof,eoa_new,eof_new = HADDR_UNDEF; /* End of File/Allocation values */ + char filename[1024]; /* File Name */ + int filesize; + hbool_t avoid_truncate; + hbool_t v = FALSE; + + /* Testing Message */ + TESTING("avoiding truncation when EOA < EOF at file close"); + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME, fapl, filename, (size_t)1024); + +#if 0 + /* Enable latest format (which should in turn enable 'avoid truncate') */ + if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, + H5F_LIBVER_LATEST) < 0) TEST_ERROR; + + /* Create a file */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; +#else + /* Create a file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) TEST_ERROR; + + /* Check to see that 'avoid truncate' feature is disabled by default */ + if (H5Pget_avoid_truncate(fcpl, &avoid_truncate) < 0) TEST_ERROR; + if (avoid_truncate != FALSE) TEST_ERROR; + + /* Enable 'avoid truncate' feature */ + if (H5Pset_avoid_truncate(fcpl, TRUE) < 0) TEST_ERROR; + + /* Verify that retrieval gets new value for 'avoid truncate' feature */ + if (H5Pget_avoid_truncate(fcpl, &avoid_truncate) < 0) TEST_ERROR; + if (avoid_truncate != TRUE) TEST_ERROR; + + /* Create a file */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR; +#endif + + /* Get internal file pointer */ + if ( NULL == (f = (H5F_t *)H5I_object(fid)) ) TEST_ERROR; + + /* Create dataspace for dataset */ + if ((sid = H5Screate(H5S_SCALAR)) < 0) TEST_ERROR; + + /* Create dataset */ + if ((did = H5Dcreate2(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; + + /* Determine EOA and EOF values */ + if ((eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + TEST_ERROR; + + if ((eof = H5FD_get_eof(f->shared->lf)) == HADDR_UNDEF) TEST_ERROR; + + /* Close property list, dataset, dataspace, and file */ + if (H5Dclose(did) < 0) TEST_ERROR; + if (H5Sclose(sid) < 0) TEST_ERROR; + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Re-open file */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Get this file's creation properties */ + if ((fcpl = H5Fget_create_plist(fid)) < 0) TEST_ERROR; + + /* Verify that 'avoid truncate' feature is detected as enabled */ + if (H5Pget_avoid_truncate(fcpl, &avoid_truncate) < 0) TEST_ERROR; + if (avoid_truncate != TRUE) TEST_ERROR; + + /* Close the fcpl */ + if (H5Pclose(fcpl) < 0) TEST_ERROR; + + /* Unlink the dataset, reducing the 'EOA' value (but not EOF) */ + if (H5Ldelete(fid, "Dataset", H5P_DEFAULT) < 0) TEST_ERROR; + + /* Determine EOA and EOF values */ + if ((eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + TEST_ERROR; + if ((eof = H5FD_get_eof(f->shared->lf)) == HADDR_UNDEF) TEST_ERROR; + + /* make sure the values are as expected */ + /* We want to be testing the case when EOA < EOF at file close */ + if (eoa >= eof) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* now get file size. */ + if ((filesize = h5_get_file_size(filename, fapl)) < 0) TEST_ERROR; + + /* File size should be greater than previous 'EOA' value, as there should + not have been a truncate call during file close */ + if (eoa >= (unsigned)filesize) TEST_ERROR; + + /* also verify that filesize is equal to previously retrieved 'EOF' value */ + if (filesize != eof) TEST_ERROR; + + /* Re-open file */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Determine EOA and EOF values */ + if ((eoa_new = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + TEST_ERROR; + if ((eof_new = H5FD_get_eof(f->shared->lf)) == HADDR_UNDEF) TEST_ERROR; + + /* Values should not have changed since before we closed the file */ + if (eoa != eoa_new) TEST_ERROR; + if (eof != eof_new) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Extending the file should be fine. Add a byte to the end of the + file, and make sure HDF5 doesn't care. (it will, however, write + over the byte). */ + if (truncate(filename, (off_t)filesize+1) != 0) TEST_ERROR; + + /* Make sure we can open the file without problem */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Manually truncate the file by a byte, rendering it unreadable by HDF5 */ + if (truncate(filename, (off_t)filesize-1) != 0) TEST_ERROR; + + /* Try to re-open file: this should fail, as the file has been truncated */ + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + } H5E_END_TRY; + if (fid >= 0) TEST_ERROR; + + /* Delete test file */ + HDremove(filename); + + /* Return */ + PASSED(); + return SUCCEED; +error: + return FAIL; +} /* test_avoid_truncate_1 */ + + +/*------------------------------------------------------------------------- + * + * Function: test_avoid_truncate_2 + * + * Purpose: Tests the library's method of stored the EOA in the superblock + * in order to detect fiel truncation rather than truncating the + * file to match the EOA on file close. + * + * This test verifies that the 'avoid truncate' feature can + * be enabled and retrieved via its associated H5P API. It + * also verifies that file truncation is detected correctly when + * a file is closed with its EOA value greater than its EOF. + * (i.e., the file is not extended in size to match the EOA + * at file close). + * + * Return: SUCCEED or FAIL. + * + * Programmer: Mike McGreevy + * January 31, 2011 + * + *------------------------------------------------------------------------- + */ +int test_avoid_truncate_2(hid_t fapl) +{ + /* Variables */ + hid_t fcpl,fid,did,sid = -1; /* Object Descriptors */ + H5F_t * f = NULL; /* Internal File Pointer */ + haddr_t eoa,eof,eoa_new,eof_new = HADDR_UNDEF; /* End of File/Allocation values */ + int filesize; + hbool_t avoid_truncate; + hbool_t v = TRUE; + hsize_t dims[4] = {150, 15, 5, 5}; + hsize_t start[4] = {0,0,0,0}; /* Starting location of hyperslab */ + hsize_t count[4] = {1,1,1,1}; /* Element count of hyperslab */ + uint16_t *wbuf; /* buffer to write to disk */ + char filename[1024]; /* File Name */ + herr_t ret; /* Generic return value */ + + /* Testing Message */ + TESTING("avoiding truncation when EOA > EOF at file close"); + + /* Allocate write buffer */ + if ((wbuf = (uint16_t *)HDmalloc(sizeof(uint16_t) * dims[0] * dims[1] + * dims[2] * dims[3])) == NULL) TEST_ERROR; + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME, fapl, filename, (size_t)1024); + + /* Create a file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) TEST_ERROR; + + /* Check to see that 'avoid truncate' feature is disabled by default */ + if (H5Pget_avoid_truncate(fcpl, &avoid_truncate) < 0) TEST_ERROR; + if (avoid_truncate != FALSE) TEST_ERROR; + + /* Enable 'avoid truncate' feature */ + if (H5Pset_avoid_truncate(fcpl, TRUE) < 0) TEST_ERROR; + + /* Verify that retrieval gets new value for 'avoid truncate' feature */ + if (H5Pget_avoid_truncate(fcpl, &avoid_truncate) < 0) TEST_ERROR; + if (avoid_truncate != TRUE) TEST_ERROR; + + /* Create a file that avoids truncation */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) TEST_ERROR; + + /* Close the fcpl */ + if (H5Pclose(fcpl) < 0) TEST_ERROR; + + /* Get internal file pointer */ + if ( NULL == (f = (H5F_t *)H5I_object(fid)) ) TEST_ERROR; + + /* Create dataspace for dataset */ + if ((sid = H5Screate_simple(4, dims, NULL)) < 0) TEST_ERROR; + + /* Select contiguous hyperslab for disk dataset */ + if (H5Sselect_hyperslab(sid,H5S_SELECT_SET,start,NULL,count,NULL) < 0) TEST_ERROR; + + /* Create a dataset */ + if ((did = H5Dcreate2(fid, "Dataset", H5T_STD_U16LE, sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; + + /* Write selection to disk */ + if (H5Dwrite(did, H5T_NATIVE_USHORT, sid, sid, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR; + + /* Free memory buffer */ + HDfree(wbuf); + + /* Close Dataspace and Dataset */ + if (H5Sclose(sid) < 0) TEST_ERROR; + if (H5Dclose(did) < 0) TEST_ERROR; + + /* Retrieve EOA and EOF files. Make sure we set up a situation where EOA > + * EOF prior to file close. */ + if ((eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + TEST_ERROR; + if ((eof = H5FD_get_eof(f->shared->lf)) == HADDR_UNDEF) TEST_ERROR; + if (eoa <= eof) TEST_ERROR; + + /* Close the file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* now get file size. */ + if ((filesize = h5_get_file_size(filename, fapl)) < 0) TEST_ERROR; + + /* Verify that file size is equal to previously retrieved EOF value (i.e., + * file was not truncated to the EOA value */ + if (filesize != eof) TEST_ERROR; + + /* Re-open file */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Determine EOA and EOF values */ + if ((eoa_new = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + TEST_ERROR; + if ((eof_new = H5FD_get_eof(f->shared->lf)) == HADDR_UNDEF) TEST_ERROR; + + /* Verify that values were properly set up to be as they were prior to + * file close */ + if (eoa != eoa_new) TEST_ERROR; + if (eof != eof_new) TEST_ERROR; + + /* Verify that EOA is still larger than EOF */ + if(eoa_new <= eof_new) TEST_ERROR; + + /* Get this file's creation properties */ + if ((fcpl = H5Fget_create_plist(fid)) < 0) TEST_ERROR; + + /* Verify that 'avoid truncate' feature is detected as enabled */ + if (H5Pget_avoid_truncate(fcpl, &avoid_truncate) < 0) TEST_ERROR; + if (avoid_truncate != TRUE) TEST_ERROR; + + /* Close the fcpl */ + if (H5Pclose(fcpl) < 0) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Extending the file should be fine. Add a byte to the end of the + * file, and make sure HDF5 doesn't care. (it will, however, write + * over the byte). */ + if (truncate(filename, (off_t)filesize+1) != 0) TEST_ERROR; + + /* Make sure we can open the file without problem */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; + + /* Close file */ + if (H5Fclose(fid) < 0) TEST_ERROR; + + /* Manually truncate the file by a byte, rendering it unreadable by HDF5 */ + if (truncate(filename, (off_t)filesize-1) != 0) TEST_ERROR; + + /* Try to re-open file: this should fail, as the file has been truncated */ + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + } H5E_END_TRY; + if (fid >= 0) TEST_ERROR; + + /* Delete test file */ + HDremove(filename); + + /* Return */ + PASSED(); + return SUCCEED; +error: + return FAIL; +} /* test_avoid_truncate_2 */ + diff --git a/testpar/t_file.c b/testpar/t_file.c index 5cd3e9d..1bd6c2d 100644 --- a/testpar/t_file.c +++ b/testpar/t_file.c @@ -19,6 +19,12 @@ #include "testphdf5.h" +#define H5F_PACKAGE +#include "hdf5.h" +#include "H5Iprivate.h" +#include "H5FDprivate.h" +#include "H5Fpkg.h" + /* * test file access by communicator besides COMM_WORLD. * Split COMM_WORLD into two, one (even_comm) contains the original @@ -95,4 +101,160 @@ test_split_comm_access(void) VRFY((mrc==MPI_SUCCESS), "final MPI_Barrier succeeded"); } + +/*------------------------------------------------------------------------- + * + * Function: test_avoid_truncation + * + * Purpose: Tests the library's method of stored the EOA in the superblock + * in order to detect file truncation rather than truncating the + * file to match the EOA on file close, whilst in parallel. + * + * This test verifies that the 'avoid truncate' feature can + * be enabled via setting the latest format flag. It also verifies + * that file truncation is detected correctly when a file is + * closed with its EOA value differs from its EOF. + * + * Return: SUCCEED or FAIL. + * + * Programmer: Mike McGreevy + * March 17, 2011 + * + *------------------------------------------------------------------------- + */ +void +test_avoid_truncation(void) +{ + int mpi_size, mpi_rank; + H5F_t * f = NULL; /* Internal File Pointer */ + int mrc; + hid_t sid,did,fid; /* file IDs */ + haddr_t eoa,eof = HADDR_UNDEF; /* End of File/Allocation values */ + hid_t fcpl,acc_tpl; /* File access properties */ + hbool_t use_gpfs = FALSE; /* Use GPFS hints */ + herr_t ret; /* generic return value */ + hid_t status; + hbool_t avoid_truncate; + int filesize; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + const char *filename; + + filename = GetTestParameters(); + + if (VERBOSE_MED) + printf("Avoid Truncate test on file %s\n", + filename); + + /* set up MPI parameters */ + MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + + int sub_mpi_rank; /* rank in the sub-comm */ + MPI_Comm_rank(comm,&sub_mpi_rank); + + /* setup file access template */ + acc_tpl = create_faccess_plist(comm, info, facc_type, use_gpfs); + VRFY((acc_tpl >= 0), ""); + + /* Create a file creation property list */ + fcpl = H5Pcreate(H5P_FILE_CREATE); + VRFY((fcpl >= 0), ""); + + /* Enable 'avoid truncate' feature */ + status = H5Pset_avoid_truncate(fcpl, TRUE); + VRFY((status >= 0), ""); + + /* create the file collectively */ + fid=H5Fcreate(filename,H5F_ACC_TRUNC,fcpl,acc_tpl); + VRFY((fid >= 0), "H5Fcreate succeeded"); + + /* Get the internal file pointer */ + f = (H5F_t *)H5I_object(fid); + VRFY((fid != NULL), ""); + + /* Create dataspace for dataset */ + sid = H5Screate(H5S_SCALAR); + VRFY((sid >= 0), "H5Screate succeeded"); + + /* Create dataset */ + did = H5Dcreate2(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + VRFY((did >= 0), "H5Dcreate succeeded"); + + /* Determine EOA and EOF values */ + eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER); + VRFY((eoa != HADDR_UNDEF), ""); + + eof = H5FD_get_eof(f->shared->lf); + VRFY((eof != HADDR_UNDEF), ""); + + /* Make sure EOA/EOF are not the same */ + VRFY((eoa != eof),""); + + /* Close the property list, dataset, dataspace, and file */ + status = H5Dclose(did); + VRFY((status >= 0), ""); + status = H5Sclose(sid); + VRFY((status >= 0), ""); + status = H5Pclose(fcpl); + VRFY((status >= 0), ""); + status = H5Fclose(fid); + VRFY((status >= 0), ""); + + /* open the file collectively */ + fid=H5Fopen(filename,H5F_ACC_RDONLY,acc_tpl); + VRFY((fid >= 0), ""); + + /* Get this file's creation properties */ + fcpl = H5Fget_create_plist(fid); + VRFY((fcpl >= 0), ""); + + /* Verify that 'avoid truncate' feature is detected as enabled */ + H5Pget_avoid_truncate(fcpl, &avoid_truncate); + VRFY((avoid_truncate == 1), ""); + + /* Close the fcpl */ + status = H5Pclose(fcpl); + VRFY((status >= 0), ""); + + /* Determine EOA and EOF values */ + eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_SUPER); + VRFY((eoa != HADDR_UNDEF), ""); + + eof = H5FD_get_eof(f->shared->lf); + VRFY((eof != HADDR_UNDEF), ""); + + /* Make sure values are not equal */ + VRFY((eoa != eof), ""); + + /* close the file */ + status=H5Fclose(fid); + VRFY((status >= 0), ""); + + /* Manually truncate the file, rendering it unreadable by HDF5 */ + status = truncate(filename, (off_t)eof-1); + VRFY((status == 0),""); + + /* Try to re-open file: this should fail, as the file has been truncated */ + H5E_BEGIN_TRY { + fid = H5Fopen(filename, H5F_ACC_RDWR, acc_tpl); + } H5E_END_TRY; + VRFY((fid < 0),""); + + /* Release file-access template */ + ret=H5Pclose(acc_tpl); + VRFY((ret >= 0), ""); + + /* Cleanup */ + mrc = MPI_Barrier(MPI_COMM_WORLD); + VRFY((mrc==MPI_SUCCESS), "pre-file removal MPI_Barrier succeeded"); + /* delete the test file */ + if (sub_mpi_rank == 0){ + mrc = MPI_File_delete((char *)filename, info); + } + mrc = MPI_Barrier(MPI_COMM_WORLD); + VRFY((mrc==MPI_SUCCESS), "final MPI_Barrier succeeded"); + +} /* test_avoid_truncation */ diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index 8b24f87..5ca5060 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -356,6 +356,9 @@ int main(int argc, char **argv) AddTest("posixdup", test_fapl_mpiposix_dup, NULL, "fapl_mpiposix duplicate", NULL); + AddTest("trunc", test_avoid_truncation, NULL, + "file avoiding truncation", PARATESTFILE); + AddTest("split", test_split_comm_access, NULL, "dataset using split communicators", PARATESTFILE); diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index 2b83047..d32edaa 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -212,6 +212,7 @@ void independent_group_read(void); void test_fapl_mpio_dup(void); void test_fapl_mpiposix_dup(void); void test_split_comm_access(void); +void test_avoid_truncation(void); void dataset_writeInd(void); void dataset_writeAll(void); void extend_writeInd(void); diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index d08edf7..47c6652 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -24,6 +24,7 @@ #include "hdf5.h" #include "h5tools_error.h" +#include "H5private.h" #define ESCAPE_HTML 1 #define OPT(X,S) ((X) ? (X) : (S)) diff --git a/tools/misc/CMakeLists.txt b/tools/misc/CMakeLists.txt index 935239d..acaf7e5 100644 --- a/tools/misc/CMakeLists.txt +++ b/tools/misc/CMakeLists.txt @@ -23,10 +23,15 @@ ADD_EXECUTABLE (h5mkgrp ${HDF5_TOOLS_MISC_SOURCE_DIR}/h5mkgrp.c) TARGET_NAMING (h5mkgrp ${LIB_TYPE}) TARGET_LINK_LIBRARIES (h5mkgrp ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +ADD_EXECUTABLE (h5extend ${HDF5_TOOLS_MISC_SOURCE_DIR}/h5extend.c) +H5_NAMING (h5extend ${LIB_TYPE}) +TARGET_LINK_LIBRARIES (h5extend ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) + SET (H5_DEP_EXECUTABLES h5debug h5repart h5mkgrp + h5extend ) ############################################################################## @@ -43,7 +48,9 @@ IF (BUILD_TESTING) ADD_EXECUTABLE (h5repart_gentest ${HDF5_TOOLS_MISC_SOURCE_DIR}/h5repart_gentest.c) TARGET_NAMING (h5repart_gentest ${LIB_TYPE}) TARGET_LINK_LIBRARIES (h5repart_gentest ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) - #ADD_TEST (NAME h5repart_gentest COMMAND $) + ADD_EXECUTABLE (h5extend_gentest ${HDF5_TOOLS_MISC_SOURCE_DIR}/h5extend_gentest.c) + H5_NAMING (h5extend_gentest ${LIB_TYPE}) + TARGET_LINK_LIBRARIES (h5extend_gentest ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) ENDIF (HDF5_BUILD_GENERATORS AND NOT BUILD_SHARED_LIBS) ADD_EXECUTABLE (h5repart_test ${HDF5_TOOLS_MISC_SOURCE_DIR}/repart_test.c) @@ -266,7 +273,7 @@ ENDIF (BUILD_TESTING) #----------------------------------------------------------------------------- INSTALL ( TARGETS - h5debug h5repart h5mkgrp + h5debug h5repart h5mkgrp h5extend RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR}/tools COMPONENT diff --git a/tools/misc/Makefile.am b/tools/misc/Makefile.am index 11c86fa..835a819 100644 --- a/tools/misc/Makefile.am +++ b/tools/misc/Makefile.am @@ -24,21 +24,22 @@ include $(top_srcdir)/config/commence.am INCLUDES=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib #test script and program -TEST_PROG=h5repart_gentest talign -TEST_SCRIPT=testh5repart.sh $(srcdir)/testh5mkgrp.sh +TEST_PROG=h5repart_gentest h5extend_gentest talign +TEST_SCRIPT=testh5repart.sh $(srcdir)/testh5mkgrp.sh testh5extend.sh check_PROGRAMS=$(TEST_PROG) repart_test check_SCRIPTS=$(TEST_SCRIPT) SCRIPT_DEPEND=h5repart$(EXEEXT) h5mkgrp$(EXEEXT) # These are our main targets, the tools -bin_PROGRAMS=h5debug h5repart h5mkgrp +bin_PROGRAMS=h5debug h5repart h5mkgrp h5extend bin_SCRIPTS=h5redeploy # Add h5debug, h5repart, and h5mkgrp specific linker flags here h5debug_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) h5repart_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) h5mkgrp_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) +h5extend_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) # Tell automake to clean h5redeploy script CLEANFILES=h5redeploy @@ -49,7 +50,7 @@ CLEANFILES=h5redeploy CHECK_CLEANFILES+=*.h5 ../testfiles/fst_family*.h5 ../testfiles/scd_family*.h5 # These were generated by configure. Remove them only when distclean. -DISTCLEANFILES=h5cc testh5repart.sh +DISTCLEANFILES=h5cc testh5repart.sh testh5extend.sh # All programs rely on hdf5 library and h5tools library LDADD=$(LIBH5TOOLS) $(LIBHDF5) diff --git a/tools/misc/Makefile.in b/tools/misc/Makefile.in index cb770a1..57e3983 100644 --- a/tools/misc/Makefile.in +++ b/tools/misc/Makefile.in @@ -53,11 +53,12 @@ POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in \ - $(srcdir)/h5cc.in $(srcdir)/testh5repart.sh.in \ - $(top_srcdir)/config/commence.am \ + $(srcdir)/h5cc.in $(srcdir)/testh5extend.sh.in \ + $(srcdir)/testh5repart.sh.in $(top_srcdir)/config/commence.am \ $(top_srcdir)/config/conclude.am check_PROGRAMS = $(am__EXEEXT_1) repart_test$(EXEEXT) -bin_PROGRAMS = h5debug$(EXEEXT) h5repart$(EXEEXT) h5mkgrp$(EXEEXT) +bin_PROGRAMS = h5debug$(EXEEXT) h5repart$(EXEEXT) h5mkgrp$(EXEEXT) \ + h5extend$(EXEEXT) TESTS = $(check_PROGRAMS) $(check_SCRIPTS) subdir = tools/misc ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 @@ -66,10 +67,11 @@ am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ $(ACLOCAL_M4) mkinstalldirs = $(SHELL) $(top_srcdir)/bin/mkinstalldirs CONFIG_HEADER = $(top_builddir)/src/H5config.h -CONFIG_CLEAN_FILES = h5cc testh5repart.sh +CONFIG_CLEAN_FILES = h5cc testh5extend.sh testh5repart.sh CONFIG_CLEAN_VPATH_FILES = am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" -am__EXEEXT_1 = h5repart_gentest$(EXEEXT) talign$(EXEEXT) +am__EXEEXT_1 = h5repart_gentest$(EXEEXT) h5extend_gentest$(EXEEXT) \ + talign$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) h5debug_SOURCES = h5debug.c h5debug_OBJECTS = h5debug.$(OBJEXT) @@ -78,6 +80,17 @@ h5debug_DEPENDENCIES = $(LIBH5TOOLS) $(LIBHDF5) h5debug_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(h5debug_LDFLAGS) \ $(LDFLAGS) -o $@ +h5extend_SOURCES = h5extend.c +h5extend_OBJECTS = h5extend.$(OBJEXT) +h5extend_LDADD = $(LDADD) +h5extend_DEPENDENCIES = $(LIBH5TOOLS) $(LIBHDF5) +h5extend_LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ + --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(h5extend_LDFLAGS) \ + $(LDFLAGS) -o $@ +h5extend_gentest_SOURCES = h5extend_gentest.c +h5extend_gentest_OBJECTS = h5extend_gentest.$(OBJEXT) +h5extend_gentest_LDADD = $(LDADD) +h5extend_gentest_DEPENDENCIES = $(LIBH5TOOLS) $(LIBHDF5) h5mkgrp_SOURCES = h5mkgrp.c h5mkgrp_OBJECTS = h5mkgrp.$(OBJEXT) h5mkgrp_LDADD = $(LDADD) @@ -139,10 +152,10 @@ CCLD = $(CC) LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ $(LDFLAGS) -o $@ -SOURCES = h5debug.c h5mkgrp.c h5repart.c h5repart_gentest.c \ - repart_test.c talign.c -DIST_SOURCES = h5debug.c h5mkgrp.c h5repart.c h5repart_gentest.c \ - repart_test.c talign.c +SOURCES = h5debug.c h5extend.c h5extend_gentest.c h5mkgrp.c h5repart.c \ + h5repart_gentest.c repart_test.c talign.c +DIST_SOURCES = h5debug.c h5extend.c h5extend_gentest.c h5mkgrp.c \ + h5repart.c h5repart_gentest.c repart_test.c talign.c ETAGS = etags CTAGS = ctags am__tty_colors = \ @@ -430,8 +443,8 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.h5 \ INCLUDES = -I$(top_srcdir)/src -I$(top_srcdir)/tools/lib #test script and program -TEST_PROG = h5repart_gentest talign -TEST_SCRIPT = testh5repart.sh $(srcdir)/testh5mkgrp.sh +TEST_PROG = h5repart_gentest h5extend_gentest talign +TEST_SCRIPT = testh5repart.sh $(srcdir)/testh5mkgrp.sh testh5extend.sh check_SCRIPTS = $(TEST_SCRIPT) SCRIPT_DEPEND = h5repart$(EXEEXT) h5mkgrp$(EXEEXT) bin_SCRIPTS = h5redeploy @@ -440,12 +453,13 @@ bin_SCRIPTS = h5redeploy h5debug_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) h5repart_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) h5mkgrp_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) +h5extend_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) # Tell automake to clean h5redeploy script CLEANFILES = h5redeploy # These were generated by configure. Remove them only when distclean. -DISTCLEANFILES = h5cc testh5repart.sh +DISTCLEANFILES = h5cc testh5repart.sh testh5extend.sh # All programs rely on hdf5 library and h5tools library LDADD = $(LIBH5TOOLS) $(LIBHDF5) @@ -506,6 +520,8 @@ $(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) $(am__aclocal_m4_deps): h5cc: $(top_builddir)/config.status $(srcdir)/h5cc.in cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ +testh5extend.sh: $(top_builddir)/config.status $(srcdir)/testh5extend.sh.in + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ testh5repart.sh: $(top_builddir)/config.status $(srcdir)/testh5repart.sh.in cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ install-binPROGRAMS: $(bin_PROGRAMS) @@ -563,6 +579,12 @@ clean-checkPROGRAMS: h5debug$(EXEEXT): $(h5debug_OBJECTS) $(h5debug_DEPENDENCIES) @rm -f h5debug$(EXEEXT) $(h5debug_LINK) $(h5debug_OBJECTS) $(h5debug_LDADD) $(LIBS) +h5extend$(EXEEXT): $(h5extend_OBJECTS) $(h5extend_DEPENDENCIES) + @rm -f h5extend$(EXEEXT) + $(h5extend_LINK) $(h5extend_OBJECTS) $(h5extend_LDADD) $(LIBS) +h5extend_gentest$(EXEEXT): $(h5extend_gentest_OBJECTS) $(h5extend_gentest_DEPENDENCIES) + @rm -f h5extend_gentest$(EXEEXT) + $(LINK) $(h5extend_gentest_OBJECTS) $(h5extend_gentest_LDADD) $(LIBS) h5mkgrp$(EXEEXT): $(h5mkgrp_OBJECTS) $(h5mkgrp_DEPENDENCIES) @rm -f h5mkgrp$(EXEEXT) $(h5mkgrp_LINK) $(h5mkgrp_OBJECTS) $(h5mkgrp_LDADD) $(LIBS) @@ -620,6 +642,8 @@ distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5debug.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5extend.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5extend_gentest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5mkgrp.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repart.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repart_gentest.Po@am__quote@ diff --git a/tools/misc/h5extend.c b/tools/misc/h5extend.c new file mode 100644 index 0000000..fcbab87 --- /dev/null +++ b/tools/misc/h5extend.c @@ -0,0 +1,264 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Mike McGreevy + * March 2, 2011 + * + * Purpose: Reads the 'EOA' message stored in the superblock + * extension of a file, and extends the file via a truncate + * call to this size. The 'EOA' message is then either updated + * with the new value, or removed entirely. (based on user input). + * + */ + +#include "hdf5.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#include "H5Iprivate.h" +#define H5F_PACKAGE +#include "H5Fpkg.h" + +/* Name of tool */ +#define PROGRAMNAME "h5extend" + +/* Command Line Switches */ +static hbool_t quiet_g = FALSE; +static hbool_t remove_g = FALSE; +static hbool_t extend_g = FALSE; + + +/*------------------------------------------------------------------------- + * Function: usage + * + * Purpose: Prints a usage message. + * + * Return: void + * + * Programmer: Mike McGreevy + * March 2, 2011 + * + *------------------------------------------------------------------------- + */ +static void +usage (FILE * outputfile) +{ + HDfprintf(outputfile, "%s [OPTIONS] SRC\n",h5tools_getprogname()); + HDfprintf(outputfile, " OPTIONS:\n"); + HDfprintf(outputfile, " -h, --help Display this help message.\n"); + HDfprintf(outputfile, " -r, --remove Remove 'EOA' message from superblock extension.\n"); + HDfprintf(outputfile, " -e, --extend Extend the file to match its 'EOA' value.\n"); + HDfprintf(outputfile, " -q, --quiet Work without displaying standard output.\n"); +} /* usage */ + + +/*------------------------------------------------------------------------- + * Function: leave + * + * Purpose: Shutdown MPI and/or HDF5 and call exit() + * + * Return: Does not return + * + * Programmer: Quincey Koziol, 2/13/2007 + * + *------------------------------------------------------------------------- + */ +static void +leave(int ret) +{ + h5tools_close(); + exit(ret); +} /* end leave() */ + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Main 'h5extend' tool. + * + * Return: Success: EXIT_SUCCESS + * Failure: EXIT_FAILURE + * + * Programmer: Mike McGreevy + * March 2, 2011 + * + *------------------------------------------------------------------------- + */ +int +main (int argc, char *argv[]) +{ + hbool_t eoa_msg_found = 0; /* Bool indicating if EOA message found */ + const char *filename; /* Source file name */ + int argno=1; /* Program argument number */ + hid_t fid = -1; + H5F_t * f = NULL; + haddr_t eoa,eof,filesize; + FILE * outputfile = stdout; + FILE * errorfile = stderr; + + /* Disable the HDF5 library's error reporting */ + H5Eset_auto2(H5E_DEFAULT, NULL, NULL); + + /* Initialize h5tools lib */ + h5tools_init(); + + /* Set program name */ + h5tools_setprogname(PROGRAMNAME); + + /* Parse Switches */ + while (argno=argc) { + error_msg("No file provided.\n"); + usage(errorfile); + leave(EXIT_FAILURE); + } /* end if */ + + /* Get the name of the source file */ + filename = argv[argno++]; + + /* Open File */ + if ((fid = h5tools_fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT,NULL,NULL,0)) < 0) { + error_msg("Unable to open source file %s.\n", filename); + leave(EXIT_FAILURE); + } /* end if */ + + /* Get internal file pointer */ + if ((f = (H5F_t *)H5I_object(fid)) == NULL) { + error_msg("Unable to get internal file pointer.\n"); + leave(EXIT_FAILURE); + } /* end if */ + + /* Check to see if the EOA value is stored via an 'EOA' message */ + if (f->shared->sblock->ext_addr != HADDR_UNDEF) { + H5O_loc_t ext_loc; /* "Object location" for superblock extension */ + + /* Open the superblock extension */ + if (H5F_super_ext_open(f, f->shared->sblock->ext_addr, &ext_loc) < 0) { + error_msg("Unable to open superblock extension.\n"); + leave(EXIT_FAILURE); + } /* end if */ + + /* Check to see if there is an 'EOA' message */ + if (H5O_msg_exists(&ext_loc, H5O_EOA_ID, H5AC_dxpl_id)) { + eoa_msg_found = 1; + + /* Retrieve 'EOA' value from message in superblock extension */ + if (H5O_msg_read(&ext_loc, H5O_EOA_ID, &eoa, H5AC_dxpl_id) == NULL) { + error_msg("Unable to read 'EOA' message in superblock extension.\n"); + leave(EXIT_FAILURE); + } /* end if */ + if (!quiet_g) HDfprintf(outputfile, " Superblock extension contains 'EOA' message with value = %a\n", eoa); + + /* If requested, remove the 'EOA' message from the superblock extension */ + if (remove_g) { + if (H5O_msg_remove(&ext_loc, H5O_EOA_ID, H5O_ALL, TRUE, H5AC_dxpl_id) < 0) { + error_msg("Unable to remove 'EOA' message.\n"); + leave(EXIT_FAILURE); + } /* end if */ + if (!quiet_g) HDfprintf(outputfile, " Removed 'EOA' message from superblock extension.\n"); + } /* end if */ + } /* end if */ + + /* Close superblock extension */ + if (H5F_super_ext_close(f, &ext_loc, H5AC_dxpl_id, FALSE) < 0) { + error_msg("Unable to close superblock extension.\n"); + leave(EXIT_FAILURE); + } /* end if */ + } /* end if */ + + /* Retrieve 'EOA' value from file driver layer if not found in sblock extension */ + if (!eoa_msg_found) { + if ((eoa = H5FDget_eoa(f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) { + error_msg("Unable to determine 'EOA' message from file driver.\n"); + leave(EXIT_FAILURE); + } /* end if */ + if (!quiet_g) HDfprintf(outputfile, " No 'EOA' message contained in this file. File driver's reported EOA value = %a\n", eoa); + } /* end if */ + + /* Get 'EOF' value */ + if ((eof = H5FDget_eof(f->shared->lf)) == HADDR_UNDEF) { + error_msg("Unable to retrieve 'EOF' value from file driver.\n"); + leave(EXIT_FAILURE); + } /* end if */ + if (!quiet_g) HDfprintf(outputfile, " File EOF value = %a\n", eof); + + /* Get file size */ + if (H5Fget_filesize(fid, &filesize) < 0) { + error_msg("Unable to retrieve file size.\n"); + leave(EXIT_FAILURE); + } /* end if */ + if (!quiet_g) HDfprintf(outputfile, " File size = %a\n", filesize); + + /* Suggest extending the file if EOA/EOF do not match (and request to remove/extend not already made) */ + if ((!extend_g && !remove_g)&&(eoa != eof)) + if (!quiet_g) HDfprintf(outputfile, " EOA and EOF do not match! Suggest extending the file for backward compatibility.\n"); + + /* Extend the file, if requested */ + if (extend_g) { + if (eoa != eof) { + /* truncate file to match 'EOA' value */ + if (!quiet_g) HDfprintf(outputfile, " Truncating file to match EOA of %a\n", eoa); + if (H5FDtruncate(f->shared->lf, H5AC_dxpl_id, 0) < 0) { + error_msg("Unable to truncate file.\n"); + leave(EXIT_FAILURE); + } /* end if */ + + /* Mark superblock dirty, so on file close it is re-written with new filesize value */ + if (H5F_super_dirty(f) < 0) { + error_msg("Unable to mark file superblock dirty.\n"); + leave(EXIT_FAILURE); + } /* end if */ + } /* end if */ + else { + if (!quiet_g) HDfprintf(outputfile," EOA and EOF are already the same. Taking no action to extend the file.\n"); + } /* end else */ + } /* end if */ + + /* Close File */ + if (H5Fclose(fid) < 0) { + error_msg("Failed while closing file.\n"); + leave(EXIT_FAILURE); + } /* end if */ + + /* Shut down h5tools lib */ + h5tools_close(); + + return EXIT_SUCCESS; +} /* main */ + diff --git a/tools/misc/h5extend_gentest.c b/tools/misc/h5extend_gentest.c new file mode 100644 index 0000000..be469b8 --- /dev/null +++ b/tools/misc/h5extend_gentest.c @@ -0,0 +1,70 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/********************************************************************* +* +* Purpose: Create testfile for use with the h5extend tests. +* +* Developer: Mike McGreevy +* March 3, 2011 +* +*********************************************************************/ + +#include "hdf5.h" +#define TESTFILE "h5extend_tfile.h5" + +int main(void) +{ + /* Variables */ + hid_t fcpl,fid,sid,did = -1; /* Object Descriptors */ + + /* Create a file creation property list */ + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) return -1; + + /* Enable 'avoid truncate' feature */ + if (H5Pset_avoid_truncate(fcpl, 1) < 0) return -1; + + /* Create a file that avoids truncation */ + if ((fid = H5Fcreate(TESTFILE, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + return -1; + + /* Close the fcpl */ + if (H5Pclose(fcpl) < 0) return -1; + + /* Create dataspace for dataset */ + if ((sid = H5Screate(H5S_SCALAR)) < 0) return -1; + + /* Create dataset */ + if ((did = H5Dcreate2(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) return -1; + + /* Close the file, dataspace, and dataset */ + H5Sclose(sid); + H5Dclose(did); + H5Fclose(fid); + + /* Re-open file */ + if ((fid = H5Fopen(TESTFILE, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) return -1; + + /* Unlink the dataset, reducing the 'EOA' value (but not EOF) */ + if (H5Ldelete(fid, "Dataset", H5P_DEFAULT) < 0) return -1; + + /* Close file */ + if (H5Fclose(fid) < 0) return -1; + + /* Return */ + return 0; +} /* main */ + diff --git a/tools/misc/testh5extend.sh.in b/tools/misc/testh5extend.sh.in new file mode 100644 index 0000000..73824ae --- /dev/null +++ b/tools/misc/testh5extend.sh.in @@ -0,0 +1,131 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from help@hdfgroup.org. +# +# Tests for the h5extend tool + +TESTNAME=h5extend +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +EXTEND=h5extend # The tool name +EXTEND_BIN=`pwd`/$EXTEND # The path of the tool binary + +EXTENDED_TEST=extend_test # The test name +EXTENDED_TEST_BIN=`pwd`/$EXTENDED_TEST # The path of the test binary + +CMP='cmp -s' +DIFF='diff -c' + +nerrors=0 +verbose=yes + +# The build (current) directory might be different than the source directory. +if test -z "$srcdir"; then + srcdir=. +fi + +test -d ../testfiles || mkdir ../testfiles + +actual_dir=`pwd`/../testfiles + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable. +# +TOOLTEST() { + + expect="$srcdir/../testfiles/$1" + actual="../testfiles/`basename $1 .ddl`.out" + actual_err="../testfiles/`basename $1 .ddl`.err" + shift + + # Run tool test. + TESTING $EXTEND $@ + ( + echo "#############################" + echo "Expected output for '$EXTEND $@'" + echo "#############################" + #cd $srcdir/../testfiles + $RUNSERIAL $EXTEND_BIN $@ + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err + fi +} + +OUTPUTTEST() { + # Run test program. + TESTING $EXTENDED_TEST $@ + ( + cd $actual_dir + $RUNSERIAL $EXTENDED_TEST_BIN $@ + ) + + if test $? -eq 0; then + echo " PASSED" + else + echo " FAILED" + nerrors=`expr $nerrors + 1` + fi +} + +# Print a "SKIP" message +SKIP() { + TESTING $EXTEND $@ + echo " -SKIP-" +} + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +TOOLTEST h5extend_exp1.ddl h5extend_tfile.h5 +TOOLTEST h5extend_exp2.ddl -e h5extend_tfile.h5 +TOOLTEST h5extend_exp3.ddl h5extend_tfile.h5 +TOOLTEST h5extend_exp4.ddl -r h5extend_tfile.h5 +TOOLTEST h5extend_exp5.ddl h5extend_tfile.h5 + +if test $nerrors -eq 0 ; then + echo "All $TESTNAME tests passed." + exit $EXIT_SUCCESS +else + echo "$TESTNAME tests failed with $nerrors errors." + exit $EXIT_FAILURE +fi diff --git a/tools/testfiles/h5extend_exp1.ddl b/tools/testfiles/h5extend_exp1.ddl new file mode 100644 index 0000000..e75eb47 --- /dev/null +++ b/tools/testfiles/h5extend_exp1.ddl @@ -0,0 +1,7 @@ +############################# +Expected output for 'h5extend h5extend_tfile.h5' +############################# + Superblock extension contains 'EOA' message with value = 792 + File EOF value = 1392 + File size = 1392 + EOA and EOF do not match! Suggest extending the file for backward compatibility. diff --git a/tools/testfiles/h5extend_exp2.ddl b/tools/testfiles/h5extend_exp2.ddl new file mode 100644 index 0000000..aa9408f --- /dev/null +++ b/tools/testfiles/h5extend_exp2.ddl @@ -0,0 +1,7 @@ +############################# +Expected output for 'h5extend -e h5extend_tfile.h5' +############################# + Superblock extension contains 'EOA' message with value = 792 + File EOF value = 1392 + File size = 1392 + Truncating file to match EOA of 792 diff --git a/tools/testfiles/h5extend_exp3.ddl b/tools/testfiles/h5extend_exp3.ddl new file mode 100644 index 0000000..1ec6922 --- /dev/null +++ b/tools/testfiles/h5extend_exp3.ddl @@ -0,0 +1,6 @@ +############################# +Expected output for 'h5extend h5extend_tfile.h5' +############################# + Superblock extension contains 'EOA' message with value = 792 + File EOF value = 792 + File size = 792 diff --git a/tools/testfiles/h5extend_exp4.ddl b/tools/testfiles/h5extend_exp4.ddl new file mode 100644 index 0000000..efad77e --- /dev/null +++ b/tools/testfiles/h5extend_exp4.ddl @@ -0,0 +1,7 @@ +############################# +Expected output for 'h5extend -r h5extend_tfile.h5' +############################# + Superblock extension contains 'EOA' message with value = 792 + Removed 'EOA' message from superblock extension. + File EOF value = 792 + File size = 792 diff --git a/tools/testfiles/h5extend_exp5.ddl b/tools/testfiles/h5extend_exp5.ddl new file mode 100644 index 0000000..147e474 --- /dev/null +++ b/tools/testfiles/h5extend_exp5.ddl @@ -0,0 +1,6 @@ +############################# +Expected output for 'h5extend h5extend_tfile.h5' +############################# + No 'EOA' message contained in this file. File driver's reported EOA value = 792 + File EOF value = 792 + File size = 792 diff --git a/tools/testfiles/h5extend_tfile.h5 b/tools/testfiles/h5extend_tfile.h5 new file mode 100644 index 0000000..c87a19d Binary files /dev/null and b/tools/testfiles/h5extend_tfile.h5 differ -- cgit v0.12