From 821ce6874d9e2b322dda171f95901d5f68a1c26a Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Thu, 28 Jan 2016 23:40:43 -0500 Subject: [svn-r29014] h5format_convert(): fix to downgrade superblock version and persistent file space so 1.8 library can open/read the file. Tested on ostrich, kituo, platypus, moohan, quail, emu. More tests will be added later. --- src/H5F.c | 40 ++++- src/H5Fprivate.h | 1 + src/H5Fpublic.h | 2 +- src/H5MF.c | 185 +++++++++++++++------ src/H5MFprivate.h | 1 + tools/h5format_convert/h5format_convert.c | 38 +---- tools/h5format_convert/testfiles/h5fc_v_all.ddl | 2 +- tools/h5format_convert/testfiles/h5fc_v_bt1.ddl | 2 +- tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl | 2 +- tools/h5format_convert/testfiles/h5fc_v_n_all.ddl | 2 +- .../testfiles/h5fc_v_ndata_bt1.ddl | 2 +- .../testfiles/h5fc_v_non_chunked.ddl | 2 +- 12 files changed, 188 insertions(+), 91 deletions(-) diff --git a/src/H5F.c b/src/H5F.c index a822132..2cb81e0 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -1875,8 +1875,9 @@ done: /*------------------------------------------------------------------------- * Function: H5Fformat_convert_super (Internal) * - * Purpose: Downgrade the superblock version for the tool h5format_convert. - * (NOTE: more needs to be done to this routine) + * Purpose: Downgrade the superblock version to v2 and + * downgrade persistent file space to non-persistent + * for 1.8 library. * * Return: Non-negative on success/Negative on failure * @@ -1885,9 +1886,10 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Fformat_convert_super(hid_t fid) +H5Fformat_convert(hid_t fid) { H5F_t *f = NULL; /* File to flush */ + hbool_t mark_dirty = FALSE; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1897,13 +1899,37 @@ H5Fformat_convert_super(hid_t fid) case H5I_FILE: if(NULL == (f = (H5F_t *)H5I_object(fid))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier") - if(f->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_LATEST) + + if(f->shared->sblock->super_vers > HDF5_SUPERBLOCK_VERSION_V18_LATEST) { + f->shared->sblock->super_vers = HDF5_SUPERBLOCK_VERSION_V18_LATEST; + mark_dirty = TRUE; + } + + if(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF && + f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF) { + if(mark_dirty) { + /* Mark superblock as dirty */ + if(H5F_super_dirty(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") + } HGOTO_DONE(SUCCEED) - f->shared->sblock->super_vers = HDF5_SUPERBLOCK_VERSION_LATEST - 1; + } + + /* Check to remove free-space manager info message from superblock extension */ + if(H5F_addr_defined(f->shared->sblock->ext_addr)) { + if(H5F_super_ext_remove_msg(f, H5AC_dxpl_id, H5O_FSINFO_ID) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension") + } + + if(H5MF_try_close(f, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to free free-space address") + + f->shared->fs_strategy = H5F_FILE_SPACE_STRATEGY_DEF; + f->shared->fs_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; /* Mark superblock as dirty */ if(H5F_super_dirty(f) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") break; @@ -1928,4 +1954,4 @@ H5Fformat_convert_super(hid_t fid) done: FUNC_LEAVE_API(ret_value) -} /* end H5Fformat_convert_super() */ +} /* end H5Fformat_convert() */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 6be0cff..ecacfaf 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -485,6 +485,7 @@ #define HDF5_SUPERBLOCK_VERSION_2 2 /* Revised version with superblock extension and checksum */ #define HDF5_SUPERBLOCK_VERSION_3 3 /* With file locking and consistency flags (at least this version for SWMR support) */ #define HDF5_SUPERBLOCK_VERSION_LATEST HDF5_SUPERBLOCK_VERSION_3 /* The maximum super block format */ +#define HDF5_SUPERBLOCK_VERSION_V18_LATEST HDF5_SUPERBLOCK_VERSION_2 /* The latest superblock version for v18 */ #define HDF5_FREESPACE_VERSION 0 /* of the Free-Space Info */ #define HDF5_OBJECTDIR_VERSION 0 /* of the Object Directory format */ #define HDF5_SHAREDHEADER_VERSION 0 /* of the Shared-Header Info */ diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 937e29f..318f374 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -254,7 +254,7 @@ H5_DLL herr_t H5Fstop_mdc_logging(hid_t file_id); H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, /*OUT*/ hbool_t *is_enabled, /*OUT*/ hbool_t *is_currently_logging); -H5_DLL herr_t H5Fformat_convert_super(hid_t fid); +H5_DLL herr_t H5Fformat_convert(hid_t fid); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag); H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); diff --git a/src/H5MF.c b/src/H5MF.c index 0f2857e..e7c122d 100644 --- a/src/H5MF.c +++ b/src/H5MF.c @@ -86,6 +86,7 @@ typedef struct { /* Allocator routines */ static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); static herr_t H5MF_alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); +static herr_t H5MF_close_delete(H5F_t *f, hid_t dxpl_id); /*********************/ @@ -1145,6 +1146,141 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5MF_close_shrink_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_close_delete + * + * Purpose: The coding is copied from H5MF_close() "else" statement + * to this routine so it can also be called by H5MF_try_close(). + * + * Return: SUCCEED/FAIL + * + * Programmer: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_close_delete(H5F_t *f, hid_t dxpl_id) +{ + H5FD_mem_t type; /* Memory type for iteration */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + HDassert(f->shared); + + /* super_vers can be 0, 1, 2 */ + /* Iterate over all the free space types that have managers and get each free list's space */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* If the free space manager for this type is open, close it */ + if(f->shared->fs_man[type]) { +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't release free space info") + f->shared->fs_man[type] = NULL; + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + } /* end if */ +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If there is free space manager info for this type, delete it */ + if(H5F_addr_defined(f->shared->fs_addr[type])) { + haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ + + /* Put address into temporary variable and reset it */ + /* (Avoids loopback in file space freeing routine) */ + tmp_fs_addr = f->shared->fs_addr[type]; + f->shared->fs_addr[type] = HADDR_UNDEF; + + /* Shift to "deleting" state, to make certain we don't track any + * file space freed as a result of deleting the free space manager. + */ + f->shared->fs_state[type] = H5F_FS_STATE_DELETING; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Delete free space manager for this type */ + if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't delete free space manager") + + /* Shift [back] to closed state */ + HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + + /* Sanity check that the free space manager for this type wasn't started up again */ + HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); + } /* end if */ + } /* end for */ + +done: +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_close_delete() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_try_close + * + * Purpose: This is called by H5Fformat_convert() to close and delete + * free-space managers when downgrading persistent free-space + * to non-persistent. + * + * Return: SUCCEED/FAIL + * + * Programmer: + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_try_close(H5F_t *f, hid_t dxpl_id) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + + /* Set the ring type in the DXPL */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + + if(H5MF_close_delete(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to close delete free-space managers") + +done: + /* Reset the ring in the DXPL */ + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5MF_try_close() */ + /*------------------------------------------------------------------------- * Function: H5MF_close @@ -1287,55 +1423,10 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); } /* end for */ } /* end if */ else { /* super_vers can be 0, 1, 2 */ - /* Iterate over all the free space types that have managers and get each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - /* If the free space manager for this type is open, close it */ - if(f->shared->fs_man[type]) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't release free space info") - f->shared->fs_man[type] = NULL; - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - } /* end if */ -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - /* If there is free space manager info for this type, delete it */ - if(H5F_addr_defined(f->shared->fs_addr[type])) { - haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ - - /* Put address into temporary variable and reset it */ - /* (Avoids loopback in file space freeing routine) */ - tmp_fs_addr = f->shared->fs_addr[type]; - f->shared->fs_addr[type] = HADDR_UNDEF; - - /* Shift to "deleting" state, to make certain we don't track any - * file space freed as a result of deleting the free space manager. - */ - f->shared->fs_state[type] = H5F_FS_STATE_DELETING; - -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* Delete free space manager for this type */ - if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't delete free space manager") - - /* Shift [back] to closed state */ - HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + if(H5MF_close_delete(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") - /* Sanity check that the free space manager for this type wasn't started up again */ - HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); - } /* end if */ - } /* end for */ } /* end else */ /* Free the space in aggregators (again) */ diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h index 024cc91..bb07f4e 100644 --- a/src/H5MFprivate.h +++ b/src/H5MFprivate.h @@ -63,6 +63,7 @@ H5_DLL herr_t H5MF_init_merge_flags(H5F_t *f); H5_DLL herr_t H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size); H5_DLL herr_t H5MF_close(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id); /* File space allocation routines */ H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); diff --git a/tools/h5format_convert/h5format_convert.c b/tools/h5format_convert/h5format_convert.c index a80e332..54c666d 100644 --- a/tools/h5format_convert/h5format_convert.c +++ b/tools/h5format_convert/h5format_convert.c @@ -408,28 +408,6 @@ main(int argc, const char *argv[]) } else if(verbose_g) printf("Open the file %s\n", fname_g); - /* A temporaray fix: - * need to handle H5O_FSINFO_ID message when downgrade superblock version from 3 to 2 - */ - if((fcpl = H5Fget_create_plist(fid)) < 0) { - error_msg("unable to get file creation property list for \"%s\"\n", fname_g); - h5tools_setstatus(EXIT_FAILURE); - goto done; - } - if(H5Pget_file_space(fcpl, &strategy, &threshold) < 0) { - error_msg("unable to get file space strategy/threshold\n"); - h5tools_setstatus(EXIT_FAILURE); - goto done; - } - /* Check for non-default strategy/threshold: - * --whether there is H5O_FSINFO_ID message in the superblock extension - */ - if(strategy != H5F_FILE_SPACE_ALL || threshold != 1) { - error_msg("unable to convert due to non-default file space strategy/threshold\n"); - h5tools_setstatus(EXIT_FAILURE); - goto done; - } - if(dset_g) { /* Convert a specified dataset in the file */ if(verbose_g) printf("Going to process dataset: %s...\n", dname_g); @@ -443,16 +421,16 @@ main(int argc, const char *argv[]) } if(verbose_g) { - if(noop_g) { - printf("Not processing the file's superblock version...\n"); - h5tools_setstatus(EXIT_SUCCESS); - goto done; - } - printf("Processing the file's superblock version...\n"); + if(noop_g) { + printf("Not processing the file's superblock...\n"); + h5tools_setstatus(EXIT_SUCCESS); + goto done; + } + printf("Processing the file's superblock...\n"); } - if(H5Fformat_convert_super(fid) < 0) { - error_msg("unable to convert file's superblock version\"%s\"\n", fname_g); + if(H5Fformat_convert(fid) < 0) { + error_msg("unable to convert file's superblock\"%s\"\n", fname_g); h5tools_setstatus(EXIT_FAILURE); goto done; } diff --git a/tools/h5format_convert/testfiles/h5fc_v_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_all.ddl index 5a35c55..9898faa 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_all.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_all.ddl @@ -23,5 +23,5 @@ Retrieve the dataset's chunk indexing type Chunk indexing type is already version 1 B-tree: no further action Close the dataset Close the dataset creation property list -Processing the file's superblock version... +Processing the file's superblock... Close the file diff --git a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl index c96b647..3bafc78 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl @@ -8,5 +8,5 @@ Retrieve the dataset's chunk indexing type Chunk indexing type is already version 1 B-tree: no further action Close the dataset Close the dataset creation property list -Processing the file's superblock version... +Processing the file's superblock... Close the file diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl index a7a622a..242386d 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl @@ -10,5 +10,5 @@ Verify the dataset's chunk indexing type is not version 1 B-tree Not converting the dataset Close the dataset Close the dataset creation property list -Not processing the file's superblock version... +Not processing the file's superblock... Close the file diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl index 3e92568..3dec4a6 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl @@ -44,5 +44,5 @@ Retrieve the dataset's layout Dataset is not chunked: no further action Close the dataset Close the dataset creation property list -Not processing the file's superblock version... +Not processing the file's superblock... Close the file diff --git a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl index bdf3380..a326dc5 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl @@ -9,5 +9,5 @@ Retrieve the dataset's chunk indexing type Chunk indexing type is already version 1 B-tree: no further action Close the dataset Close the dataset creation property list -Not processing the file's superblock version... +Not processing the file's superblock... Close the file diff --git a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl index 4caafe9..bbc7229 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl @@ -6,5 +6,5 @@ Retrieve the dataset's layout Dataset is not chunked: no further action Close the dataset Close the dataset creation property list -Processing the file's superblock version... +Processing the file's superblock... Close the file -- cgit v0.12