[svn-r29014] h5format_convert(): fix to downgrade superblock version and persistent file space so

1.8 library can open/read the file.
Tested on ostrich, kituo, platypus, moohan, quail, emu.
More tests will be added later.

12 files changed, 188 insertions, 91 deletions

diff --git a/src/H5F.c b/src/H5F.c
index a822132..2cb81e0 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -1875,8 +1875,9 @@ done:
 /*-------------------------------------------------------------------------
  * Function:	H5Fformat_convert_super (Internal)
  *
- * Purpose:	Downgrade the superblock version for the tool h5format_convert.
- *		(NOTE: more needs to be done to this routine)
+ * Purpose:	Downgrade the superblock version to v2 and
+ *		downgrade persistent file space to non-persistent
+ *		for 1.8 library.
  *
  * Return:	Non-negative on success/Negative on failure
  *
@@ -1885,9 +1886,10 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5Fformat_convert_super(hid_t fid)
+H5Fformat_convert(hid_t fid)
 {
     H5F_t	*f = NULL;              /* File to flush */
+    hbool_t	mark_dirty = FALSE;
     herr_t      ret_value = SUCCEED;    /* Return value */
 
     FUNC_ENTER_API(FAIL)
@@ -1897,13 +1899,37 @@ H5Fformat_convert_super(hid_t fid)
         case H5I_FILE:
             if(NULL == (f = (H5F_t *)H5I_object(fid)))
                 HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier")
-	    if(f->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_LATEST)
+
+	    if(f->shared->sblock->super_vers > HDF5_SUPERBLOCK_VERSION_V18_LATEST) {
+		f->shared->sblock->super_vers = HDF5_SUPERBLOCK_VERSION_V18_LATEST;
+		mark_dirty = TRUE;
+	    }
+
+            if(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF &&
+	       f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF) {
+		if(mark_dirty) {
+		    /* Mark superblock as dirty */
+		    if(H5F_super_dirty(f) < 0)
+			HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
+		}
 		HGOTO_DONE(SUCCEED)
-	    f->shared->sblock->super_vers = HDF5_SUPERBLOCK_VERSION_LATEST - 1;
+	    }
+
+	    /* Check to remove free-space manager info message from superblock extension */
+            if(H5F_addr_defined(f->shared->sblock->ext_addr)) {
+                if(H5F_super_ext_remove_msg(f, H5AC_dxpl_id, H5O_FSINFO_ID) < 0)
+                    HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
+	    }
+
+            if(H5MF_try_close(f, H5AC_dxpl_id) < 0)
+                HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to free free-space address")
+
+            f->shared->fs_strategy = H5F_FILE_SPACE_STRATEGY_DEF;
+            f->shared->fs_threshold = H5F_FREE_SPACE_THRESHOLD_DEF;
 
 	    /* Mark superblock as dirty */
 	    if(H5F_super_dirty(f) < 0)
-		HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
+		HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
 
             break;
 
@@ -1928,4 +1954,4 @@ H5Fformat_convert_super(hid_t fid)
 
 done:
     FUNC_LEAVE_API(ret_value)
-} /* end H5Fformat_convert_super() */
+} /* end H5Fformat_convert() */
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 6be0cff..ecacfaf 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -485,6 +485,7 @@
 #define HDF5_SUPERBLOCK_VERSION_2	2	/* Revised version with superblock extension and checksum */
 #define HDF5_SUPERBLOCK_VERSION_3	3	/* With file locking and consistency flags (at least this version for SWMR support) */
 #define HDF5_SUPERBLOCK_VERSION_LATEST	HDF5_SUPERBLOCK_VERSION_3	/* The maximum super block format    */
+#define HDF5_SUPERBLOCK_VERSION_V18_LATEST HDF5_SUPERBLOCK_VERSION_2    /* The latest superblock version for v18 */
 #define HDF5_FREESPACE_VERSION	        0	/* of the Free-Space Info	  */
 #define HDF5_OBJECTDIR_VERSION	        0	/* of the Object Directory format */
 #define HDF5_SHAREDHEADER_VERSION       0	/* of the Shared-Header Info	  */
diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h
index 937e29f..318f374 100644
--- a/src/H5Fpublic.h
+++ b/src/H5Fpublic.h
@@ -254,7 +254,7 @@ H5_DLL herr_t H5Fstop_mdc_logging(hid_t file_id);
 H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id,
                                         /*OUT*/ hbool_t *is_enabled,
                                         /*OUT*/ hbool_t *is_currently_logging);
-H5_DLL herr_t H5Fformat_convert_super(hid_t fid);
+H5_DLL herr_t H5Fformat_convert(hid_t fid);
 #ifdef H5_HAVE_PARALLEL
 H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag);
 H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag);
diff --git a/src/H5MF.c b/src/H5MF.c
index 0f2857e..e7c122d 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -86,6 +86,7 @@ typedef struct {
 /* Allocator routines */
 static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
 static herr_t H5MF_alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
+static herr_t H5MF_close_delete(H5F_t *f, hid_t dxpl_id);
 
 
 /*********************/
@@ -1145,6 +1146,141 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5MF_close_shrink_eoa() */
 
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5MF_close_delete
+ *
+ * Purpose:     The coding is copied from H5MF_close() "else" statement
+ *		to this routine so it can also be called by H5MF_try_close().
+ *
+ * Return:	SUCCEED/FAIL
+ *
+ * Programmer:  
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF_close_delete(H5F_t *f, hid_t dxpl_id)
+{
+    H5FD_mem_t type;                    /* Memory type for iteration */
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Entering\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+
+    /* check args */
+    HDassert(f);
+    HDassert(f->shared);
+
+    /* super_vers can be 0, 1, 2 */
+    /* Iterate over all the free space types that have managers and get each free list's space */
+    for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+	    /* If the free space manager for this type is open, close it */
+	    if(f->shared->fs_man[type]) {
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+		if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0)
+		    HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't release free space info")
+		f->shared->fs_man[type] = NULL;
+		f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
+	    } /* end if */
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+	    /* If there is free space manager info for this type, delete it */
+	    if(H5F_addr_defined(f->shared->fs_addr[type])) {
+		haddr_t tmp_fs_addr;            /* Temporary holder for free space manager address */
+
+		/* Put address into temporary variable and reset it */
+		/* (Avoids loopback in file space freeing routine) */
+		tmp_fs_addr = f->shared->fs_addr[type];
+		f->shared->fs_addr[type] = HADDR_UNDEF;
+
+		/* Shift to "deleting" state, to make certain we don't track any
+		 *  file space freed as a result of deleting the free space manager.
+		 */
+		f->shared->fs_state[type] = H5F_FS_STATE_DELETING;
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+		/* Delete free space manager for this type */
+		if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0)
+		    HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't delete free space manager")
+
+		/* Shift [back] to closed state */
+		HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING);
+		f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
+
+		/* Sanity check that the free space manager for this type wasn't started up again */
+		HDassert(!H5F_addr_defined(f->shared->fs_addr[type]));
+	    } /* end if */
+    } /* end for */
+
+done:
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_close_delete() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5MF_try_close
+ *
+ * Purpose:     This is called by H5Fformat_convert() to close and delete
+ *		free-space managers when downgrading persistent free-space
+ *		to non-persistent.
+ *
+ * Return:	SUCCEED/FAIL
+ *
+ * Programmer:  
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_try_close(H5F_t *f, hid_t dxpl_id)
+{
+    H5P_genplist_t *dxpl = NULL;        /* DXPL for setting ring */
+    H5AC_ring_t orig_ring = H5AC_RING_INV;      /* Original ring value */
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Entering\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+
+    /* check args */
+    HDassert(f);
+
+    /* Set the ring type in the DXPL */
+    if(H5AC_set_ring(dxpl_id, H5AC_RING_FSM, &dxpl, &orig_ring) < 0)
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+
+    if(H5MF_close_delete(f, dxpl_id) < 0)
+        HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to close delete free-space managers")
+
+done:
+    /* Reset the ring in the DXPL */
+    if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+        HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5MF_try_close() */
+
 
 /*-------------------------------------------------------------------------
  * Function:    H5MF_close
@@ -1287,55 +1423,10 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
 	} /* end for */
     } /* end if */
     else {  /* super_vers can be 0, 1, 2 */
-	/* Iterate over all the free space types that have managers and get each free list's space */
-	for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
-	    /* If the free space manager for this type is open, close it */
-	    if(f->shared->fs_man[type]) {
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
-		if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0)
-		    HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't release free space info")
-		f->shared->fs_man[type] = NULL;
-		f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
-	    } /* end if */
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
 
-	    /* If there is free space manager info for this type, delete it */
-	    if(H5F_addr_defined(f->shared->fs_addr[type])) {
-		haddr_t tmp_fs_addr;            /* Temporary holder for free space manager address */
-
-		/* Put address into temporary variable and reset it */
-		/* (Avoids loopback in file space freeing routine) */
-		tmp_fs_addr = f->shared->fs_addr[type];
-		f->shared->fs_addr[type] = HADDR_UNDEF;
-
-		/* Shift to "deleting" state, to make certain we don't track any
-		 *  file space freed as a result of deleting the free space manager.
-		 */
-		f->shared->fs_state[type] = H5F_FS_STATE_DELETING;
-
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
-
-		/* Delete free space manager for this type */
-		if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0)
-		    HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "can't delete free space manager")
-
-		/* Shift [back] to closed state */
-		HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING);
-		f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
+	if(H5MF_close_delete(f, dxpl_id) < 0)
+            HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
 
-		/* Sanity check that the free space manager for this type wasn't started up again */
-		HDassert(!H5F_addr_defined(f->shared->fs_addr[type]));
-	    } /* end if */
-	} /* end for */
     } /* end else */
 
     /* Free the space in aggregators (again) */
diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h
index 024cc91..bb07f4e 100644
--- a/src/H5MFprivate.h
+++ b/src/H5MFprivate.h
@@ -63,6 +63,7 @@ H5_DLL herr_t H5MF_init_merge_flags(H5F_t *f);
 H5_DLL herr_t H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space,
     hsize_t *meta_size);
 H5_DLL herr_t H5MF_close(H5F_t *f, hid_t dxpl_id);
+H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id);
 
 /* File space allocation routines */
 H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
diff --git a/tools/h5format_convert/h5format_convert.c b/tools/h5format_convert/h5format_convert.c
index a80e332..54c666d 100644
--- a/tools/h5format_convert/h5format_convert.c
+++ b/tools/h5format_convert/h5format_convert.c
@@ -408,28 +408,6 @@ main(int argc, const char *argv[])
     } else if(verbose_g)
 	printf("Open the file %s\n", fname_g);
 
-    /* A temporaray fix: 
-     * need to handle H5O_FSINFO_ID message when downgrade superblock version from 3 to 2 
-     */
-    if((fcpl = H5Fget_create_plist(fid)) < 0) {
-	error_msg("unable to get file creation property list for \"%s\"\n", fname_g);
-	h5tools_setstatus(EXIT_FAILURE);
-	goto done;
-    }
-    if(H5Pget_file_space(fcpl, &strategy, &threshold) < 0) {
-	error_msg("unable to get file space strategy/threshold\n");
-	h5tools_setstatus(EXIT_FAILURE);
-	goto done;
-    }
-    /* Check for non-default strategy/threshold: 
-     * --whether there is H5O_FSINFO_ID message in the superblock extension 
-     */
-    if(strategy != H5F_FILE_SPACE_ALL || threshold != 1) {
-	error_msg("unable to convert due to non-default file space strategy/threshold\n");
-	h5tools_setstatus(EXIT_FAILURE);
-	goto done;
-    }
-
     if(dset_g) { /* Convert a specified dataset in the file */
 	if(verbose_g)
 	    printf("Going to process dataset: %s...\n", dname_g);
@@ -443,16 +421,16 @@ main(int argc, const char *argv[])
     }
 
     if(verbose_g) {
-	if(noop_g) {
-            printf("Not processing the file's superblock version...\n");
-	    h5tools_setstatus(EXIT_SUCCESS);
-	    goto done;
-	}
-	printf("Processing the file's superblock version...\n");
+        if(noop_g) {
+            printf("Not processing the file's superblock...\n");
+            h5tools_setstatus(EXIT_SUCCESS);
+            goto done;
+        }
+        printf("Processing the file's superblock...\n");
     }
 
-    if(H5Fformat_convert_super(fid) < 0) {
-        error_msg("unable to convert file's superblock version\"%s\"\n", fname_g);
+    if(H5Fformat_convert(fid) < 0) {
+        error_msg("unable to convert file's superblock\"%s\"\n", fname_g);
         h5tools_setstatus(EXIT_FAILURE);
         goto done;
     }
diff --git a/tools/h5format_convert/testfiles/h5fc_v_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_all.ddl
index 5a35c55..9898faa 100644
--- a/tools/h5format_convert/testfiles/h5fc_v_all.ddl
+++ b/tools/h5format_convert/testfiles/h5fc_v_all.ddl
@@ -23,5 +23,5 @@ Retrieve the dataset's chunk indexing type
 Chunk indexing type is already version 1 B-tree: no further action
 Close the dataset
 Close the dataset creation property list
-Processing the file's superblock version...
+Processing the file's superblock...
 Close the file
diff --git a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl
index c96b647..3bafc78 100644
--- a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl
+++ b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl
@@ -8,5 +8,5 @@ Retrieve the dataset's chunk indexing type
 Chunk indexing type is already version 1 B-tree: no further action
 Close the dataset
 Close the dataset creation property list
-Processing the file's superblock version...
+Processing the file's superblock...
 Close the file
diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl
index a7a622a..242386d 100644
--- a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl
+++ b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl
@@ -10,5 +10,5 @@ Verify the dataset's chunk indexing type is not version 1 B-tree
 Not converting the dataset
 Close the dataset
 Close the dataset creation property list
-Not processing the file's superblock version...
+Not processing the file's superblock...
 Close the file
diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl
index 3e92568..3dec4a6 100644
--- a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl
+++ b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl
@@ -44,5 +44,5 @@ Retrieve the dataset's layout
 Dataset is not chunked: no further action
 Close the dataset
 Close the dataset creation property list
-Not processing the file's superblock version...
+Not processing the file's superblock...
 Close the file
diff --git a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl
index bdf3380..a326dc5 100644
--- a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl
+++ b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl
@@ -9,5 +9,5 @@ Retrieve the dataset's chunk indexing type
 Chunk indexing type is already version 1 B-tree: no further action
 Close the dataset
 Close the dataset creation property list
-Not processing the file's superblock version...
+Not processing the file's superblock...
 Close the file
diff --git a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl
index 4caafe9..bbc7229 100644
--- a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl
+++ b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl
@@ -6,5 +6,5 @@ Retrieve the dataset's layout
 Dataset is not chunked: no further action
 Close the dataset
 Close the dataset creation property list
-Processing the file's superblock version...
+Processing the file's superblock...
 Close the file