[svn-r29969] Description:

    Bring r29934 from revise_chunks branch to trunk:

(1) Fix for HDFFV-9434: throw an error instead of assertion when v1 btree level hits the 1 byte limit.
(2) Modifications to better handle error recovery when conversion by h5format_convert fails.

Tested on:
    MacOSX/64 10.11.5 (amazon) w/serial, parallel & production
    (h5committest forthcoming)

8 files changed, 152 insertions, 25 deletions

diff --git a/src/H5AC.c b/src/H5AC.c
index c0c6604..42f328e 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -2464,6 +2464,41 @@ done:
 } /* H5AC_flush_tagged_metadata */
 
 
+
+/*------------------------------------------------------------------------------
+ * Function:    H5AC_expunge_tag_type_metadata()
+ *
+ * Purpose:     Wrapper for cache level function which expunge entries with
+ *              a specific tag and type id.
+ * 
+ * Return:      SUCCEED on success, FAIL otherwise.
+ *
+ * Programmer:  Vailin Choi; May 2016
+ *
+ *------------------------------------------------------------------------------
+ */
+herr_t
+H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags)
+{
+    /* Variable Declarations */
+    herr_t ret_value = SUCCEED;
+ 
+    /* Function Enter Macro */   
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Assertions */
+    HDassert(f);
+    HDassert(f->shared);
+
+    /* Call cache level function to expunge entries with specified tag and type id */
+    if(H5C_expunge_tag_type_metadata(f, dxpl_id, tag, type_id, flags)<0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Cannot expunge tagged type entries")
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5AC_expunge_tag_type_metadata*/
+
 #if H5AC_DO_TAGGING_SANITY_CHECKS
 
 /*-------------------------------------------------------------------------
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 5b871ff..23b909f 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -365,6 +365,7 @@ H5_DLL herr_t H5AC_get_entry_ring(const H5F_t *f, haddr_t addr, H5AC_ring_t *rin
 H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dxpl,
     H5AC_ring_t *orig_ring);
 H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring);
+H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags);
 
 #ifdef H5_HAVE_PARALLEL
 H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr);
diff --git a/src/H5Bcache.c b/src/H5Bcache.c
index 747e4c4..8354e8e 100644
--- a/src/H5Bcache.c
+++ b/src/H5Bcache.c
@@ -312,6 +312,11 @@ H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len,
 
     /* node type and level */
     *image++ = (uint8_t)shared->type->id;
+
+    /* 2^8 limit: only 1 byte is used to store node level */
+    if(bt->level >= HDpow(2, LEVEL_BITS))
+	HGOTO_ERROR(H5E_BTREE, H5E_CANTENCODE, FAIL, "unable to encode node level")
+
     H5_CHECK_OVERFLOW(bt->level, unsigned, uint8_t);
     *image++ = (uint8_t)bt->level;
 
diff --git a/src/H5Bpkg.h b/src/H5Bpkg.h
index e645626..41e0951 100644
--- a/src/H5Bpkg.h
+++ b/src/H5Bpkg.h
@@ -42,6 +42,7 @@
 
 /* Get the native key at a given index */
 #define H5B_NKEY(b, shared, idx)  ((b)->native + (shared)->nkey[(idx)])
+#define LEVEL_BITS	8 	/* # of bits for node level: 1 byte */
 
 
 /****************************/
diff --git a/src/H5C.c b/src/H5C.c
index 0082a98..cd42512 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -9902,6 +9902,59 @@ H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest_tag)
 
 
 /*-------------------------------------------------------------------------
+ *
+ * Function:    H5C_expunge_tag_type_metadata
+ *
+ * Purpose:     Search and expunge from the cache entries associated
+ *              with 'tag' and type id.
+ *
+ * Return:      FAIL if error is detected, SUCCEED otherwise.
+ *
+ * Programmer:  Vailin Choi; May 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t 
+H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags)
+{
+    unsigned u;                 	/* Local index variable */
+    H5C_t *cache_ptr = NULL;
+    herr_t ret_value = SUCCEED;         /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity check */
+    HDassert(f);
+    HDassert(f->shared);
+    HDassert(f->shared->cache);
+    HDassert(f->shared->cache->magic == H5C__H5C_T_MAGIC);
+
+    /* Get cache pointer */
+    cache_ptr = f->shared->cache;
+
+    /* Iterate through hash table entries, expunge those with specified tag and type id */
+    for(u = 0; u < H5C__HASH_TABLE_LEN; u++) {
+        H5C_cache_entry_t *entry_ptr;   /* Entry pointer */
+
+	entry_ptr = cache_ptr->index[u];
+	while(entry_ptr != NULL) {
+	    /* Found one with the same tag and type id */
+	    if(entry_ptr->tag == tag && entry_ptr->type->id == type_id) {
+
+		if(H5C_expunge_entry(f, dxpl_id, entry_ptr->type, entry_ptr->addr, flags) < 0)
+		    HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "H5C_expunge_entry() failed.")
+	    } /* end if */
+
+	    entry_ptr = entry_ptr->ht_next;
+	} /* end while */
+    } /* end for */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_expunge_tag_type_metadata */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5C_get_entry_ring
  *
  * Purpose:     Given a file address, retrieve the ring for an entry at that
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index d43c845..fb04d84 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -1970,6 +1970,7 @@ H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id,
     const H5C_class_t *type, haddr_t addr, unsigned flags);
 H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags);
 H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); 
+H5_DLL herr_t H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags);
 #if H5C_DO_TAGGING_SANITY_CHECKS
 herr_t H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality);
 #endif
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index f4f9104..8cf86ac 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -6471,7 +6471,7 @@ H5D__chunk_format_convert(H5D_t *dset, H5D_chk_idx_info_t *idx_info, H5D_chk_idx
     udata.dset_dims = dset->shared->curr_dims;
 
     /*  terate over the chunks in the current index and insert the chunk addresses into version 1 B-tree index */
-    if((dset->shared->layout.storage.u.chunk.ops->iterate)(idx_info, H5D__chunk_format_convert_cb, &udata) < 0)
+    if((idx_info->storage->ops->iterate)(idx_info, H5D__chunk_format_convert_cb, &udata) < 0)
 	HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk index to chunk info")
 
 done:
diff --git a/src/H5Dint.c b/src/H5Dint.c
index 322776b..c68a355 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -2848,10 +2848,12 @@ done:
 herr_t
 H5D__format_convert(H5D_t *dataset, hid_t dxpl_id)
 {
-    H5O_t *oh = NULL;                   	/* Pointer to dataset's object header */
     H5D_chk_idx_info_t new_idx_info;    	/* Index info for the new layout */
     H5D_chk_idx_info_t idx_info;        	/* Index info for the current layout */
     H5O_layout_t newlayout;			/* The new layout */
+    hbool_t init_new_index = FALSE;		/* Indicate that the new chunk index is initialized */
+    hbool_t delete_old_layout = FALSE;		/* Indicate that the old layout message is deleted */
+    hbool_t add_new_layout = FALSE;		/* Indicate that the new layout message is added */
     herr_t ret_value = SUCCEED;         	/* Return value */
 
     FUNC_ENTER_PACKAGE_TAG(dxpl_id, dataset->oloc.addr, FAIL)
@@ -2888,37 +2890,42 @@ H5D__format_convert(H5D_t *dataset, hid_t dxpl_id)
 	    new_idx_info.storage = &newlayout.storage.u.chunk;
 
 	    /* Initialize version 1 B-tree */
-	    if(newlayout.storage.u.chunk.ops->init && (newlayout.storage.u.chunk.ops->init)(&new_idx_info, dataset->shared->space, dataset->oloc.addr) < 0)
+	    if(new_idx_info.storage->ops->init && 
+              (new_idx_info.storage->ops->init)(&new_idx_info, dataset->shared->space, dataset->oloc.addr) < 0)
 		HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize indexing information")
-    
+	    init_new_index = TRUE;
+
 	    /* If the current chunk index exists */
-	    if(H5F_addr_defined(dataset->shared->layout.storage.u.chunk.idx_addr)) {
+	    if(H5F_addr_defined(idx_info.storage->idx_addr)) {
+
 		/* Create v1 B-tree chunk index */
-		if((newlayout.storage.u.chunk.ops->create)(&new_idx_info) < 0)
+		if((new_idx_info.storage->ops->create)(&new_idx_info) < 0)
 		    HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create chunk index")
 
 		/* Iterate over the chunks in the current index and insert the chunk addresses 
 		 * into the version 1 B-tree chunk index */
 		if(H5D__chunk_format_convert(dataset, &idx_info, &new_idx_info) < 0)
-		    HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk index to chunk info")
+		    HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate/convert chunk index")
 	    } /* end if */
 
-	    /* Release the old (i.e. current) chunk index */
-	    if(dataset->shared->layout.storage.u.chunk.ops->dest && (dataset->shared->layout.storage.u.chunk.ops->dest)(&idx_info) < 0)
-		HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to release chunk index info")
-
-	    /* Delete the "layout" message */
-	    if(H5O_msg_remove(&dataset->oloc, H5O_LAYOUT_ID, H5O_ALL, TRUE, dxpl_id) < 0)
+	    /* Delete the old "current" layout message */
+	    if(H5O_msg_remove(&dataset->oloc, H5O_LAYOUT_ID, H5O_ALL, FALSE, dxpl_id) < 0)
 		HGOTO_ERROR(H5E_SYM, H5E_CANTDELETE, FAIL, "unable to delete layout message")
 
-	    HDmemcpy(&dataset->shared->layout, &newlayout, sizeof(H5O_layout_t));
-
-	    if(NULL == (oh = H5O_pin(&dataset->oloc, dxpl_id)))
-		HGOTO_ERROR(H5E_DATASET, H5E_CANTPIN, FAIL, "unable to pin dataset object header")
+	    delete_old_layout = TRUE;
 
 	    /* Append the new layout message to the object header */
-	    if(H5O_msg_append_oh(dataset->oloc.file, dxpl_id, oh, H5O_LAYOUT_ID, 0, H5O_UPDATE_TIME, &newlayout) < 0)
-		HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update old fill value header message")
+	    if(H5O_msg_create(&dataset->oloc, H5O_LAYOUT_ID, 0, H5O_UPDATE_TIME, &newlayout, dxpl_id) < 0)
+		HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update layout header message")
+
+	    add_new_layout = TRUE;
+
+	    /* Release the old (current) chunk index */
+	    if(idx_info.storage->ops->dest && (idx_info.storage->ops->dest)(&idx_info) < 0)
+		HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to release chunk index info")
+
+	    /* Copy the new layout to the dataset's layout */
+	    HDmemcpy(&dataset->shared->layout, &newlayout, sizeof(H5O_layout_t));
 
 	    break;
 
@@ -2935,17 +2942,41 @@ H5D__format_convert(H5D_t *dataset, hid_t dxpl_id)
 
         case H5D_LAYOUT_ERROR:
         case H5D_NLAYOUTS:
-            HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset layout type")
-            
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset layout type")
+
 	default: 
 	    HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "unknown dataset layout type")
     } /* end switch */
 
 done:
-    /* Release pointer to object header */
-    if(oh != NULL)
-        if(H5O_unpin(oh) < 0)
-            HDONE_ERROR(H5E_DATASET, H5E_CANTUNPIN, FAIL, "unable to unpin dataset object header")
+    if(ret_value < 0 && dataset->shared->layout.type == H5D_CHUNKED) {
+	/* Remove new layout message */
+	if(add_new_layout)
+	    if(H5O_msg_remove(&dataset->oloc, H5O_LAYOUT_ID, H5O_ALL, FALSE, dxpl_id) < 0)
+		HDONE_ERROR(H5E_SYM, H5E_CANTDELETE, FAIL, "unable to delete layout message")
+
+	/* Add back old layout message */
+	if(delete_old_layout)
+	    if(H5O_msg_create(&dataset->oloc, H5O_LAYOUT_ID, 0, H5O_UPDATE_TIME, &dataset->shared->layout, dxpl_id) < 0)
+		HDONE_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to add layout header message")
+
+	/* Clean up v1 b-tree chunk index */
+	if(init_new_index) {
+	    if(H5F_addr_defined(new_idx_info.storage->idx_addr)) {
+		/* Check for valid address i.e. tag */
+		if(!H5F_addr_defined(dataset->oloc.addr))
+		    HDONE_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "address undefined")
+
+		/* Expunge from cache all v1 B-tree type entries associated with tag */
+		if(H5AC_expunge_tag_type_metadata(dataset->oloc.file, dxpl_id, dataset->oloc.addr, H5AC_BT_ID, H5AC__NO_FLAGS_SET))
+		    HDONE_ERROR(H5E_DATASET, H5E_CANTEXPUNGE, FAIL, "unable to expunge index metadata")
+	    } /* end if */
+
+	    /* Delete v1 B-tree chunk index */
+	    if(new_idx_info.storage->ops->dest && (new_idx_info.storage->ops->dest)(&new_idx_info) < 0)
+		HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to release chunk index info")
+	} /* end if */
+    } /* end if */
 
     FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
 } /* end H5D__format_convert() */