[svn-r16619] Description:

	Bring r16606:16618 from trunk to the revise_chunks branch.

Tested on:
	FreeBSD/32 6.3 (duty)
	h5committest not necessary on this branch

5 files changed, 227 insertions, 91 deletions

diff --git a/src/H5C.c b/src/H5C.c
index 27b98d0..b7d64f9 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -4881,6 +4881,7 @@ H5C_insert_entry(H5F_t * 	     f,
     entry_ptr->ro_ref_count = 0;
 
     entry_ptr->is_pinned = insert_pinned;
+    entry_ptr->pinned_from_client = insert_pinned;
 
     /* newly inserted entries are assumed to be dirty */
     entry_ptr->is_dirty = TRUE;
@@ -5973,6 +5974,56 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5C_pin_entry_from_client()
+ *
+ * Purpose:	Internal routine to pin a cache entry from a client action.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Quincey Koziol
+ *              3/26/09
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef NDEBUG
+static herr_t
+H5C_pin_entry_from_client(H5C_t *	          cache_ptr,
+                        H5C_cache_entry_t * entry_ptr)
+#else
+static herr_t
+H5C_pin_entry_from_client(H5C_t UNUSED *	cache_ptr,
+                        H5C_cache_entry_t * entry_ptr)
+#endif
+{
+    herr_t              ret_value = SUCCEED;    /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT(H5C_pin_entry_from_client)
+
+    /* Sanity checks */
+    HDassert( cache_ptr );
+    HDassert( entry_ptr );
+
+    /* Check if the entry is already pinned */
+    if(entry_ptr->is_pinned) {
+        /* Check if the entry was pinned through an explicit pin from a client */
+        if(entry_ptr->pinned_from_client)
+            HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, "Entry is already pinned")
+    } /* end if */
+    else {
+        entry_ptr->is_pinned = TRUE;
+
+        H5C__UPDATE_STATS_FOR_PIN(cache_ptr, entry_ptr)
+    } /* end else */
+
+    /* Mark that the entry was pinned through an explicit pin from a client */
+    entry_ptr->pinned_from_client = TRUE;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_pin_entry_from_client() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5C_pin_protected_entry()
  *
  * Purpose:	Pin a protected cache entry.  The entry must be protected
@@ -6000,47 +6051,32 @@ done:
  *
  *-------------------------------------------------------------------------
  */
-#ifndef NDEBUG
 herr_t
 H5C_pin_protected_entry(H5C_t *	          cache_ptr,
                         void *		  thing)
-#else
-herr_t
-H5C_pin_protected_entry(H5C_t UNUSED *	cache_ptr,
-                        void *		  thing)
-#endif
 {
+    H5C_cache_entry_t *	entry_ptr;              /* Pointer to entry to pin */
     herr_t              ret_value = SUCCEED;    /* Return value */
-    H5C_cache_entry_t *	entry_ptr;
 
     FUNC_ENTER_NOAPI(H5C_pin_protected_entry, FAIL)
 
     HDassert( cache_ptr );
     HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
     HDassert( thing );
-
     entry_ptr = (H5C_cache_entry_t *)thing;
-
+    HDassert( entry_ptr );
     HDassert( H5F_addr_defined(entry_ptr->addr) );
 
-    if ( ! ( entry_ptr->is_protected ) ) {
-
+    /* Only protected entries can be pinned */
+    if(!entry_ptr->is_protected)
         HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, "Entry isn't protected")
-    }
-
-    if ( entry_ptr->is_pinned ) {
-
-        HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, "Entry is already pinned")
-    }
-
-    entry_ptr->is_pinned = TRUE;
 
-    H5C__UPDATE_STATS_FOR_PIN(cache_ptr, entry_ptr)
+    /* Pin the entry from a client */
+    if(H5C_pin_entry_from_client(cache_ptr, entry_ptr) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, "Can't pin entry by client")
 
 done:
-
     FUNC_LEAVE_NOAPI(ret_value)
-
 } /* H5C_pin_protected_entry() */
 
 
@@ -7671,6 +7707,58 @@ H5C_stats__reset(H5C_t UNUSED * cache_ptr)
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5C_unpin_entry_from_client()
+ *
+ * Purpose:	Internal routine to unpin a cache entry from a client action.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  Quincey Koziol
+ *              3/24/09
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5C_unpin_entry_from_client(H5C_t *		  cache_ptr,
+                H5C_cache_entry_t *	  entry_ptr,
+                hbool_t update_rp)
+{
+    herr_t              ret_value = SUCCEED;    /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT(H5C_unpin_entry_from_client)
+
+    /* Sanity checking */
+    HDassert( cache_ptr );
+    HDassert( entry_ptr );
+
+    /* Error checking (should be sanity checks?) */
+    if(!entry_ptr->is_pinned)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "Entry isn't pinned")
+    if(!entry_ptr->pinned_from_client)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "Entry wasn't pinned by cache client")
+
+    /* If requested, update the replacement policy if the entry is not protected */
+    if(update_rp && !entry_ptr->is_protected)
+        H5C__UPDATE_RP_FOR_UNPIN(cache_ptr, entry_ptr, FAIL)
+
+    /* Check if the entry is not pinned from a flush dependency */
+    if(!entry_ptr->pinned_from_cache) {
+        /* Unpin the entry now */
+        entry_ptr->is_pinned = FALSE;
+
+        /* Update the stats for an unpin operation */
+        H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, entry_ptr)
+    } /* end if */
+
+    /* Mark the entry as explicitly unpinned by the client */
+    entry_ptr->pinned_from_client = FALSE;
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_unpin_entry_from_client() */
+
+
+/*-------------------------------------------------------------------------
  * Function:    H5C_unpin_entry()
  *
  * Purpose:	Unpin a cache entry.  The entry can be either protected or
@@ -7687,35 +7775,24 @@ herr_t
 H5C_unpin_entry(H5C_t *		  cache_ptr,
                 void *		  thing)
 {
+    H5C_cache_entry_t *	entry_ptr;              /* Pointer to entry to unpin */
     herr_t              ret_value = SUCCEED;    /* Return value */
-    H5C_cache_entry_t *	entry_ptr;
 
     FUNC_ENTER_NOAPI(H5C_unpin_entry, FAIL)
 
+    /* Sanity checking */
     HDassert( cache_ptr );
     HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC );
     HDassert( thing );
-
     entry_ptr = (H5C_cache_entry_t *)thing;
+    HDassert( entry_ptr );
 
-    if ( ! ( entry_ptr->is_pinned ) ) {
-
-        HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "Entry isn't pinned")
-    }
-
-    if ( ! ( entry_ptr->is_protected ) ) {
-
-        H5C__UPDATE_RP_FOR_UNPIN(cache_ptr, entry_ptr, FAIL)
-    }
-
-    entry_ptr->is_pinned = FALSE;
-
-    H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, entry_ptr)
+    /* Unpin the entry */
+    if(H5C_unpin_entry_from_client(cache_ptr, entry_ptr, TRUE) < 0)
+        HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "Can't unpin entry from client")
 
 done:
-
     FUNC_LEAVE_NOAPI(ret_value)
-
 } /* H5C_unpin_entry() */
 
 
@@ -7846,7 +7923,6 @@ H5C_unprotect(H5F_t *		  f,
               unsigned int        flags,
               size_t              new_size)
 {
-    /* const char *	fcn_name = "H5C_unprotect()"; */
     hbool_t		deleted;
     hbool_t		dirtied;
     hbool_t             set_flush_marker;
@@ -7938,23 +8014,15 @@ H5C_unprotect(H5F_t *		  f,
         /* Pin or unpin the entry as requested. */
         if ( pin_entry ) {
 
-            if ( entry_ptr->is_pinned ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, \
-                            "Entry already pinned???")
-            }
-	    entry_ptr->is_pinned = TRUE;
-	    H5C__UPDATE_STATS_FOR_PIN(cache_ptr, entry_ptr)
+            /* Pin the entry from a client */
+            if(H5C_pin_entry_from_client(cache_ptr, entry_ptr) < 0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, "Can't pin entry by client")
 
         } else if ( unpin_entry ) {
 
-            if ( ! ( entry_ptr->is_pinned ) ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, \
-			    "Entry already unpinned???")
-            }
-	    entry_ptr->is_pinned = FALSE;
-	    H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, entry_ptr)
+            /* Unpin the entry from a client */
+            if(H5C_unpin_entry_from_client(cache_ptr, entry_ptr, FALSE) < 0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "Can't unpin entry by client")
 
         }
 
@@ -8071,23 +8139,15 @@ H5C_unprotect(H5F_t *		  f,
         /* Pin or unpin the entry as requested. */
         if ( pin_entry ) {
 
-            if ( entry_ptr->is_pinned ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, \
-                            "Entry already pinned???")
-            }
-	    entry_ptr->is_pinned = TRUE;
-	    H5C__UPDATE_STATS_FOR_PIN(cache_ptr, entry_ptr)
+            /* Pin the entry from a client */
+            if(H5C_pin_entry_from_client(cache_ptr, entry_ptr) < 0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, "Can't pin entry by client")
 
         } else if ( unpin_entry ) {
 
-            if ( ! ( entry_ptr->is_pinned ) ) {
-
-                HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, \
-			    "Entry already unpinned???")
-            }
-	    entry_ptr->is_pinned = FALSE;
-	    H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, entry_ptr)
+            /* Unpin the entry from a client */
+            if(H5C_unpin_entry_from_client(cache_ptr, entry_ptr, FALSE) < 0)
+                HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "Can't unpin entry by client")
 
         }
 
@@ -8675,21 +8735,21 @@ H5C_create_flush_dependency(H5C_t UNUSED * cache_ptr, void * parent_thing,
         HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "Combined flush dependency height too large")
     }
 
-    /* Check for parent already pinned */
-    if(parent_entry->is_pinned) {
-        /* Verify that the parent entry was pinned through a flush dependency relationship */
-        if(0 == parent_entry->flush_dep_height)
-            HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "Parent entry wasn't pinned through flush dependency")
-    } /* end if */
-    else {
+    /* Check for parent not pinned */
+    if(!parent_entry->is_pinned) {
         /* Sanity check */
         HDassert(parent_entry->flush_dep_height == 0);
+        HDassert(!parent_entry->pinned_from_client);
+        HDassert(!parent_entry->pinned_from_cache);
 
         /* Pin the parent entry */
         parent_entry->is_pinned = TRUE;
         H5C__UPDATE_STATS_FOR_PIN(cache_ptr, parent_entry)
     } /* end else */
 
+    /* Mark the entry as pinned from the cache's action (possibly redundantly) */
+    parent_entry->pinned_from_cache = TRUE;
+
     /* Increment ref. count for parent's flush dependency children heights */
     parent_entry->child_flush_dep_height_rc[child_entry->flush_dep_height]++;
 
@@ -8811,12 +8871,24 @@ H5C_destroy_flush_dependency(H5C_t * cache_ptr, void *parent_thing,
          *  parent of _any_ child flush dependencies).
          */
         if(0 == parent_entry->flush_dep_height) {
-            if(!parent_entry->is_protected)
-                H5C__UPDATE_RP_FOR_UNPIN(cache_ptr, parent_entry, FAIL)
+            /* Sanity check */
+            HDassert(parent_entry->pinned_from_cache);
+
+            /* Check if we should unpin parent entry now */
+            if(!parent_entry->pinned_from_client) {
+                /* Update the replacement policy if the entry is not protected */
+                if(!parent_entry->is_protected)
+                    H5C__UPDATE_RP_FOR_UNPIN(cache_ptr, parent_entry, FAIL)
+
+                /* Unpin the entry now */
+                parent_entry->is_pinned = FALSE;
+
+                /* Update the stats for an unpin operation */
+                H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, parent_entry)
+            } /* end if */
 
-            /* Unpin parent entry */
-            parent_entry->is_pinned = FALSE;
-            H5C__UPDATE_STATS_FOR_UNPIN(cache_ptr, parent_entry)
+            /* Mark the entry as unpinned from the cache's action */
+            parent_entry->pinned_from_cache = FALSE;
         } /* end if */
     } /* end if */
 
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 855c861..5d89313 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -378,6 +378,37 @@ typedef herr_t (*H5C_log_flush_func_t)(H5C_t * cache_ptr,
  *              'dest' callback routine.
  *
  *
+ * Fields supporting the 'flush dependency' feature:
+ *
+ * Entries in the cache may have a 'flush dependency' on another entry in the
+ * cache.  A flush dependency requires that all dirty child entries be flushed
+ * to the file before a dirty parent entry (of those child entries) can be
+ * flushed to the file.  This can be used by cache clients to create data
+ * structures that allow Single-Writer/Multiple-Reader (SWMR) access for the
+ * data structure.
+ *
+ * The leaf child entry will have a "height" of 0, with any parent entries
+ * having a height of 1 greater than the maximum height of any of their child
+ * entries (flush dependencies are allowed to create asymmetric trees of
+ * relationships).
+ *
+ * flush_dep_parent:	Pointer to the parent entry for an entry in a flush
+ *		dependency relationship.
+ *
+ * child_flush_dep_height_rc:	An array of reference counts for child entries,
+ *		where the number of children of each height is tracked.
+ *
+ * flush_dep_height:	The height of the entry, which is one greater than the
+ *		maximum height of any of its child entries..
+ *
+ * pinned_from_client:	Whether the entry was pinned by an explicit pin request
+ *		from a cache client.
+ *
+ * pinned_from_cache:	Whether the entry was pinned implicitly as a
+ *		request of being a parent entry in a flush dependency
+ *		relationship.
+ *
+ *
  * Fields supporting the hash table:
  *
  * Fields in the cache are indexed by a more or less conventional hash table.
@@ -504,11 +535,13 @@ typedef struct H5C_cache_entry_t
     hbool_t		destroy_in_progress;
     hbool_t		free_file_space_on_destroy;
 
-    /* fields supporting the 'flush dependency height': */
+    /* fields supporting the 'flush dependency' feature: */
 
     struct H5C_cache_entry_t *	flush_dep_parent;
     uint64_t            child_flush_dep_height_rc[H5C__NUM_FLUSH_DEP_HEIGHTS];
     unsigned            flush_dep_height;
+    hbool_t		pinned_from_client;
+    hbool_t		pinned_from_cache;
 
     /* fields supporting the hash table: */
 
diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c
index cfad5e3..1d508dc 100644
--- a/src/H5FDfamily.c
+++ b/src/H5FDfamily.c
@@ -626,8 +626,15 @@ H5FD_family_sb_encode(H5FD_t *_file, char *name/*out*/, unsigned char *buf/*out*
     HDstrncpy(name, "NCSAfami", (size_t)8);
     name[8] = '\0';
 
-    /* Store member file size */
-    UINT64ENCODE(buf, (uint64_t)file->memb_size);
+    /* Store member file size.  Use the member file size from the property here.
+     * This is to guarantee backward compatibility.  If a file is created with 
+     * v1.6 library and the driver info isn't saved in the superblock.  We open
+     * it with v1.8, the FILE->MEMB_SIZE will be the actual size of the first 
+     * member file (see H5FD_family_open).  So it isn't safe to use FILE->MEMB_SIZE.
+     * If the file is created with v1.8, the correctness of FILE->PMEM_SIZE is
+     * checked in H5FD_family_sb_decode. SLU - 2009/3/21
+     */
+    UINT64ENCODE(buf, (uint64_t)file->pmem_size);
 
     FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5FD_family_sb_encode() */
@@ -668,9 +675,9 @@ H5FD_family_sb_decode(H5FD_t *_file, const char UNUSED *name, const unsigned cha
     /* For h5repart only. Private property of new member size is used to signal
      * h5repart is being used to change member file size.  h5repart will open
      * files for read and write.  When the files are closed, metadata will be
-     * flushed to the files and updated this new size */
+     * flushed to the files and updated to this new size */
     if(file->mem_newsize) {
-        file->memb_size = file->mem_newsize;
+        file->memb_size = file->pmem_size = file->mem_newsize;
         HGOTO_DONE(ret_value)
     } /* end if */
 
@@ -682,7 +689,7 @@ H5FD_family_sb_decode(H5FD_t *_file, const char UNUSED *name, const unsigned cha
     if(msize != file->pmem_size) {
         char                err_msg[128];
 
-        sprintf(err_msg, "family member size should be %lu, is %lu", (unsigned long)msize, (unsigned long)file->pmem_size);
+        sprintf(err_msg, "Family member size should be %lu.  But the size from file access property is %lu", (unsigned long)msize, (unsigned long)file->pmem_size);
         HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, err_msg)
     } /* end if */
 
diff --git a/src/H5FDint.c b/src/H5FDint.c
index aa375e2..74b3bf6 100644
--- a/src/H5FDint.c
+++ b/src/H5FDint.c
@@ -194,6 +194,11 @@ done:
  *
  * Purpose:	Private version of H5FDset_eoa()
  *
+ *              This function expects the EOA is a RELATIVE address, i.e.
+ *              relative to the base address.  This is NOT the same as the
+ *              EOA stored in the superblock, which is an absolute
+ *              address.  Object addresses are relative.
+ *
  * Return:	Success:	Non-negative
  *		Failure:	Negative, no side effect
  *
@@ -212,7 +217,7 @@ H5FD_set_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr)
     HDassert(file && file->cls);
     HDassert(H5F_addr_defined(addr) && addr <= file->maxaddr);
 
-    /* Dispatch to driver */
+    /* Dispatch to driver, convert to absolute address */
     if((file->cls->set_eoa)(file, type, addr + file->base_addr) < 0)
 	HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver set_eoa request failed")
 
@@ -226,6 +231,11 @@ done:
  *
  * Purpose:	Private version of H5FDget_eoa()
  *
+ *              This function returns the EOA as a RELATIVE address, i.e.
+ *              relative to the base address.  This is NOT the same as the
+ *              EOA stored in the superblock, which is an absolute
+ *              address.  Object addresses are relative.
+ *
  * Return:	Success:	First byte after allocated memory.
  *		Failure:	HADDR_UNDEF
  *
@@ -247,7 +257,7 @@ H5FD_get_eoa(const H5FD_t *file, H5FD_mem_t type)
     if(HADDR_UNDEF == (ret_value = (file->cls->get_eoa)(file, type)))
 	HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "driver get_eoa request failed")
 
-    /* Adjust for base address in file */
+    /* Adjust for base address in file (convert to relative address) */
     ret_value -= file->base_addr;
 
 done:
@@ -260,6 +270,11 @@ done:
  *
  * Purpose:	Private version of H5FDget_eof()
  *
+ *              This function returns the EOF as a RELATIVE address, i.e.
+ *              relative to the base address.  This will be different
+ *              from  the end of the physical file if there is a user
+ *              block.
+ *
  * Return:	Success:	The EOF address.
  *
  *		Failure:	HADDR_UNDEF
@@ -288,7 +303,7 @@ H5FD_get_eof(const H5FD_t *file)
     else
 	ret_value = file->maxaddr;
 
-    /* Adjust for base address in file */
+    /* Adjust for base address in file (convert to relative address)  */
     ret_value -= file->base_addr;
 
 done:
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 70e0954..8040554 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -560,7 +560,8 @@ H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc)
      * Tell the file driver how much address space has already been
      * allocated so that it knows how to allocate additional memory.
      */
-    if(H5FD_set_eoa(lf, H5FD_MEM_SUPER, stored_eoa) < 0)
+    /* (Account for the stored EOA being absolute offset -NAF) */
+    if(H5FD_set_eoa(lf, H5FD_MEM_SUPER, stored_eoa - H5F_BASE_ADDR(f)) < 0)
         HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to set end-of-address marker for file")
 
     /* Read the file's superblock extension, if there is one. */
@@ -934,7 +935,15 @@ H5F_super_write(H5F_t *f, hid_t dxpl_id)
 
         /* Encode the driver information block. */
         H5_ASSIGN_OVERFLOW(driver_size, H5FD_sb_size(f->shared->lf), hsize_t, size_t);
-        if(driver_size > 0) {
+
+        /* Checking whether driver block address is defined here is to handle backward
+         * compatibility.  If the file was created with v1.6 library or earlier and no
+         * driver info block was written in the superblock, we don't write it either even
+         * though there's some driver info.  Otherwise, the driver block extended will
+         * overwrite the (meta)data right after the superblock. This situation happens to
+         * the family driver particularly.  SLU - 2009/3/24 
+         */
+        if(driver_size > 0 && H5F_addr_defined(f->shared->driver_addr)) {
             char driver_name[9];    /* Name of driver, for driver info block */
             uint8_t *dbuf = p;      /* Pointer to beginning of driver info */