[svn-r16137] Description:

	Add support for paging large data blocks to extensible arrays

	Clean up allocation/destroy code for extensible array index, super and
data blocks

	Add a couple of routines to set/get bit values in memory buffers

Tested on:
        Mac OS X/32 10.5.5 (amazon) in debug mode
        Mac OS X/32 10.5.5 (amazon) w/C++ & FORTRAN, w/threadsafe,
                                in production mode
        FreeBSD/32 6.3 (duty) in debug mode
        FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode
        Linux/32 2.6 (jam) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe,
                                in debug mode
        Linux/64-amd64 2.6 (smirom) w/Intel compilers w/default API=1.6.x,
                                w/C++ & FORTRAN, in production mode
        Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN,
                                w/szip filter, in production mode
        Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN,
                                in production mode
        Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode
        Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in production mode

1 files changed, 342 insertions, 16 deletions

diff --git a/src/H5EAcache.c b/src/H5EAcache.c
index 2590edc..a3c9f87 100644
--- a/src/H5EAcache.c
+++ b/src/H5EAcache.c
@@ -62,6 +62,7 @@
 #define H5EA_IBLOCK_BUF_SIZE    512
 #define H5EA_SBLOCK_BUF_SIZE    512
 #define H5EA_DBLOCK_BUF_SIZE    512
+#define H5EA_DBLK_PAGE_BUF_SIZE 512
 
 
 /******************/
@@ -99,6 +100,11 @@ static herr_t H5EA__cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy,
 static herr_t H5EA__cache_dblock_clear(H5F_t *f, H5EA_dblock_t *dblock, hbool_t destroy);
 static herr_t H5EA__cache_dblock_size(const H5F_t *f, const H5EA_dblock_t *dblock, size_t *size_ptr);
 static herr_t H5EA__cache_dblock_dest(H5F_t *f, H5EA_dblock_t *dblock);
+static H5EA_dblk_page_t *H5EA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udata, void *udata2);
+static herr_t H5EA__cache_dblk_page_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5EA_dblk_page_t *dblk_page, unsigned * flags_ptr);
+static herr_t H5EA__cache_dblk_page_clear(H5F_t *f, H5EA_dblk_page_t *dblk_page, hbool_t destroy);
+static herr_t H5EA__cache_dblk_page_size(const H5F_t *f, const H5EA_dblk_page_t *dblk_page, size_t *size_ptr);
+static herr_t H5EA__cache_dblk_page_dest(H5F_t *f, H5EA_dblk_page_t *dblk_page);
 
 
 /*********************/
@@ -145,6 +151,16 @@ const H5AC_class_t H5AC_EARRAY_DBLOCK[1] = {{
     (H5AC_size_func_t)H5EA__cache_dblock_size,
 }};
 
+/* H5EA data block page inherits cache-like properties from H5AC */
+const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{
+    H5AC_EARRAY_DBLK_PAGE_ID,
+    (H5AC_load_func_t)H5EA__cache_dblk_page_load,
+    (H5AC_flush_func_t)H5EA__cache_dblk_page_flush,
+    (H5AC_dest_func_t)H5EA__cache_dblk_page_dest,
+    (H5AC_clear_func_t)H5EA__cache_dblk_page_clear,
+    (H5AC_size_func_t)H5EA__cache_dblk_page_size,
+}};
+
 
 /*****************************/
 /* Library Private Variables */
@@ -1267,7 +1283,10 @@ H5EA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr,
 	H5E_THROW(H5E_CANTINIT, "can't wrap buffer")
 
     /* Compute the size of the extensible array data block on disk */
-    size = H5EA_DBLOCK_SIZE(dblock);
+    if(!dblock->npages)
+        size = H5EA_DBLOCK_SIZE(dblock);
+    else
+        size = H5EA_DBLOCK_PREFIX_SIZE(dblock);
 
     /* Get a pointer to a buffer that's large enough for serialized info */
     if(NULL == (buf = (uint8_t *)H5WB_actual(wb, size)))
@@ -1303,18 +1322,21 @@ H5EA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr,
 
     /* Internal information */
 
-    /* Decode elements in data block */
-    /* Convert from raw elements on disk into native elements in memory */
-    if((hdr->cparam.cls->decode)(p, dblock->elmts, *nelmts) < 0)
-        H5E_THROW(H5E_CANTDECODE, "can't decode extensible array data elements")
-    p += (*nelmts * hdr->cparam.raw_elmt_size);
+    /* Only decode elements if the data block is not paged */
+    if(!dblock->npages) {
+        /* Decode elements in data block */
+        /* Convert from raw elements on disk into native elements in memory */
+        if((hdr->cparam.cls->decode)(p, dblock->elmts, *nelmts) < 0)
+            H5E_THROW(H5E_CANTDECODE, "can't decode extensible array data elements")
+        p += (*nelmts * hdr->cparam.raw_elmt_size);
+    } /* end if */
 
     /* Sanity check */
     /* (allow for checksum not decoded yet) */
     HDassert((size_t)(p - buf) == (size - H5EA_SIZEOF_CHKSUM));
 
-    /* Save the data block's size */
-    dblock->size = size;
+    /* Set the data block's size */
+    dblock->size = H5EA_DBLOCK_SIZE(dblock);
 
     /* Compute checksum on data block */
     computed_chksum = H5_checksum_metadata(buf, (size_t)(p - buf), 0);
@@ -1323,7 +1345,7 @@ H5EA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr,
     UINT32DECODE(p, stored_chksum);
 
     /* Sanity check */
-    HDassert((size_t)(p - buf) == dblock->size);
+    HDassert((size_t)(p - buf) == size);
 
     /* Verify checksum */
     if(stored_chksum != computed_chksum)
@@ -1383,7 +1405,10 @@ H5EA__cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr,
             H5E_THROW(H5E_CANTINIT, "can't wrap buffer")
 
         /* Compute the size of the data block on disk */
-        size = dblock->size;
+        if(!dblock->npages)
+            size = dblock->size;
+        else
+            size = H5EA_DBLOCK_PREFIX_SIZE(dblock);
 
         /* Get a pointer to a buffer that's large enough for serialized info */
         if(NULL == (buf = (uint8_t *)H5WB_actual(wb, size)))
@@ -1410,12 +1435,15 @@ H5EA__cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr,
 
         /* Internal information */
 
-        /* Encode elements in data block */
+        /* Only encode elements if the data block is not paged */
+        if(!dblock->npages) {
+            /* Encode elements in data block */
 
-        /* Convert from native elements in memory into raw elements on disk */
-        if((dblock->hdr->cparam.cls->encode)(p, dblock->elmts, dblock->nelmts) < 0)
-            H5E_THROW(H5E_CANTENCODE, "can't encode extensible array data elements")
-        p += (dblock->nelmts * dblock->hdr->cparam.raw_elmt_size);
+            /* Convert from native elements in memory into raw elements on disk */
+            if((dblock->hdr->cparam.cls->encode)(p, dblock->elmts, dblock->nelmts) < 0)
+                H5E_THROW(H5E_CANTENCODE, "can't encode extensible array data elements")
+            p += (dblock->nelmts * dblock->hdr->cparam.raw_elmt_size);
+        } /* end if */
 
         /* Compute metadata checksum */
         metadata_chksum = H5_checksum_metadata(buf, (size_t)(p - buf), 0);
@@ -1503,7 +1531,10 @@ H5EA__cache_dblock_size(const H5F_t UNUSED *f, const H5EA_dblock_t *dblock,
     HDassert(size_ptr);
 
     /* Set size value */
-    *size_ptr = dblock->size;
+    if(!dblock->npages)
+        *size_ptr = dblock->size;
+    else
+        *size_ptr = H5EA_DBLOCK_PREFIX_SIZE(dblock);
 
 END_FUNC(STATIC)   /* end H5EA__cache_dblock_size() */
 
@@ -1542,6 +1573,7 @@ H5EA__cache_dblock_dest(H5F_t *f, H5EA_dblock_t *dblock))
         HDassert(H5F_addr_eq(dblock->addr, dblock->cache_info.addr));
 
         /* Release the space on disk */
+        /* (Includes space for pages!) */
         /* (XXX: Nasty usage of internal DXPL value! -QAK) */
         if(H5MF_xfree(f, H5FD_MEM_EARRAY_DBLOCK, H5AC_dxpl_id, dblock->cache_info.addr, (hsize_t)dblock->size) < 0)
             H5E_THROW(H5E_CANTFREE, "unable to free extensible array data block")
@@ -1555,3 +1587,297 @@ CATCH
 
 END_FUNC(STATIC)   /* end H5EA__cache_dblock_dest() */
 
+
+/*-------------------------------------------------------------------------
+ * Function:	H5EA__cache_dblk_page_load
+ *
+ * Purpose:	Loads an extensible array data block page from the disk.
+ *
+ * Return:	Success:	Pointer to a new extensible array data block page
+ *		Failure:	NULL
+ *
+ * Programmer:	Quincey Koziol
+ *		koziol@hdfgroup.org
+ *		Nov 20 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+BEGIN_FUNC(STATIC, ERR,
+H5EA_dblk_page_t *, NULL, NULL,
+H5EA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr,
+    const void UNUSED *udata1, void *_hdr))
+
+    /* Local variables */
+    H5EA_hdr_t          *hdr = (H5EA_hdr_t *)_hdr;  /* Shared extensible array information */
+    H5EA_dblk_page_t	*dblk_page = NULL; /* Data block page info */
+    size_t		size;           /* Data block page size */
+    H5WB_t              *wb = NULL;     /* Wrapped buffer for data block page data */
+    uint8_t             dblk_page_buf[H5EA_DBLK_PAGE_BUF_SIZE]; /* Buffer for data block page */
+    uint8_t		*buf;           /* Pointer to data block page buffer */
+    const uint8_t	*p;             /* Pointer into raw data buffer */
+    uint32_t            stored_chksum;  /* Stored metadata checksum value */
+    uint32_t            computed_chksum; /* Computed metadata checksum value */
+
+    /* Sanity check */
+    HDassert(f);
+    HDassert(H5F_addr_defined(addr));
+    HDassert(hdr);
+#ifdef QAK
+HDfprintf(stderr, "%s: addr = %a\n", FUNC, addr);
+#endif /* QAK */
+
+    /* Allocate the extensible array data block page */
+    if(NULL == (dblk_page = H5EA__dblk_page_alloc(hdr)))
+	H5E_THROW(H5E_CANTALLOC, "memory allocation failed for extensible array data block page")
+
+    /* Set the extensible array data block's information */
+    dblk_page->addr = addr;
+
+    /* Wrap the local buffer for serialized info */
+    if(NULL == (wb = H5WB_wrap(dblk_page_buf, sizeof(dblk_page_buf))))
+	H5E_THROW(H5E_CANTINIT, "can't wrap buffer")
+
+    /* Compute the size of the extensible array data block page on disk */
+    size = H5EA_DBLK_PAGE_SIZE(dblk_page);
+
+    /* Get a pointer to a buffer that's large enough for serialized info */
+    if(NULL == (buf = (uint8_t *)H5WB_actual(wb, size)))
+	H5E_THROW(H5E_CANTGET, "can't get actual buffer")
+
+    /* Read data block page from disk */
+    if(H5F_block_read(f, H5FD_MEM_EARRAY_DBLK_PAGE, addr, size, dxpl_id, buf) < 0)
+	H5E_THROW(H5E_READERROR, "can't read extensible array data block page")
+
+    /* Get temporary pointer to serialized header */
+    p = buf;
+
+    /* Internal information */
+
+    /* Decode elements in data block page */
+    /* Convert from raw elements on disk into native elements in memory */
+    if((hdr->cparam.cls->decode)(p, dblk_page->elmts, hdr->dblk_page_nelmts) < 0)
+        H5E_THROW(H5E_CANTDECODE, "can't decode extensible array data elements")
+    p += (hdr->dblk_page_nelmts * hdr->cparam.raw_elmt_size);
+
+    /* Sanity check */
+    /* (allow for checksum not decoded yet) */
+    HDassert((size_t)(p - buf) == (size - H5EA_SIZEOF_CHKSUM));
+
+    /* Set the data block page's size */
+    dblk_page->size = size;
+
+    /* Compute checksum on data block */
+    computed_chksum = H5_checksum_metadata(buf, (size_t)(p - buf), 0);
+
+    /* Metadata checksum */
+    UINT32DECODE(p, stored_chksum);
+
+    /* Sanity check */
+    HDassert((size_t)(p - buf) == dblk_page->size);
+
+    /* Verify checksum */
+    if(stored_chksum != computed_chksum)
+{
+HDfprintf(stderr, "%s: stored_chksum = %8x, computed_chksum = %8x\n", FUNC, stored_chksum, computed_chksum);
+	H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block page")
+}
+
+    /* Set return value */
+    ret_value = dblk_page;
+
+CATCH
+
+    /* Release resources */
+    if(wb && H5WB_unwrap(wb) < 0)
+	H5E_THROW(H5E_CLOSEERROR, "can't close wrapped buffer")
+    if(!ret_value)
+        if(dblk_page && H5EA__cache_dblk_page_dest(f, dblk_page) < 0)
+            H5E_THROW(H5E_CANTFREE, "unable to destroy extensible array data block page")
+
+END_FUNC(STATIC)   /* end H5EA__cache_dblk_page_load() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5EA__cache_dblk_page_flush
+ *
+ * Purpose:	Flushes a dirty extensible array data block page to disk.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Quincey Koziol
+ *		koziol@hdfgroup.org
+ *		Nov 20 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+BEGIN_FUNC(STATIC, ERR,
+herr_t, SUCCEED, FAIL,
+H5EA__cache_dblk_page_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr,
+    H5EA_dblk_page_t *dblk_page, unsigned UNUSED * flags_ptr))
+
+    /* Local variables */
+    H5WB_t *wb = NULL;                  /* Wrapped buffer for serializing data */
+    uint8_t ser_buf[H5EA_DBLK_PAGE_BUF_SIZE]; /* Serialization buffer */
+
+    /* Sanity check */
+    HDassert(f);
+    HDassert(H5F_addr_defined(addr));
+    HDassert(dblk_page);
+    HDassert(dblk_page->hdr);
+
+    if(dblk_page->cache_info.is_dirty) {
+        uint8_t	*buf;           /* Temporary raw data buffer */
+        uint8_t *p;             /* Pointer into raw data buffer */
+        size_t	size;           /* Index block size on disk */
+        uint32_t metadata_chksum; /* Computed metadata checksum value */
+
+        /* Wrap the local buffer for serialized info */
+        if(NULL == (wb = H5WB_wrap(ser_buf, sizeof(ser_buf))))
+            H5E_THROW(H5E_CANTINIT, "can't wrap buffer")
+
+        /* Compute the size of the data block on disk */
+        size = dblk_page->size;
+
+        /* Get a pointer to a buffer that's large enough for serialized info */
+        if(NULL == (buf = (uint8_t *)H5WB_actual(wb, size)))
+            H5E_THROW(H5E_CANTGET, "can't get actual buffer")
+
+        /* Get temporary pointer to serialized info */
+        p = buf;
+
+        /* Internal information */
+
+        /* Encode elements in data block page */
+
+        /* Convert from native elements in memory into raw elements on disk */
+        if((dblk_page->hdr->cparam.cls->encode)(p, dblk_page->elmts, dblk_page->hdr->dblk_page_nelmts) < 0)
+            H5E_THROW(H5E_CANTENCODE, "can't encode extensible array data elements")
+        p += (dblk_page->hdr->dblk_page_nelmts * dblk_page->hdr->cparam.raw_elmt_size);
+
+        /* Compute metadata checksum */
+        metadata_chksum = H5_checksum_metadata(buf, (size_t)(p - buf), 0);
+
+        /* Metadata checksum */
+        UINT32ENCODE(p, metadata_chksum);
+
+	/* Write the data block */
+        HDassert((size_t)(p - buf) == size);
+	if(H5F_block_write(f, H5FD_MEM_EARRAY_DBLK_PAGE, addr, size, dxpl_id, buf) < 0)
+            H5E_THROW(H5E_WRITEERROR, "unable to save extensible array data block page to disk")
+
+	dblk_page->cache_info.is_dirty = FALSE;
+    } /* end if */
+
+    if(destroy)
+        if(H5EA__cache_dblk_page_dest(f, dblk_page) < 0)
+            H5E_THROW(H5E_CANTFREE, "unable to destroy extensible array data block page")
+
+CATCH
+
+    /* Release resources */
+    if(wb && H5WB_unwrap(wb) < 0)
+	H5E_THROW(H5E_CLOSEERROR, "can't close wrapped buffer")
+
+END_FUNC(STATIC)   /* end H5EA__cache_dblk_page_flush() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5EA__cache_dblk_page_clear
+ *
+ * Purpose:	Mark a extensible array data block page in memory as non-dirty.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Quincey Koziol
+ *		koziol@hdfgroup.org
+ *		Nov 20 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+BEGIN_FUNC(STATIC, ERR,
+herr_t, SUCCEED, FAIL,
+H5EA__cache_dblk_page_clear(H5F_t *f, H5EA_dblk_page_t *dblk_page, hbool_t destroy))
+
+    /* Sanity check */
+    HDassert(dblk_page);
+
+    /* Reset the dirty flag */
+    dblk_page->cache_info.is_dirty = FALSE;
+
+    if(destroy)
+        if(H5EA__cache_dblk_page_dest(f, dblk_page) < 0)
+            H5E_THROW(H5E_CANTFREE, "unable to destroy extensible array data block page")
+
+CATCH
+
+END_FUNC(STATIC)   /* end H5EA__cache_dblk_page_clear() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5EA__cache_dblk_page_size
+ *
+ * Purpose:	Compute the size in bytes of a extensible array data block page
+ *		on disk, and return it in *size_ptr.  On failure,
+ *		the value of *size_ptr is undefined.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Quincey Koziol
+ *		koziol@hdfgroup.org
+ *		Nov 20 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+/* ARGSUSED */
+BEGIN_FUNC(STATIC, NOERR,
+herr_t, SUCCEED, -,
+H5EA__cache_dblk_page_size(const H5F_t UNUSED *f, const H5EA_dblk_page_t *dblk_page,
+    size_t *size_ptr))
+
+    /* Sanity check */
+    HDassert(f);
+    HDassert(dblk_page);
+    HDassert(size_ptr);
+
+    /* Set size value */
+    *size_ptr = dblk_page->size;
+
+END_FUNC(STATIC)   /* end H5EA__cache_dblk_page_size() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5EA__cache_dblk_page_dest
+ *
+ * Purpose:	Destroys an extensible array data block page in memory.
+ *
+ * Note:	Does _not_ free the space for the page on disk, that is
+ *              handled through the data block that "owns" the page.
+ *
+ * Return:	Non-negative on success/Negative on failure
+ *
+ * Programmer:	Quincey Koziol
+ *		koziol@hdfgroup.org
+ *		Nov 20 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+/* ARGSUSED */
+BEGIN_FUNC(STATIC, ERR,
+herr_t, SUCCEED, FAIL,
+H5EA__cache_dblk_page_dest(H5F_t UNUSED *f, H5EA_dblk_page_t *dblk_page))
+
+    /* Sanity check */
+    HDassert(f);
+    HDassert(dblk_page);
+
+    /* Verify that data block page is clean */
+    HDassert(dblk_page->cache_info.is_dirty == FALSE);
+
+    /* Release the data block page */
+    if(H5EA__dblk_page_dest(dblk_page) < 0)
+        H5E_THROW(H5E_CANTFREE, "can't free extensible array data block page")
+
+CATCH
+
+END_FUNC(STATIC)   /* end H5EA__cache_dblk_page_dest() */
+