summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/H5FDcore.c387
-rw-r--r--src/H5Fprivate.h2
-rw-r--r--src/H5Pfapl.c101
-rw-r--r--src/H5Ppublic.h4
-rw-r--r--test/Makefile.am2
-rw-r--r--test/Makefile.in3
-rw-r--r--test/enc_dec_plist.c2
-rw-r--r--test/gen_plist.c2
-rw-r--r--test/h5test.c104
-rw-r--r--test/testfiles/plist_files/fapl_bebin1402 -> 1462 bytes
-rw-r--r--test/testfiles/plist_files/fapl_lebin1402 -> 1462 bytes
-rw-r--r--test/testfiles/plist_files/lapl_bebin1502 -> 1562 bytes
-rw-r--r--test/testfiles/plist_files/lapl_lebin1502 -> 1562 bytes
-rw-r--r--test/vfd.c13
14 files changed, 529 insertions, 91 deletions
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index 74aeed3..2f51264 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -25,19 +25,26 @@
/* Interface initialization */
#define H5_INTERFACE_INIT_FUNC H5FD_core_init_interface
-
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fprivate.h" /* File access */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDcore.h" /* Core file driver */
+#include "H5FLprivate.h" /* Free lists */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
#include "H5Pprivate.h" /* Property lists */
+#include "H5SLprivate.h" /* Skip lists */
/* The driver identification number, initialized at runtime */
static hid_t H5FD_CORE_g = 0;
+/* The skip list node type. Represents a region in the file. */
+typedef struct H5FD_core_region_t {
+ haddr_t start; /* Start address of the region */
+ haddr_t end; /* End address of the region */
+} H5FD_core_region_t;
+
/* The description of a file belonging to this driver. The 'eoa' and 'eof'
* determine the amount of hdf5 address space in use and the high-water mark
* of the file (the current size of the underlying memory).
@@ -50,6 +57,7 @@ typedef struct H5FD_core_t {
haddr_t eof; /* current allocated size */
size_t increment; /* multiples for mem allocation */
hbool_t backing_store; /* write to file name on flush */
+ size_t bstore_page_size; /* backing store page size */
int fd; /* backing store file descriptor */
/* Information for determining uniqueness of a file with a backing store */
#ifndef H5_HAVE_WIN32_API
@@ -85,6 +93,7 @@ typedef struct H5FD_core_t {
#endif /* H5_HAVE_WIN32_API */
hbool_t dirty; /* changes not saved? */
H5FD_file_image_callbacks_t fi_callbacks; /* file image callbacks */
+ H5SL_t *dirty_list; /* dirty parts of the file */
} H5FD_core_t;
/* Driver-specific file access properties */
@@ -96,6 +105,9 @@ typedef struct H5FD_core_fapl_t {
/* Allocate memory in multiples of this size by default */
#define H5FD_CORE_INCREMENT 8192
+/* Define the block size for aggregation */
+//#define H5FD_CORE_BLOCK_SIZE 524288 /* 512 K */
+
/* These macros check for overflow of various quantities. These macros
* assume that file_offset_t is signed and haddr_t and size_t are unsigned.
*
@@ -117,6 +129,9 @@ typedef struct H5FD_core_fapl_t {
(size_t)((A)+(Z))<(size_t)(A))
/* Prototypes */
+static herr_t H5FD_core_add_dirty_region(H5FD_core_t *file, haddr_t start, haddr_t end);
+static herr_t H5FD_core_destroy_dirty_list(H5FD_core_t *file);
+static herr_t H5FD_core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size);
static herr_t H5FD_core_term(void);
static void *H5FD_core_fapl_get(H5FD_t *_file);
static H5FD_t *H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id,
@@ -170,6 +185,256 @@ static const H5FD_class_t H5FD_core_g = {
H5FD_FLMAP_DICHOTOMY /* fl_map */
};
+/* Define a free list to manage the region type */
+H5FL_DEFINE(H5FD_core_region_t);
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_core_add_dirty_region
+ *
+ * Purpose: Add a new dirty region to the list for later flushing
+ * to the backing store.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_core_add_dirty_region(H5FD_core_t *file, haddr_t start, haddr_t end)
+{
+ H5FD_core_region_t *b_item = NULL;
+ H5FD_core_region_t *a_item = NULL;
+ H5FD_core_region_t *item = NULL;
+ haddr_t b_addr = 0;
+ haddr_t a_addr = 0;
+ hbool_t create_new_node = TRUE;
+ herr_t ret_value = SUCCEED;
+#ifdef DER
+ hbool_t was_adjusted = FALSE;
+#endif
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HDassert(file);
+ HDassert(file->dirty_list);
+ HDassert(start <= end);
+
+#ifdef DER
+fprintf(stderr, "Add region: (%llu, %llu)\n", start, end);
+#endif
+
+// /* Adjust the dirty region to the nearest block boundaries */
+// if(start % H5FD_CORE_BLOCK_SIZE != 0) {
+// start = (start / H5FD_CORE_BLOCK_SIZE) * H5FD_CORE_BLOCK_SIZE;
+ /* Adjust the dirty region to the nearest block boundaries */
+ if(start % file->bstore_page_size != 0) {
+ start = (start / file->bstore_page_size) * file->bstore_page_size;
+#ifdef DER
+ was_adjusted = TRUE;
+#endif
+ }
+// if(end % H5FD_CORE_BLOCK_SIZE != (H5FD_CORE_BLOCK_SIZE - 1)) {
+// end = (((end / H5FD_CORE_BLOCK_SIZE) + 1) * H5FD_CORE_BLOCK_SIZE) - 1;
+ if(end % file->bstore_page_size != (file->bstore_page_size - 1)) {
+ end = (((end / file->bstore_page_size) + 1) * file->bstore_page_size) - 1;
+ if(end > file->eof){
+#ifdef DER
+fprintf(stderr, "Adjusted to EOF\n");
+#endif
+ end = file->eof - 1;
+ }
+#ifdef DER
+ was_adjusted = TRUE;
+#endif
+ }
+
+#ifdef DER
+if(was_adjusted)
+ fprintf(stderr, "Adjusted region: (%llu, %llu)\n", start, end);
+#endif
+
+ /* Get the regions before and after the intended insertion point */
+ b_addr = start +1;
+ a_addr = end + 2;
+ b_item = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &b_addr);
+ a_item = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &a_addr);
+
+ /* Check to see if we need to extend the upper end of the NEW region */
+ if(a_item) {
+ if(start < a_item->start && end < a_item->end) {
+
+ /* Extend the end of the NEW region to match the existing AFTER region */
+ end = a_item->end;
+ }
+ }
+ /* Attempt to extend the PREV region */
+ if(b_item) {
+ if(start <= b_item->end + 1) {
+
+ /* Need to set this for the delete algorithm */
+ start = b_item->start;
+
+ /* We won't need to insert a new node since we can
+ * just update an existing one instead.
+ */
+ create_new_node = FALSE;
+ }
+ }
+
+ /* Remove any old nodes that are no longer needed */
+ while(a_item && a_item->start > start) {
+
+ H5FD_core_region_t *less;
+ haddr_t key = a_item->start - 1;
+
+ /* Save the previous node before we trash this one */
+ less = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &key);
+
+ /* Delete this node */
+ a_item = (H5FD_core_region_t *)H5SL_remove(file->dirty_list, &a_item->start);
+ a_item = H5FL_FREE(H5FD_core_region_t, a_item);
+
+ /* Set up to check the next node */
+ if(less)
+ a_item = less;
+ }
+
+ /* Insert the new node */
+ if(create_new_node) {
+ if(NULL == (item = (H5FD_core_region_t *)H5SL_search(file->dirty_list, &start))) {
+ /* Ok to insert. No pre-existing node with that key. */
+ item = (H5FD_core_region_t *)H5FL_CALLOC(H5FD_core_region_t);
+ item->start = start;
+ item->end = end;
+ if(H5SL_insert(file->dirty_list, item, &item->start) < 0)
+ HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, FAIL, "can't insert new dirty region: (%llu, %llu)\n", start, end)
+ }
+ else {
+ /* Store the new item endpoint if it's bigger */
+ item->end = (item->end < end) ? end : item->end;
+ }
+ }
+ else {
+ /* Update the size of the before region */
+ if(b_item->end < end)
+ b_item->end = end;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_core_add_dirty_region() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_core_destroy_dirty_list
+ *
+ * Purpose: Completely destroy the dirty list.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_core_destroy_dirty_list(H5FD_core_t *file)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HDassert(file);
+
+ /* Destroy the list, including any remaining list elements */
+ if(file->dirty_list) {
+ H5FD_core_region_t *region = NULL;
+
+#ifdef DER
+{
+size_t count = H5SL_count(file->dirty_list);
+if(count != 0)
+ fprintf(stderr, "LIST NOT EMPTY AT DESTROY\n");
+}
+#endif
+ while(NULL != (region = (H5FD_core_region_t *)H5SL_remove_first(file->dirty_list)))
+ region = H5FL_FREE(H5FD_core_region_t, region);
+
+ if(H5SL_close(file->dirty_list) < 0)
+ HGOTO_ERROR(H5E_SLIST, H5E_CLOSEERROR, FAIL, "can't close core vfd dirty list")
+ file->dirty_list = NULL;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_core_destroy_dirty_list() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_core_write_to_bstore
+ *
+ * Purpose: Write data to the backing store.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t H5FD_core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size)
+{
+ unsigned char *ptr = file->mem + addr; /* mutable pointer into the
+ * buffer (can't change mem)
+ */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HDassert(file);
+
+ /* Write to backing store */
+ if((off_t)addr != HDlseek(file->fd, (off_t)addr, SEEK_SET))
+ HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store")
+
+ while (size > 0) {
+
+ h5_posix_io_t bytes_in = 0; /* # of bytes to write */
+ h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */
+
+ /* Trying to write more bytes than the return type can handle is
+ * undefined behavior in POSIX.
+ */
+ if(size > H5_POSIX_MAX_IO_BYTES)
+ bytes_in = H5_POSIX_MAX_IO_BYTES;
+ else
+ bytes_in = (h5_posix_io_t)size;
+
+#ifdef DER
+fprintf(stderr, "\nNEW\n");
+#endif
+ do {
+ bytes_wrote = HDwrite(file->fd, ptr, bytes_in);
+#ifdef DER
+fprintf(stderr, "bytes wrote: %lu\n", bytes_wrote);
+#endif
+ } while(-1 == bytes_wrote && EINTR == errno);
+
+ if(-1 == bytes_wrote) { /* error */
+ int myerrno = errno;
+ time_t mytime = HDtime(NULL);
+ HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
+
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', ptr = %p, total write size = %llu, bytes this sub-write = %llu, bytes actually written = %llu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_wrote, (unsigned long long)myoffset);
+ } /* end if */
+
+ HDassert(bytes_wrote > 0);
+ HDassert((size_t)bytes_wrote <= size);
+
+ size -= (size_t)bytes_wrote;
+ ptr = (unsigned char *)ptr + bytes_wrote;
+
+ } /* end while */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_core_write_to_bstore() */
+
/*-------------------------------------------------------------------------
* Function: H5FD_core_init_interface
@@ -566,6 +831,40 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
} /* end if */
} /* end if */
+ /* Set up write tracking if the backing store is on */
+ file->dirty_list = NULL;
+ if(fa->backing_store) {
+ hbool_t write_tracking_flag = FALSE; /* what the user asked for */
+ hbool_t use_write_tracking = FALSE; /* what we're actually doing */
+
+ /* Get the write tracking flag */
+ if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, &write_tracking_flag) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get core VFD write tracking flag");
+
+ /* Get the page size */
+ if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, &(file->bstore_page_size)) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get core VFD write tracking page size");
+
+ /* default is to have write tracking OFF for create (hence the check to see
+ * if the user explicitly set a page size) and ON with the default page size
+ * on open (when not read-only).
+ */
+ /* Only use write tracking if the file is open for writing */
+ use_write_tracking =
+ TRUE == write_tracking_flag /* user asked for write tracking */
+ && !(o_flags & O_RDONLY) /* file is open for writing (i.e. not read-only) */
+ && file->bstore_page_size != 0; /* page size is not zero */
+
+ /* initialize the dirty list */
+ if(use_write_tracking) {
+ if(NULL == (file->dirty_list = H5SL_create(H5SL_TYPE_HADDR, NULL)))
+ HGOTO_ERROR(H5E_SLIST, H5E_CANTCREATE, NULL, "can't create core vfd dirty region list");
+#ifdef DER
+fprintf(stderr, "\n");
+#endif
+ } /* end if */
+ } /* end if */
+
/* Set return value */
ret_value = (H5FD_t *)file;
@@ -604,7 +903,12 @@ H5FD_core_close(H5FD_t *_file)
/* Flush any changed buffers */
if(H5FD_core_flush(_file, (hid_t)-1, TRUE) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush core vfd backing store")
+
+ /* Destroy the dirty region list */
+ if(file->dirty_list)
+ if(H5FD_core_destroy_dirty_list(file) != SUCCEED)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free core vfd dirty region list")
/* Release resources */
if(file->fd >= 0)
@@ -1025,6 +1329,14 @@ H5FD_core_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, had
file->eof = new_eof;
} /* end if */
+ /* Add the buffer region to the dirty list if using that optimization */
+ if(file->dirty_list) {
+ haddr_t start = addr;
+ haddr_t end = addr + (haddr_t)size - 1;
+ if(H5FD_core_add_dirty_region(file, start, end) != SUCCEED)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINSERT, FAIL, "unable to add core VFD dirty region during write call - addresses: start=%llu end=%llu", start, end)
+ }
+
/* Write from BUF to memory */
HDmemcpy(file->mem + addr, buf, size);
@@ -1059,45 +1371,41 @@ H5FD_core_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned UNUSED closing)
/* Write to backing store */
if (file->dirty && file->fd >= 0 && file->backing_store) {
- haddr_t size = file->eof;
- unsigned char *ptr = file->mem;
-
- if(0 != HDlseek(file->fd, (off_t)0, SEEK_SET))
- HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store")
-
- while (size > 0) {
- h5_posix_io_t bytes_in = 0; /* # of bytes to write */
- h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */
+ /* Use the dirty list, if available */
+ if(file->dirty_list) {
+ H5FD_core_region_t *item = NULL;
+ size_t size;
- /* Trying to write more bytes than the return type can handle is
- * undefined behavior in POSIX.
- */
- if(size > H5_POSIX_MAX_IO_BYTES)
- bytes_in = H5_POSIX_MAX_IO_BYTES;
- else
- bytes_in = (h5_posix_io_t)size;
-
- do {
- bytes_wrote = HDwrite(file->fd, ptr, bytes_in);
- } while(-1 == bytes_wrote && EINTR == errno);
-
- if(-1 == bytes_wrote) { /* error */
- int myerrno = errno;
- time_t mytime = HDtime(NULL);
- HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
-
- HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', ptr = %p, total write size = %llu, bytes this sub-write = %llu, bytes actually written = %llu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_wrote, (unsigned long long)myoffset);
- } /* end if */
-
- HDassert(bytes_wrote > 0);
- HDassert((size_t)bytes_wrote <= size);
+#ifdef DER
+ fprintf(stderr, "FLUSHING. DIRTY LIST:\n");
+#endif
+ while(NULL != (item = (H5FD_core_region_t *)H5SL_remove_first(file->dirty_list))) {
+ size = (size_t)((item->end - item->start) + 1);
+#ifdef DER
+fprintf(stderr, "(%llu, %llu : %lu)\n", item->start, item->end, size);
+#endif
+ if(H5FD_core_write_to_bstore(file, item->start, size) != SUCCEED)
+ HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write to backing store")
+ item = H5FL_FREE(H5FD_core_region_t, item);
+ }
+
+
+#ifdef DER
+fprintf(stderr, "EOF: %llu\n", file->eof);
+fprintf(stderr, "EOA: %llu\n", file->eoa);
+if(file->eoa > file->eof)
+ fprintf(stderr, "*** EOA BADNESS ***\n");
+fprintf(stderr, "\n");
+#endif
+ }
+ /* Otherwise, write the entire file out at once */
+ else {
+ if(H5FD_core_write_to_bstore(file, (haddr_t)0, (size_t)file->eof) != SUCCEED)
+ HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write to backing store")
- size -= (size_t)bytes_wrote;
- ptr = (unsigned char *)ptr + bytes_wrote;
-
} /* end while */
-
+
file->dirty = FALSE;
}
@@ -1221,6 +1529,10 @@ H5FD_core_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t closing)
if(-1 == HDftruncate(file->fd, (HDoff_t)new_eof))
HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly")
#endif /* H5_HAVE_WIN32_API */
+
+#ifdef DER
+fprintf(stderr, "OLD: Truncated to: %llu\n", file->eoa);
+#endif
} /* end if */
/* Update the eof value */
@@ -1231,3 +1543,4 @@ H5FD_core_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t closing)
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_core_truncate() */
+
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 2690311..cfecf98 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -462,6 +462,8 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t;
#define H5F_ACS_WANT_POSIX_FD_NAME "want_posix_fd" /* Internal: query the file descriptor from the core VFD, instead of the memory address */
#define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */
#define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */
+#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */
+#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */
/* ======================== File Mount properties ====================*/
#define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index a4c09e5..c217acb 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -160,7 +160,16 @@
#define H5F_ACS_FILE_IMAGE_INFO_DEL H5P_file_image_info_del
#define H5F_ACS_FILE_IMAGE_INFO_COPY H5P_file_image_info_copy
#define H5F_ACS_FILE_IMAGE_INFO_CLOSE H5P_file_image_info_close
-
+/* Definition of core VFD write tracking flag */
+#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_SIZE sizeof(hbool_t)
+#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF FALSE
+#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_ENC H5P__encode_hbool_t
+#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEC H5P__decode_hbool_t
+/* Definition of core VFD write tracking page size */
+#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_SIZE sizeof(size_t)
+#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF 524288
+#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC H5P__encode_size_t
+#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC H5P__decode_size_t
/******************/
/* Local Typedefs */
@@ -246,8 +255,9 @@ static const H5FD_mem_t H5F_def_mem_type_g = H5F_ACS_MULTI_TYPE_DEF;
static const hbool_t H5F_def_latest_format_g = H5F_ACS_LATEST_FORMAT_DEF; /* Default setting for "use the latest version of the format" flag */
static const hbool_t H5F_def_want_posix_fd_g = H5F_ACS_WANT_POSIX_FD_DEF; /* Default setting for retrieving 'handle' from core VFD */
static const unsigned H5F_def_efc_size_g = H5F_ACS_EFC_SIZE_DEF; /* Default external file cache size */
-static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */
-
+static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */
+static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */
+static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */
/*-------------------------------------------------------------------------
@@ -397,6 +407,18 @@ H5P_facc_reg_prop(H5P_genclass_t *pclass)
H5F_ACS_FILE_IMAGE_INFO_DEL, H5F_ACS_FILE_IMAGE_INFO_COPY, NULL, H5F_ACS_FILE_IMAGE_INFO_CLOSE) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+ /* Register the core VFD backing store write tracking flag */
+ if(H5P_register_real(pclass, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, H5F_ACS_CORE_WRITE_TRACKING_FLAG_SIZE, &H5F_def_core_write_tracking_flag_g ,
+ NULL, NULL, NULL, H5F_ACS_CORE_WRITE_TRACKING_FLAG_ENC, H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEC,
+ NULL, NULL, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
+ /* Register the size of the core VFD backing store page size */
+ if(H5P_register_real(pclass, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_SIZE, &H5F_def_core_write_tracking_page_size_g ,
+ NULL, NULL, NULL, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC,
+ NULL, NULL, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P_facc_reg_prop() */
@@ -2980,3 +3002,76 @@ H5P__facc_multi_type_dec(const void **_pp, void *_value)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5P__facc_multi_type_dec() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_core_write_tracking
+ *
+ * Purpose: Enables/disables core VFD write tracking and page
+ * aggregation size.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size)
+{
+ H5P_genplist_t *plist; /* Property list pointer */
+ herr_t ret_value = SUCCEED; /* return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE3("e", "ibz", plist_id, is_enabled, page_size);
+
+ /* Get the plist structure */
+ if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+ /* Set values */
+ if(H5P_set(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, &is_enabled) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD write tracking flag")
+ if(H5P_set(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, &page_size) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD write tracking page size")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_core_write_tracking
+ *
+ * Purpose: Gets information about core VFD write tracking and page
+ * aggregation size.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_core_write_tracking(hid_t plist_id, hbool_t *is_enabled, size_t *page_size)
+{
+ H5P_genplist_t *plist; /* Property list pointer */
+ herr_t ret_value = SUCCEED; /* return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE3("e", "i*b*z", plist_id, is_enabled, page_size);
+
+ /* Get the plist structure */
+ if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+ /* Get values */
+ if(is_enabled) {
+ if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, is_enabled) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get core VFD write tracking flag")
+ } /* end if */
+
+ if(page_size) {
+ if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, page_size) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get core VFD write tracking page size")
+ } /* end if */
+
+done:
+ FUNC_LEAVE_API(ret_value)
+}
+
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index d35f4da..fd5ff5f 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -296,7 +296,6 @@ H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list
H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold);
H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold);
-
/* File access property list (FAPL) routines */
H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold,
hsize_t alignment);
@@ -344,6 +343,9 @@ H5_DLL herr_t H5Pset_file_image_callbacks(hid_t fapl_id,
H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id,
H5FD_file_image_callbacks_t *callbacks_ptr);
+H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size);
+H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size);
+
/* Dataset creation property list (DCPL) routines */
H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout);
H5_DLL H5D_layout_t H5Pget_layout(hid_t plist_id);
diff --git a/test/Makefile.am b/test/Makefile.am
index 6938133..b5ebcf3 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -103,7 +103,7 @@ LDADD=libh5test.la $(LIBHDF5)
ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \
ttsafe_acreate.c
-VFD_LIST = sec2 stdio core split multi family
+VFD_LIST = sec2 stdio core core_paged split multi family
if DIRECT_VFD_CONDITIONAL
VFD_LIST += direct
endif
diff --git a/test/Makefile.in b/test/Makefile.in
index 628b004..16071a8 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -1155,7 +1155,8 @@ LDADD = libh5test.la $(LIBHDF5)
ttsafe_SOURCES = ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \
ttsafe_acreate.c
-VFD_LIST = sec2 stdio core split multi family $(am__append_4)
+VFD_LIST = sec2 stdio core core_paged split multi family \
+ $(am__append_4)
# Sources for testhdf5 executable
testhdf5_SOURCES = testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \
diff --git a/test/enc_dec_plist.c b/test/enc_dec_plist.c
index 2d0d732..422ad1b 100644
--- a/test/enc_dec_plist.c
+++ b/test/enc_dec_plist.c
@@ -381,6 +381,8 @@ main(void)
FAIL_STACK_ERROR
if((H5Pset_mdc_config(fapl, &my_cache_config)) < 0)
FAIL_STACK_ERROR
+ if((H5Pset_core_write_tracking(fapl, TRUE, 1024 * 1024)) < 0)
+ FAIL_STACK_ERROR
/* Test encoding & decoding property list */
if(test_encode_decode(fapl) < 0)
diff --git a/test/gen_plist.c b/test/gen_plist.c
index b4da261..acc5f3e 100644
--- a/test/gen_plist.c
+++ b/test/gen_plist.c
@@ -327,6 +327,8 @@ main(void)
assert(ret > 0);
if((ret = H5Pset_mdc_config(fapl1, &my_cache_config)) < 0)
assert(ret > 0);
+ if((ret = H5Pset_core_write_tracking(fapl1, TRUE, (size_t)(1024 * 1024))) < 0)
+ assert(ret > 0);
if((ret = encode_plist(fapl1, little_endian, "testfiles/plist_files/fapl_le", "testfiles/plist_files/fapl_be")) < 0)
assert(ret > 0);
diff --git a/test/h5test.c b/test/h5test.c
index 9fe05ae..769e03f 100644
--- a/test/h5test.c
+++ b/test/h5test.c
@@ -511,64 +511,71 @@ h5_fileaccess(void)
/* First use the environment variable, then the constant */
val = HDgetenv("HDF5_DRIVER");
#ifdef HDF5_DRIVER
- if (!val) val = HDF5_DRIVER;
+ if (!val)
+ val = HDF5_DRIVER;
#endif
- if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1;
- if (!val || !*val) return fapl; /*use default*/
+ if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0)
+ return -1;
+ if (!val || !*val)
+ return fapl; /*use default*/
HDstrncpy(s, val, sizeof s);
s[sizeof(s)-1] = '\0';
if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl;
if (!HDstrcmp(name, "sec2")) {
- /* Unix read() and write() system calls */
- if (H5Pset_fapl_sec2(fapl)<0) return -1;
+ /* Unix read() and write() system calls */
+ if (H5Pset_fapl_sec2(fapl)<0) return -1;
} else if (!HDstrcmp(name, "stdio")) {
- /* Standard C fread() and fwrite() system calls */
- if (H5Pset_fapl_stdio(fapl)<0) return -1;
+ /* Standard C fread() and fwrite() system calls */
+ if (H5Pset_fapl_stdio(fapl)<0) return -1;
} else if (!HDstrcmp(name, "core")) {
- /* In-core temporary file with 1MB increment */
- if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1;
- } else if (!HDstrcmp(name, "split")) {
- /* Split meta data and raw data each using default driver */
- if (H5Pset_fapl_split(fapl,
+ /* In-memory driver settings (backing store on, 1 MB increment) */
+ if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1;
+ } else if (!HDstrcmp(name, "core_paged")) {
+ /* In-memory driver with write tracking and paging on */
+ if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1;
+ if (H5Pset_core_write_tracking(fapl, TRUE, (size_t)4096)<0) return -1;
+ } else if (!HDstrcmp(name, "split")) {
+ /* Split meta data and raw data each using default driver */
+ if (H5Pset_fapl_split(fapl,
"-m.h5", H5P_DEFAULT,
"-r.h5", H5P_DEFAULT)<0)
- return -1;
+ return -1;
} else if (!HDstrcmp(name, "multi")) {
- /* Multi-file driver, general case of the split driver */
- H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
- hid_t memb_fapl[H5FD_MEM_NTYPES];
- const char *memb_name[H5FD_MEM_NTYPES];
- char sv[H5FD_MEM_NTYPES][1024];
- haddr_t memb_addr[H5FD_MEM_NTYPES];
+ /* Multi-file driver, general case of the split driver */
+ H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
+ hid_t memb_fapl[H5FD_MEM_NTYPES];
+ const char *memb_name[H5FD_MEM_NTYPES];
+ char sv[H5FD_MEM_NTYPES][1024];
+ haddr_t memb_addr[H5FD_MEM_NTYPES];
H5FD_mem_t mt;
- HDmemset(memb_map, 0, sizeof memb_map);
- HDmemset(memb_fapl, 0, sizeof memb_fapl);
- HDmemset(memb_name, 0, sizeof memb_name);
- HDmemset(memb_addr, 0, sizeof memb_addr);
-
- HDassert(HDstrlen(multi_letters)==H5FD_MEM_NTYPES);
- for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) {
- memb_fapl[mt] = H5P_DEFAULT;
- sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
- memb_name[mt] = sv[mt];
- memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
- } /* end for */
-
- if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name,
- memb_addr, FALSE)<0) {
- return -1;
- }
+ HDmemset(memb_map, 0, sizeof memb_map);
+ HDmemset(memb_fapl, 0, sizeof memb_fapl);
+ HDmemset(memb_name, 0, sizeof memb_name);
+ HDmemset(memb_addr, 0, sizeof memb_addr);
+
+ HDassert(HDstrlen(multi_letters)==H5FD_MEM_NTYPES);
+ for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) {
+ memb_fapl[mt] = H5P_DEFAULT;
+ sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
+ memb_name[mt] = sv[mt];
+ memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
+ } /* end for */
+
+ if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name,
+ memb_addr, FALSE)<0) {
+ return -1;
+ }
} else if (!HDstrcmp(name, "family")) {
hsize_t fam_size = 100*1024*1024; /*100 MB*/
- /* Family of files, each 1MB and using the default driver */
- if ((val=HDstrtok(NULL, " \t\n\r")))
- fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
- if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
+ /* Family of files, each 1MB and using the default driver */
+ if ((val=HDstrtok(NULL, " \t\n\r")))
+ fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
+ if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
return -1;
} else if (!HDstrcmp(name, "log")) {
unsigned log_flags = H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC;
@@ -578,20 +585,21 @@ h5_fileaccess(void)
log_flags = (unsigned)HDstrtol(val, NULL, 0);
if (H5Pset_fapl_log(fapl, NULL, log_flags, (size_t)0) < 0)
- return -1;
+ return -1;
} else if (!HDstrcmp(name, "direct")) {
#ifdef H5_HAVE_DIRECT
- /* Linux direct read() and write() system calls. Set memory boundary, file block size,
- * and copy buffer size to the default values. */
- if (H5Pset_fapl_direct(fapl, 1024, 4096, 8*4096)<0) return -1;
+ /* Linux direct read() and write() system calls. Set memory boundary, file block size,
+ * and copy buffer size to the default values. */
+ if (H5Pset_fapl_direct(fapl, 1024, 4096, 8*4096)<0)
+ return -1;
#endif
} else if(!HDstrcmp(name, "latest")) {
- /* use the latest format */
- if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+ /* use the latest format */
+ if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
return -1;
} else {
- /* Unknown driver */
- return -1;
+ /* Unknown driver */
+ return -1;
}
return fapl;
diff --git a/test/testfiles/plist_files/fapl_be b/test/testfiles/plist_files/fapl_be
index 8fcefa2..0b050d0 100644
--- a/test/testfiles/plist_files/fapl_be
+++ b/test/testfiles/plist_files/fapl_be
Binary files differ
diff --git a/test/testfiles/plist_files/fapl_le b/test/testfiles/plist_files/fapl_le
index 8fcefa2..0b050d0 100644
--- a/test/testfiles/plist_files/fapl_le
+++ b/test/testfiles/plist_files/fapl_le
Binary files differ
diff --git a/test/testfiles/plist_files/lapl_be b/test/testfiles/plist_files/lapl_be
index 30f52a4..e58bfb4 100644
--- a/test/testfiles/plist_files/lapl_be
+++ b/test/testfiles/plist_files/lapl_be
Binary files differ
diff --git a/test/testfiles/plist_files/lapl_le b/test/testfiles/plist_files/lapl_le
index 30f52a4..e58bfb4 100644
--- a/test/testfiles/plist_files/lapl_le
+++ b/test/testfiles/plist_files/lapl_le
Binary files differ
diff --git a/test/vfd.c b/test/vfd.c
index b85c054..4ece2ba 100644
--- a/test/vfd.c
+++ b/test/vfd.c
@@ -28,6 +28,7 @@
#define FAMILY_SIZE2 (5*KB)
#define MULTI_SIZE 128
#define CORE_INCREMENT (4*KB)
+#define CORE_PAGE_SIZE (1024 * 1024)
#define DSET1_NAME "dset1"
#define DSET1_DIM1 1024
#define DSET1_DIM2 32
@@ -376,6 +377,8 @@ test_core(void)
char filename[1024];
void *fhandle=NULL;
hsize_t file_size;
+ hbool_t use_write_tracking;
+ size_t write_tracking_page_size;
int *points = NULL, *check = NULL, *p1, *p2;
hid_t dset1=-1, space1=-1;
hsize_t dims1[2];
@@ -387,6 +390,8 @@ test_core(void)
fapl = h5_fileaccess();
if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0)
TEST_ERROR;
+ if(H5Pset_core_write_tracking(fapl, TRUE, CORE_PAGE_SIZE) < 0)
+ TEST_ERROR;
h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
@@ -400,6 +405,14 @@ test_core(void)
if(H5FD_CORE != H5Pget_driver(access_fapl))
TEST_ERROR;
+ /* Check that the backing store write tracking info was saved */
+ if(H5Pget_core_write_tracking(fapl, &use_write_tracking, &write_tracking_page_size) < 0)
+ TEST_ERROR;
+ if(TRUE != use_write_tracking)
+ TEST_ERROR;
+ if(CORE_PAGE_SIZE != write_tracking_page_size)
+ TEST_ERROR;
+
/* ...and close the property list */
if (H5Pclose(access_fapl) < 0)
TEST_ERROR;
generated by cgit v0.12 at 2024-11-18 18:00:21 (GMT)