diff options
| -rw-r--r-- | release_docs/RELEASE.txt | 10 | ||||
| -rw-r--r-- | src/H5FDcore.c | 388 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 2 | ||||
| -rw-r--r-- | src/H5Pfapl.c | 93 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 4 | ||||
| -rw-r--r-- | test/Makefile.am | 2 | ||||
| -rw-r--r-- | test/Makefile.in | 3 | ||||
| -rw-r--r-- | test/h5test.c | 104 | ||||
| -rw-r--r-- | test/vfd.c | 13 |
9 files changed, 529 insertions, 90 deletions
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 47de245..1187881 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -73,6 +73,16 @@ New Features than the application, such as when a debug memory manager is in use or when the HDF5 library is wrapped for use in a managed language like Python or Java. Fixes HDFFV-7710, 8519, and 8851. (DER - 2014/04/11) + - The core VFD can now be configured to track dirty regions in the file + and only write out the changed regions on flush/close. Additionally, + a "page aggregation" size can be set that will aggregate small writes + into larger writes. e.g., setting a 1 MiB page aggregation size will + logically partition the the in-memory file into 1 MiB pages that will be + written out in their entirety if even a single byte is dirtied. The + feature is controlled via the new H5Pset/get_core_write_tracking() API + call. A new "core_paged" target has been added to the check-vfd target + in test/Makefile.am that exercises the feature over all HDF5 VFD-aware + tests. (DER - 2014/04/12) Parallel Library ---------------- diff --git a/src/H5FDcore.c b/src/H5FDcore.c index 7721f12..7c7dd39 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -25,19 +25,26 @@ /* Interface initialization */ #define H5_INTERFACE_INIT_FUNC H5FD_core_init_interface - #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ #include "H5FDcore.h" /* Core file driver */ +#include "H5FLprivate.h" /* Free lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5SLprivate.h" /* Skip lists */ /* The driver identification number, initialized at runtime */ static hid_t H5FD_CORE_g = 0; +/* The skip list node type. Represents a region in the file. */ +typedef struct H5FD_core_region_t { + haddr_t start; /* Start address of the region */ + haddr_t end; /* End address of the region */ +} H5FD_core_region_t; + /* The description of a file belonging to this driver. The 'eoa' and 'eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying memory). @@ -50,6 +57,7 @@ typedef struct H5FD_core_t { haddr_t eof; /* current allocated size */ size_t increment; /* multiples for mem allocation */ hbool_t backing_store; /* write to file name on flush */ + size_t bstore_page_size; /* backing store page size */ int fd; /* backing store file descriptor */ /* Information for determining uniqueness of a file with a backing store */ #ifndef H5_HAVE_WIN32_API @@ -85,6 +93,7 @@ typedef struct H5FD_core_t { #endif /* H5_HAVE_WIN32_API */ hbool_t dirty; /* changes not saved? */ H5FD_file_image_callbacks_t fi_callbacks; /* file image callbacks */ + H5SL_t *dirty_list; /* dirty parts of the file */ } H5FD_core_t; /* Driver-specific file access properties */ @@ -117,6 +126,9 @@ typedef struct H5FD_core_fapl_t { (size_t)((A)+(Z))<(size_t)(A)) /* Prototypes */ +static herr_t H5FD_core_add_dirty_region(H5FD_core_t *file, haddr_t start, haddr_t end); +static herr_t H5FD_core_destroy_dirty_list(H5FD_core_t *file); +static herr_t H5FD_core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size); static void *H5FD_core_fapl_get(H5FD_t *_file); static H5FD_t *H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); @@ -168,6 +180,251 @@ static const H5FD_class_t H5FD_core_g = { H5FD_FLMAP_DICHOTOMY /* fl_map */ }; +/* Define a free list to manage the region type */ +H5FL_DEFINE(H5FD_core_region_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD_core_add_dirty_region + * + * Purpose: Add a new dirty region to the list for later flushing + * to the backing store. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_core_add_dirty_region(H5FD_core_t *file, haddr_t start, haddr_t end) +{ + H5FD_core_region_t *b_item = NULL; + H5FD_core_region_t *a_item = NULL; + H5FD_core_region_t *item = NULL; + haddr_t b_addr = 0; + haddr_t a_addr = 0; + hbool_t create_new_node = TRUE; + herr_t ret_value = SUCCEED; +#ifdef DER + hbool_t was_adjusted = FALSE; +#endif + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(file); + HDassert(file->dirty_list); + HDassert(start <= end); + +#ifdef DER +fprintf(stderr, "Add region: (%llu, %llu)\n", start, end); +#endif + + /* Adjust the dirty region to the nearest block boundaries */ + if(start % file->bstore_page_size != 0) { + start = (start / file->bstore_page_size) * file->bstore_page_size; +#ifdef DER + was_adjusted = TRUE; +#endif + } + if(end % file->bstore_page_size != (file->bstore_page_size - 1)) { + end = (((end / file->bstore_page_size) + 1) * file->bstore_page_size) - 1; + if(end > file->eof){ +#ifdef DER +fprintf(stderr, "Adjusted to EOF\n"); +#endif + end = file->eof - 1; + } +#ifdef DER + was_adjusted = TRUE; +#endif + } + +#ifdef DER +if(was_adjusted) + fprintf(stderr, "Adjusted region: (%llu, %llu)\n", start, end); +#endif + + /* Get the regions before and after the intended insertion point */ + b_addr = start +1; + a_addr = end + 2; + b_item = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &b_addr); + a_item = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &a_addr); + + /* Check to see if we need to extend the upper end of the NEW region */ + if(a_item) { + if(start < a_item->start && end < a_item->end) { + + /* Extend the end of the NEW region to match the existing AFTER region */ + end = a_item->end; + } + } + /* Attempt to extend the PREV region */ + if(b_item) { + if(start <= b_item->end + 1) { + + /* Need to set this for the delete algorithm */ + start = b_item->start; + + /* We won't need to insert a new node since we can + * just update an existing one instead. + */ + create_new_node = FALSE; + } + } + + /* Remove any old nodes that are no longer needed */ + while(a_item && a_item->start > start) { + + H5FD_core_region_t *less; + haddr_t key = a_item->start - 1; + + /* Save the previous node before we trash this one */ + less = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &key); + + /* Delete this node */ + a_item = (H5FD_core_region_t *)H5SL_remove(file->dirty_list, &a_item->start); + a_item = H5FL_FREE(H5FD_core_region_t, a_item); + + /* Set up to check the next node */ + if(less) + a_item = less; + } + + /* Insert the new node */ + if(create_new_node) { + if(NULL == (item = (H5FD_core_region_t *)H5SL_search(file->dirty_list, &start))) { + /* Ok to insert. No pre-existing node with that key. */ + item = (H5FD_core_region_t *)H5FL_CALLOC(H5FD_core_region_t); + item->start = start; + item->end = end; + if(H5SL_insert(file->dirty_list, item, &item->start) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, FAIL, "can't insert new dirty region: (%llu, %llu)\n", start, end) + } + else { + /* Store the new item endpoint if it's bigger */ + item->end = (item->end < end) ? end : item->end; + } + } + else { + /* Update the size of the before region */ + if(b_item->end < end) + b_item->end = end; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_core_add_dirty_region() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_core_destroy_dirty_list + * + * Purpose: Completely destroy the dirty list. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_core_destroy_dirty_list(H5FD_core_t *file) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(file); + + /* Destroy the list, including any remaining list elements */ + if(file->dirty_list) { + H5FD_core_region_t *region = NULL; + +#ifdef DER +{ +size_t count = H5SL_count(file->dirty_list); +if(count != 0) + fprintf(stderr, "LIST NOT EMPTY AT DESTROY\n"); +} +#endif + while(NULL != (region = (H5FD_core_region_t *)H5SL_remove_first(file->dirty_list))) + region = H5FL_FREE(H5FD_core_region_t, region); + + if(H5SL_close(file->dirty_list) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CLOSEERROR, FAIL, "can't close core vfd dirty list") + file->dirty_list = NULL; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_core_destroy_dirty_list() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_core_write_to_bstore + * + * Purpose: Write data to the backing store. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t H5FD_core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size) +{ + unsigned char *ptr = file->mem + addr; /* mutable pointer into the + * buffer (can't change mem) + */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(file); + + /* Write to backing store */ + if((off_t)addr != HDlseek(file->fd, (off_t)addr, SEEK_SET)) + HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store") + + while (size > 0) { + + h5_posix_io_t bytes_in = 0; /* # of bytes to write */ + h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */ + + /* Trying to write more bytes than the return type can handle is + * undefined behavior in POSIX. + */ + if(size > H5_POSIX_MAX_IO_BYTES) + bytes_in = H5_POSIX_MAX_IO_BYTES; + else + bytes_in = (h5_posix_io_t)size; + +#ifdef DER +fprintf(stderr, "\nNEW\n"); +#endif + do { + bytes_wrote = HDwrite(file->fd, ptr, bytes_in); +#ifdef DER +fprintf(stderr, "bytes wrote: %lu\n", bytes_wrote); +#endif + } while(-1 == bytes_wrote && EINTR == errno); + + if(-1 == bytes_wrote) { /* error */ + int myerrno = errno; + time_t mytime = HDtime(NULL); + HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR); + + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', ptr = %p, total write size = %llu, bytes this sub-write = %llu, bytes actually written = %llu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_wrote, (unsigned long long)myoffset); + } /* end if */ + + HDassert(bytes_wrote > 0); + HDassert((size_t)bytes_wrote <= size); + + size -= (size_t)bytes_wrote; + ptr = (unsigned char *)ptr + bytes_wrote; + + } /* end while */ + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_core_write_to_bstore() */ + /*------------------------------------------------------------------------- * Function: H5FD_core_init_interface @@ -564,6 +821,40 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) } /* end if */ } /* end if */ + /* Set up write tracking if the backing store is on */ + file->dirty_list = NULL; + if(fa->backing_store) { + hbool_t write_tracking_flag = FALSE; /* what the user asked for */ + hbool_t use_write_tracking = FALSE; /* what we're actually doing */ + + /* Get the write tracking flag */ + if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, &write_tracking_flag) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get core VFD write tracking flag"); + + /* Get the page size */ + if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, &(file->bstore_page_size)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get core VFD write tracking page size"); + + /* default is to have write tracking OFF for create (hence the check to see + * if the user explicitly set a page size) and ON with the default page size + * on open (when not read-only). + */ + /* Only use write tracking if the file is open for writing */ + use_write_tracking = + TRUE == write_tracking_flag /* user asked for write tracking */ + && !(o_flags & O_RDONLY) /* file is open for writing (i.e. not read-only) */ + && file->bstore_page_size != 0; /* page size is not zero */ + + /* initialize the dirty list */ + if(use_write_tracking) { + if(NULL == (file->dirty_list = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_SLIST, H5E_CANTCREATE, NULL, "can't create core vfd dirty region list"); +#ifdef DER +fprintf(stderr, "\n"); +#endif + } /* end if */ + } /* end if */ + /* Set return value */ ret_value = (H5FD_t *)file; @@ -602,7 +893,12 @@ H5FD_core_close(H5FD_t *_file) /* Flush any changed buffers */ if(H5FD_core_flush(_file, (hid_t)-1, TRUE) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file") + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush core vfd backing store") + + /* Destroy the dirty region list */ + if(file->dirty_list) + if(H5FD_core_destroy_dirty_list(file) != SUCCEED) + HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free core vfd dirty region list") /* Release resources */ if(file->fd >= 0) @@ -1023,6 +1319,14 @@ H5FD_core_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, had file->eof = new_eof; } /* end if */ + /* Add the buffer region to the dirty list if using that optimization */ + if(file->dirty_list) { + haddr_t start = addr; + haddr_t end = addr + (haddr_t)size - 1; + if(H5FD_core_add_dirty_region(file, start, end) != SUCCEED) + HGOTO_ERROR(H5E_VFL, H5E_CANTINSERT, FAIL, "unable to add core VFD dirty region during write call - addresses: start=%llu end=%llu", start, end) + } + /* Write from BUF to memory */ HDmemcpy(file->mem + addr, buf, size); @@ -1057,45 +1361,50 @@ H5FD_core_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned UNUSED closing) /* Write to backing store */ if (file->dirty && file->fd >= 0 && file->backing_store) { - haddr_t size = file->eof; - unsigned char *ptr = file->mem; - if(0 != HDlseek(file->fd, (off_t)0, SEEK_SET)) - HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store") - - while (size > 0) { + /* Use the dirty list, if available */ + if(file->dirty_list) { + H5FD_core_region_t *item = NULL; + size_t size; - h5_posix_io_t bytes_in = 0; /* # of bytes to write */ - h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */ +#ifdef DER + fprintf(stderr, "FLUSHING. DIRTY LIST:\n"); +#endif + while(NULL != (item = (H5FD_core_region_t *)H5SL_remove_first(file->dirty_list))) { - /* Trying to write more bytes than the return type can handle is - * undefined behavior in POSIX. - */ - if(size > H5_POSIX_MAX_IO_BYTES) - bytes_in = H5_POSIX_MAX_IO_BYTES; - else - bytes_in = (h5_posix_io_t)size; - - do { - bytes_wrote = HDwrite(file->fd, ptr, bytes_in); - } while(-1 == bytes_wrote && EINTR == errno); - - if(-1 == bytes_wrote) { /* error */ - int myerrno = errno; - time_t mytime = HDtime(NULL); - HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR); - - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', ptr = %p, total write size = %llu, bytes this sub-write = %llu, bytes actually written = %llu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_wrote, (unsigned long long)myoffset); - } /* end if */ - - HDassert(bytes_wrote > 0); - HDassert((size_t)bytes_wrote <= size); + /* The file may have been truncated, so check for that + * and skip or adjust as necessary. + */ + if(item->start < file->eof) { + if(item->end >= file->eof) + item->end = file->eof - 1; + size = (size_t)((item->end - item->start) + 1); +#ifdef DER +fprintf(stderr, "(%llu, %llu : %lu)\n", item->start, item->end, size); +#endif + if(H5FD_core_write_to_bstore(file, item->start, size) != SUCCEED) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write to backing store") + } /* end if */ + + item = H5FL_FREE(H5FD_core_region_t, item); + } /* end while */ + + +#ifdef DER +fprintf(stderr, "EOF: %llu\n", file->eof); +fprintf(stderr, "EOA: %llu\n", file->eoa); +if(file->eoa > file->eof) + fprintf(stderr, "*** EOA BADNESS ***\n"); +fprintf(stderr, "\n"); +#endif + } + /* Otherwise, write the entire file out at once */ + else { + if(H5FD_core_write_to_bstore(file, (haddr_t)0, (size_t)file->eof) != SUCCEED) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write to backing store") - size -= (size_t)bytes_wrote; - ptr = (unsigned char *)ptr + bytes_wrote; - } /* end while */ - + file->dirty = FALSE; } @@ -1215,10 +1524,14 @@ H5FD_core_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t closing) * re-extended later. This may happen on Open VMS. */ if(-1 == HDlseek(file->fd, (HDoff_t)0, SEEK_SET)) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") -#endif +#endif /* H5_VMS */ if(-1 == HDftruncate(file->fd, (HDoff_t)new_eof)) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") #endif /* H5_HAVE_WIN32_API */ + +#ifdef DER +fprintf(stderr, "OLD: Truncated to: %llu\n", file->eoa); +#endif } /* end if */ /* Update the eof value */ @@ -1229,3 +1542,4 @@ H5FD_core_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t closing) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_core_truncate() */ + diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 1a21ac3..4d6e5ba 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -402,6 +402,8 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_ACS_WANT_POSIX_FD_NAME "want_posix_fd" /* Internal: query the file descriptor from the core VFD, instead of the memory address */ #define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */ #define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */ +#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */ +#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index ad717e7..ee483cd 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -129,7 +129,12 @@ #define H5F_ACS_FILE_IMAGE_INFO_DEL H5P_file_image_info_del #define H5F_ACS_FILE_IMAGE_INFO_COPY H5P_file_image_info_copy #define H5F_ACS_FILE_IMAGE_INFO_CLOSE H5P_file_image_info_close - +/* Definition of core VFD write tracking flag */ +#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_SIZE sizeof(hbool_t) +#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF FALSE +/* Definition of core VFD write tracking page size */ +#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_SIZE sizeof(size_t) +#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF 524288 /******************/ /* Local Typedefs */ @@ -186,7 +191,6 @@ const H5P_libclass_t H5P_CLS_FACC[1] = {{ /* Local Variables */ /*******************/ - /*------------------------------------------------------------------------- * Function: H5P_facc_reg_prop @@ -223,6 +227,8 @@ H5P_facc_reg_prop(H5P_genclass_t *pclass) hbool_t want_posix_fd = H5F_ACS_WANT_POSIX_FD_DEF; /* Default setting for retrieving 'handle' from core VFD */ unsigned efc_size = H5F_ACS_EFC_SIZE_DEF; /* Default external file cache size */ H5FD_file_image_info_t file_image_info = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */ + hbool_t core_write_tracking_flag = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */ + size_t core_write_tracking_page_size = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -312,6 +318,16 @@ H5P_facc_reg_prop(H5P_genclass_t *pclass) if(H5P_register_real(pclass, H5F_ACS_FILE_IMAGE_INFO_NAME, H5F_ACS_FILE_IMAGE_INFO_SIZE, &file_image_info, NULL, NULL, NULL, H5F_ACS_FILE_IMAGE_INFO_DEL, H5F_ACS_FILE_IMAGE_INFO_COPY, NULL, H5F_ACS_FILE_IMAGE_INFO_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the core VFD backing store write tracking flag */ + if(H5P_register_real(pclass, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, H5F_ACS_CORE_WRITE_TRACKING_FLAG_SIZE, &core_write_tracking_flag, + NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the size of the core VFD backing store page size */ + if(H5P_register_real(pclass, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_SIZE, &core_write_tracking_page_size, + NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_facc_reg_prop() */ @@ -2347,3 +2363,76 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_file_image_info_close() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_core_write_tracking + * + * Purpose: Enables/disables core VFD write tracking and page + * aggregation size. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ibz", plist_id, is_enabled, page_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, &is_enabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD write tracking flag") + if(H5P_set(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, &page_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD write tracking page size") + +done: + FUNC_LEAVE_API(ret_value) +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_core_write_tracking + * + * Purpose: Gets information about core VFD write tracking and page + * aggregation size. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_core_write_tracking(hid_t plist_id, hbool_t *is_enabled, size_t *page_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*b*z", plist_id, is_enabled, page_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get values */ + if(is_enabled) { + if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, is_enabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get core VFD write tracking flag") + } /* end if */ + + if(page_size) { + if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, page_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get core VFD write tracking page size") + } /* end if */ + +done: + FUNC_LEAVE_API(ret_value) +} + diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 10fa782..1d8ced4 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -293,7 +293,6 @@ H5_DLL herr_t H5Pget_shared_mesg_index(hid_t plist_id, unsigned index_num, unsig H5_DLL herr_t H5Pset_shared_mesg_phase_change(hid_t plist_id, unsigned max_list, unsigned min_btree); H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *min_btree); - /* File access property list (FAPL) routines */ H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold, hsize_t alignment); @@ -341,6 +340,9 @@ H5_DLL herr_t H5Pset_file_image_callbacks(hid_t fapl_id, H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callbacks_ptr); +H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size); +H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size); + /* Dataset creation property list (DCPL) routines */ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); H5_DLL H5D_layout_t H5Pget_layout(hid_t plist_id); diff --git a/test/Makefile.am b/test/Makefile.am index 18bb88d..08b8b4f 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -103,7 +103,7 @@ LDADD=libh5test.la $(LIBHDF5) ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \ ttsafe_acreate.c -VFD_LIST = sec2 stdio core split multi family +VFD_LIST = sec2 stdio core core_paged split multi family if DIRECT_VFD_CONDITIONAL VFD_LIST += direct endif diff --git a/test/Makefile.in b/test/Makefile.in index 4850cf7..9b435de 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -1119,7 +1119,8 @@ LDADD = libh5test.la $(LIBHDF5) ttsafe_SOURCES = ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \ ttsafe_acreate.c -VFD_LIST = sec2 stdio core split multi family $(am__append_4) +VFD_LIST = sec2 stdio core core_paged split multi family \ + $(am__append_4) # Sources for testhdf5 executable testhdf5_SOURCES = testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ diff --git a/test/h5test.c b/test/h5test.c index d3fdc8d..2afda36 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -511,64 +511,71 @@ h5_fileaccess(void) /* First use the environment variable, then the constant */ val = HDgetenv("HDF5_DRIVER"); #ifdef HDF5_DRIVER - if (!val) val = HDF5_DRIVER; + if (!val) + val = HDF5_DRIVER; #endif - if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1; - if (!val || !*val) return fapl; /*use default*/ + if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) + return -1; + if (!val || !*val) + return fapl; /*use default*/ HDstrncpy(s, val, sizeof s); s[sizeof(s)-1] = '\0'; if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl; if (!HDstrcmp(name, "sec2")) { - /* Unix read() and write() system calls */ - if (H5Pset_fapl_sec2(fapl)<0) return -1; + /* Unix read() and write() system calls */ + if (H5Pset_fapl_sec2(fapl)<0) return -1; } else if (!HDstrcmp(name, "stdio")) { - /* Standard C fread() and fwrite() system calls */ - if (H5Pset_fapl_stdio(fapl)<0) return -1; + /* Standard C fread() and fwrite() system calls */ + if (H5Pset_fapl_stdio(fapl)<0) return -1; } else if (!HDstrcmp(name, "core")) { - /* In-core temporary file with 1MB increment */ - if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1; - } else if (!HDstrcmp(name, "split")) { - /* Split meta data and raw data each using default driver */ - if (H5Pset_fapl_split(fapl, + /* In-memory driver settings (backing store on, 1 MB increment) */ + if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1; + } else if (!HDstrcmp(name, "core_paged")) { + /* In-memory driver with write tracking and paging on */ + if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1; + if (H5Pset_core_write_tracking(fapl, TRUE, (size_t)524288)<0) return -1; + } else if (!HDstrcmp(name, "split")) { + /* Split meta data and raw data each using default driver */ + if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT)<0) - return -1; + return -1; } else if (!HDstrcmp(name, "multi")) { - /* Multi-file driver, general case of the split driver */ - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; - hid_t memb_fapl[H5FD_MEM_NTYPES]; - const char *memb_name[H5FD_MEM_NTYPES]; - char sv[H5FD_MEM_NTYPES][1024]; - haddr_t memb_addr[H5FD_MEM_NTYPES]; + /* Multi-file driver, general case of the split driver */ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl[H5FD_MEM_NTYPES]; + const char *memb_name[H5FD_MEM_NTYPES]; + char sv[H5FD_MEM_NTYPES][1024]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; H5FD_mem_t mt; - HDmemset(memb_map, 0, sizeof memb_map); - HDmemset(memb_fapl, 0, sizeof memb_fapl); - HDmemset(memb_name, 0, sizeof memb_name); - HDmemset(memb_addr, 0, sizeof memb_addr); - - HDassert(HDstrlen(multi_letters)==H5FD_MEM_NTYPES); - for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { - memb_fapl[mt] = H5P_DEFAULT; - sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); - memb_name[mt] = sv[mt]; - memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); - } /* end for */ - - if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, - memb_addr, FALSE)<0) { - return -1; - } + HDmemset(memb_map, 0, sizeof memb_map); + HDmemset(memb_fapl, 0, sizeof memb_fapl); + HDmemset(memb_name, 0, sizeof memb_name); + HDmemset(memb_addr, 0, sizeof memb_addr); + + HDassert(HDstrlen(multi_letters)==H5FD_MEM_NTYPES); + for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) { + memb_fapl[mt] = H5P_DEFAULT; + sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); + memb_name[mt] = sv[mt]; + memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); + } /* end for */ + + if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, + memb_addr, FALSE)<0) { + return -1; + } } else if (!HDstrcmp(name, "family")) { hsize_t fam_size = 100*1024*1024; /*100 MB*/ - /* Family of files, each 1MB and using the default driver */ - if ((val=HDstrtok(NULL, " \t\n\r"))) - fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024); - if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0) + /* Family of files, each 1MB and using the default driver */ + if ((val=HDstrtok(NULL, " \t\n\r"))) + fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024); + if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0) return -1; } else if (!HDstrcmp(name, "log")) { unsigned log_flags = H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC; @@ -578,20 +585,21 @@ h5_fileaccess(void) log_flags = (unsigned)HDstrtol(val, NULL, 0); if (H5Pset_fapl_log(fapl, NULL, log_flags, (size_t)0) < 0) - return -1; + return -1; } else if (!HDstrcmp(name, "direct")) { #ifdef H5_HAVE_DIRECT - /* Linux direct read() and write() system calls. Set memory boundary, file block size, - * and copy buffer size to the default values. */ - if (H5Pset_fapl_direct(fapl, 1024, 4096, 8*4096)<0) return -1; + /* Linux direct read() and write() system calls. Set memory boundary, file block size, + * and copy buffer size to the default values. */ + if (H5Pset_fapl_direct(fapl, 1024, 4096, 8*4096)<0) + return -1; #endif } else if(!HDstrcmp(name, "latest")) { - /* use the latest format */ - if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + /* use the latest format */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) return -1; } else { - /* Unknown driver */ - return -1; + /* Unknown driver */ + return -1; } return fapl; @@ -28,6 +28,7 @@ #define FAMILY_SIZE2 (5*KB) #define MULTI_SIZE 128 #define CORE_INCREMENT (4*KB) +#define CORE_PAGE_SIZE (1024 * 1024) #define DSET1_NAME "dset1" #define DSET1_DIM1 1024 #define DSET1_DIM2 32 @@ -376,6 +377,8 @@ test_core(void) char filename[1024]; void *fhandle=NULL; hsize_t file_size; + hbool_t use_write_tracking; + size_t write_tracking_page_size; int *points = NULL, *check = NULL, *p1, *p2; hid_t dset1=-1, space1=-1; hsize_t dims1[2]; @@ -387,6 +390,8 @@ test_core(void) fapl = h5_fileaccess(); if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0) TEST_ERROR; + if(H5Pset_core_write_tracking(fapl, TRUE, CORE_PAGE_SIZE) < 0) + TEST_ERROR; h5_fixname(FILENAME[1], fapl, filename, sizeof filename); if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) @@ -400,6 +405,14 @@ test_core(void) if(H5FD_CORE != H5Pget_driver(access_fapl)) TEST_ERROR; + /* Check that the backing store write tracking info was saved */ + if(H5Pget_core_write_tracking(fapl, &use_write_tracking, &write_tracking_page_size) < 0) + TEST_ERROR; + if(TRUE != use_write_tracking) + TEST_ERROR; + if(CORE_PAGE_SIZE != write_tracking_page_size) + TEST_ERROR; + /* ...and close the property list */ if (H5Pclose(access_fapl) < 0) TEST_ERROR; |