diff options
| -rw-r--r-- | src/H5FDprivate.h | 33 | ||||
| -rw-r--r-- | src/H5FDvfd_swmr.c | 135 | ||||
| -rw-r--r-- | src/H5Fint.c | 196 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 10 | ||||
| -rw-r--r-- | src/H5Ftest.c | 361 | ||||
| -rw-r--r-- | src/H5private.h | 14 | ||||
| -rw-r--r-- | test/vfd_swmr.c | 626 |
7 files changed, 1116 insertions, 259 deletions
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 16bb2b7..45ad4c2 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -85,21 +85,6 @@ #define H5FD_VFD_SWMR_MD_INDEX_RETRY_MAX 5 /* Maximum retries when deserializing the MD file index */ -/* - * fs_page_size: Size of pages in both the HDF5 file and the metadata file IN BYTES - * tick_num: Sequence number of the current tick. - * Initialized to zero on file creation/open, and incremented by the - * VFD SWMR writer at the end of each tick. - * index_offset: The offset of the current metadata file index in the metadata file - * IN BYTES. - * index_length: The length of the current metadata file index IN BYTES. - */ -typedef struct H5FD_vfd_swmr_md_header { - uint32_t fs_page_size; - uint64_t tick_num; - uint64_t index_offset; - uint64_t index_length; -} H5FD_vfd_swmr_md_header; /* Internal representation of metadata file index entry */ /* @@ -179,6 +164,21 @@ typedef struct H5FD_vfd_swmr_md_index { } H5FD_vfd_swmr_md_index; +/* + * fs_page_size: Size of pages in both the HDF5 file and the metadata file IN BYTES + * tick_num: Sequence number of the current tick. + * Initialized to zero on file creation/open, and incremented by the + * VFD SWMR writer at the end of each tick. + * index_offset: The offset of the current metadata file index in the metadata file + * IN BYTES. + * index_length: The length of the current metadata file index IN BYTES. + */ +typedef struct H5FD_vfd_swmr_md_header { + uint32_t fs_page_size; + uint64_t tick_num; + uint64_t index_offset; + uint64_t index_length; +} H5FD_vfd_swmr_md_header; #ifdef H5_HAVE_PARALLEL /* ======== Temporary data transfer properties ======== */ @@ -296,10 +296,13 @@ H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle); H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); +H5_DLL herr_t H5FD_get_driver_name(const H5FD_t *file, char **driver_name); /* Function prototypes for VFD SWMR */ H5_DLL herr_t H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t read_index, uint64_t *tick_ptr, uint32_t *num_entries_ptr, H5FD_vfd_swmr_idx_entry_t index[]); +H5_DLL hbool_t H5FD_is_vfd_swmr_driver(H5FD_t *_file); +H5_DLL H5FD_t *H5FD_vfd_swmr_get_underlying_vfd(H5FD_t *_file); /* Function prototypes for MPI based VFDs*/ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c index 9dcf8ef..7400383 100644 --- a/src/H5FDvfd_swmr.c +++ b/src/H5FDvfd_swmr.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Purpose: VFD SWMR driver + * Purpose: VFD SWMR driver for the reader */ #include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */ @@ -38,7 +38,7 @@ typedef struct H5FD_vfd_swmr_t { H5FD_t *hdf5_file_lf; /* Driver info for the HDF5 file */ /* Metadata file */ - int md_fd; /* File descriptor the metadata file */ + int md_fd; /* File descriptor for the metadata file */ int32_t md_pages_reserved; /* # of pages reserved at the head of the metadata file */ char md_file_path[H5FD_MAX_FILENAME_LEN]; /* Name of the metadate file */ H5FD_vfd_swmr_md_header md_header; /* Metadata file header */ @@ -67,12 +67,12 @@ static herr_t H5FD_vfd_swmr_lock(H5FD_t *_file, hbool_t rw); static herr_t H5FD_vfd_swmr_unlock(H5FD_t *_file); /* VFD SWMR */ -static herr_t H5FD_vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_header); -static herr_t H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index, H5FD_vfd_swmr_md_header *md_header); -static herr_t H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open); +static herr_t H5FD__vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_header); +static herr_t H5FD__vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index, H5FD_vfd_swmr_md_header *md_header); +static herr_t H5FD__vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open); static const H5FD_class_t H5FD_vfd_swmr_g = { - "swmr", /* name */ + "vfd_swmr", /* name */ MAXADDR, /* maxaddr */ H5F_CLOSE_WEAK, /* fc_degree */ H5FD_vfd_swmr_term, /* terminate */ @@ -118,6 +118,7 @@ H5FL_SEQ_DEFINE(H5FD_vfd_swmr_idx_entry_t); * * Purpose: Initializes any interface-specific data or routines. * +b * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- @@ -290,7 +291,7 @@ H5FD_vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxa HGOTO_ERROR(H5E_VFL, H5E_OPENERROR, NULL, "unable to open the metadata file after all retry attempts") /* Retry on loading and decoding the header and index in the metadata file */ - if(H5FD_vfd_swmr_load_hdr_and_idx((H5FD_t *)file, TRUE) < 0) + if(H5FD__vfd_swmr_load_hdr_and_idx((H5FD_t *)file, TRUE) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "unable to load/decode the md file header/index") /* Hard-wired to open the underlying HDF5 file with SEC2 */ @@ -565,17 +566,10 @@ H5FD_vfd_swmr_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, HDassert(buf); /* Try loading and decoding the header and index in the metadata file */ - if(H5FD_vfd_swmr_load_hdr_and_idx(_file, FALSE) < 0) + if(H5FD__vfd_swmr_load_hdr_and_idx(_file, FALSE) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to load/decode the md file header/index") - fs_page_size = file->md_header.fs_page_size; - num_entries = file->md_index.num_entries; - if(num_entries) { - /* Allocate memory for index entries */ - if(NULL == (index = H5FL_SEQ_MALLOC(H5FD_vfd_swmr_idx_entry_t, num_entries))) - HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "memory allocation failed for index entries") - HDmemcpy(index, file->md_index.entries, num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)); - } + index = file->md_index.entries; /* Try finding the addr from the index */ cmp = -1; @@ -583,7 +577,7 @@ H5FD_vfd_swmr_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hi = num_entries; while(lo < hi && cmp) { my_idx = (lo + hi) / 2; - cmp = H5F_addr_cmp(index[my_idx].hdf5_page_offset * fs_page_size, addr); + cmp = H5F_addr_cmp(index[my_idx].hdf5_page_offset * file->md_header.fs_page_size, addr); if(cmp < 0) hi = my_idx; else @@ -648,13 +642,6 @@ H5FD_vfd_swmr_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, } done: - if(ret_value < 0) { - if(index) { - HDassert(num_entries); - index = H5FL_SEQ_FREE(H5FD_vfd_swmr_idx_entry_t, num_entries); - } - } /* end if */ - FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_vfd_swmr_read() */ @@ -778,7 +765,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD_vfd_swmr_load_hdr_and_idx() + * Function: H5FD__vfd_swmr_load_hdr_and_idx() * * Purpose: Load and decode the header and index in the metadata file * Try to load and decode the header: @@ -803,7 +790,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open) +H5FD__vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open) { H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */ unsigned load_retries = H5FD_VFD_SWMR_MD_LOAD_RETRY_MAX; /* Retries for loading header and index */ @@ -812,16 +799,16 @@ H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open) H5FD_vfd_swmr_md_index md_index; /* Metadata file index */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC do { HDmemset(&md_header, 0, sizeof(H5FD_vfd_swmr_md_header)); HDmemset(&md_index, 0, sizeof(H5FD_vfd_swmr_md_index)); /* Load and decode the header */ - if(H5FD_vfd_swmr_header_deserialize(_file, &md_header) >= 0) { + if(H5FD__vfd_swmr_header_deserialize(_file, &md_header) >= 0) { - /* Error if header + index fit does not within md_pages_reserved */ + /* Error if header + index does not fit within md_pages_reserved */ if((H5FD_MD_HEADER_SIZE + md_header.index_length) > (uint64_t)((hsize_t)file->md_pages_reserved * md_header.fs_page_size)) HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "header + index does not fit within md_pages_reserved") @@ -836,7 +823,7 @@ H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open) HDassert(md_header.tick_num > file->md_header.tick_num || open); /* Load and decode the index */ - if(H5FD_vfd_swmr_index_deserialize(_file, &md_index, &md_header) >= 0) { + if(H5FD__vfd_swmr_index_deserialize(_file, &md_index, &md_header) >= 0) { /* tick_num is the same in both header and index */ if(md_header.tick_num == md_index.tick_num) { @@ -848,6 +835,11 @@ H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open) else if(md_header.tick_num > (md_index.tick_num + 1)) HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "tick number mis-match in header and index") + if(md_index.entries) { + HDassert(md_index.num_entries); + md_index.entries = (H5FD_vfd_swmr_idx_entry_t *)H5FL_SEQ_FREE(H5FD_vfd_swmr_idx_entry_t, md_index.entries); + } + } /* end if index ok */ } /* end if header ok */ @@ -862,12 +854,19 @@ H5FD_vfd_swmr_load_hdr_and_idx(H5FD_t *_file, hbool_t open) HGOTO_ERROR(H5E_VFL, H5E_CANTLOAD, FAIL, "error in loading/decoding the metadata file header and index") done: + if(ret_value < 0) { + if(md_index.entries) { + HDassert(md_index.num_entries); + md_index.entries = (H5FD_vfd_swmr_idx_entry_t *)H5FL_SEQ_FREE(H5FD_vfd_swmr_idx_entry_t, md_index.entries); + } + } + FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_vfd_swmr_load_hdr_and_idx() */ +} /* H5FD__vfd_swmr_load_hdr_and_idx() */ /*------------------------------------------------------------------------- - * Function: H5FD_vfd_swmr_header_deserialize() + * Function: H5FD__vfd_swmr_header_deserialize() * * Purpose: To load and decode the header in the metadata file * --Retry to get a file with size at least the size of the header @@ -880,7 +879,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_header) +H5FD__vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_header) { H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */ struct stat stat_buf; /* Buffer for stat info */ @@ -893,7 +892,7 @@ H5FD_vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_head uint8_t *p = NULL; /* Pointer to buffer */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Try to stat the metadata file till md header size */ do { @@ -958,12 +957,12 @@ H5FD_vfd_swmr_header_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_header *md_head done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_vfd_swmr_header_deserialize() */ +} /* H5FD__vfd_swmr_header_deserialize() */ /*------------------------------------------------------------------------- - * Function: H5FD_vfd_swmr_index_deserialize() + * Function: H5FD__vfd_swmr_index_deserialize() * * Purpose: Load and decode the index in the metadata file * --Retry to get a file with size at least the size of the @@ -979,7 +978,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index, H5FD_vfd_swmr_md_header *md_header) +H5FD__vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index, H5FD_vfd_swmr_md_header *md_header) { H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */ uint8_t *image; /* Buffer */ @@ -993,7 +992,7 @@ H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index, unsigned index_retries = H5FD_VFD_SWMR_MD_INDEX_RETRY_MAX; /* Retries for loading the index */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Try to stat the metadata file till at least md (header+index) size */ do { @@ -1071,9 +1070,15 @@ H5FD_vfd_swmr_index_deserialize(H5FD_t *_file, H5FD_vfd_swmr_md_index *md_index, done: if(image) image = (uint8_t *)H5MM_xfree(image); + if(ret_value < 0) { + if(md_index->entries) { + HDassert(md_index->num_entries); + md_index->entries = (H5FD_vfd_swmr_idx_entry_t *)H5FL_SEQ_FREE(H5FD_vfd_swmr_idx_entry_t, md_index->entries); + } + } FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD_vfd_swmr_index_deserialize() */ +} /* H5FD__vfd_swmr_index_deserialize() */ /*------------------------------------------------------------------------- @@ -1081,7 +1086,7 @@ done: * * Purpose: Retrieve tick_num, num_entries and index from the metadata file * --If the parameter "reload_hdr_and_index" is true, load and decode - * the header and index via H5FD_vfd_swmr_load_hdr_and_idx(), which + * the header and index via H5FD__vfd_swmr_load_hdr_and_idx(), which * may replace the VFD's local copies of header and index with the * latest info read. * --Return tick_num, num_entries and index from the VFD's local copies. @@ -1102,7 +1107,7 @@ H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t reload_hdr_and_index, /* Load and decode the header and index as indicated */ if(reload_hdr_and_index) { - if(H5FD_vfd_swmr_load_hdr_and_idx(_file, FALSE) < 0) + if(H5FD__vfd_swmr_load_hdr_and_idx(_file, FALSE) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTLOAD, FAIL, "unable to load/decode md header and index") } @@ -1111,8 +1116,10 @@ H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t reload_hdr_and_index, *tick_ptr = file->md_header.tick_num; if(num_entries_ptr != NULL) { - if(*num_entries_ptr >= file->md_index.num_entries && index != NULL) + if(*num_entries_ptr >= file->md_index.num_entries && index != NULL) { + HDassert(*num_entries_ptr); HDmemcpy(index, file->md_index.entries, (file->md_index.num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t))); + } *num_entries_ptr = file->md_index.num_entries; } @@ -1120,3 +1127,47 @@ H5FD_vfd_swmr_get_tick_and_idx(H5FD_t *_file, hbool_t reload_hdr_and_index, done: FUNC_LEAVE_NOAPI(ret_value) } /* H5FD_vfd_swmr_get_tick_and_idx() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_is_vfd_swmr_driver() + * + * Purpose: Determine if the driver is a VFD SWMR driver + * + * Return: Success: TRUE/FALSE + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +hbool_t +H5FD_is_vfd_swmr_driver(H5FD_t *_file) +{ + H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +done: + FUNC_LEAVE_NOAPI(file->pub.driver_id == H5FD_VFD_SWMR) +} /* H5FD_is_vfd_swmr_driver() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_vfd_swmr_get_underlying_vfd() + * + * Purpose: Retrieve the underlying driver for the HDF5 file + * + * Return: Success: SUCCEED + * Failure: FAIL + * + *------------------------------------------------------------------------- + */ +H5FD_t * +H5FD_vfd_swmr_get_underlying_vfd(H5FD_t *_file) +{ + H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +done: + FUNC_LEAVE_NOAPI(file->hdf5_file_lf) +} /* H5FD_vfd_swmr_get_underlying_vfd() */ diff --git a/src/H5Fint.c b/src/H5Fint.c index 248d7a4..9a654d5 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -45,38 +45,39 @@ /****************/ /* VFD SWMR */ -/* Append entry to the delayed free spaced release linked list */ -#define H5F_DC_APPEND(entry_ptr, head_ptr, tail_ptr, len) \ -{ \ - if((head_ptr) == NULL) { \ - (head_ptr) = (entry_ptr); \ - (tail_ptr) = (entry_ptr); \ - } else { \ - (tail_ptr)->next = (entry_ptr); \ - (entry_ptr)->prev = (tail_ptr); \ - (tail_ptr) = (entry_ptr); \ - } \ - (len)++; \ -} /* H5F_DC_APPEND() */ + +/* Prepend entry to the delayed free spaced release linked list */ +#define H5F_DC_PREPEND(entry_ptr, head_ptr, tail_ptr, len) \ +{ \ + if((head_ptr) == NULL) { \ + (head_ptr) = (entry_ptr); \ + (tail_ptr) = (entry_ptr); \ + } else { \ + (head_ptr)->prev = (entry_ptr); \ + (entry_ptr)->next = (head_ptr); \ + (head_ptr) = (entry_ptr); \ + } \ + (len)++; \ +} /* H5F_DC_PREPEND() */ /* Remove entry from delayed free spaced release linked list */ -#define H5F_DC_REMOVE(entry_ptr, head_ptr, tail_ptr, len) \ -{ \ - if((head_ptr) == (entry_ptr)) { \ - (head_ptr) = (entry_ptr)->next; \ - if((head_ptr) != NULL ) \ - (head_ptr)->prev = NULL; \ - } else \ - (entry_ptr)->prev->next = (entry_ptr)->next; \ - if((tail_ptr) == (entry_ptr)) { \ - (tail_ptr) = (entry_ptr)->prev; \ - if((tail_ptr) != NULL) \ - (tail_ptr)->next = NULL; \ - } else \ - (entry_ptr)->next->prev = (entry_ptr)->prev; \ - entry_ptr->next = NULL; \ - entry_ptr->prev = NULL; \ - (len)--; \ +#define H5F_DC_REMOVE(entry_ptr, head_ptr, tail_ptr, len) \ +{ \ + if((head_ptr) == (entry_ptr)) { \ + (head_ptr) = (entry_ptr)->next; \ + if((head_ptr) != NULL ) \ + (head_ptr)->prev = NULL; \ + } else \ + (entry_ptr)->prev->next = (entry_ptr)->next; \ + if((tail_ptr) == (entry_ptr)) { \ + (tail_ptr) = (entry_ptr)->prev; \ + if((tail_ptr) != NULL) \ + (tail_ptr)->next = NULL; \ + } else \ + (entry_ptr)->next->prev = (entry_ptr)->prev; \ + entry_ptr->next = NULL; \ + entry_ptr->prev = NULL; \ + (len)--; \ } /* H5F_DC_REMOVE() */ /******************/ @@ -120,7 +121,7 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* VFD SWMR */ static herr_t H5F__vfd_swmr_init(H5F_t *f, hbool_t file_create); static herr_t H5F__vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing); -static herr_t H5F__vfd_swmr_update_end_of_tick(H5F_t *f); +static herr_t H5F__vfd_swmr_update_end_of_tick_and_tick_num(H5F_t *f, hbool_t incr_tick_num); static herr_t H5F__vfd_swmr_construct_write_md_hdr(H5F_t *f, uint32_t index_len); static herr_t H5F__vfd_swmr_construct_write_md_idx(H5F_t *f, uint32_t index_len, struct H5FD_vfd_swmr_idx_entry_t index[]); static herr_t H5F__idx_entry_cmp(const void *_entry1, const void *_entry2); @@ -1503,6 +1504,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) hbool_t ci_load = FALSE; /* whether MDC ci load requested */ hbool_t ci_write = FALSE; /* whether MDC CI write requested */ hbool_t file_create = FALSE; /* creating a new file or not */ + H5FD_t *underlying_lf = NULL; /* underlying file driver for VFD SWMR */ H5F_t *ret_value = NULL; /*actual return value */ FUNC_ENTER_NOAPI(NULL) @@ -1567,8 +1569,14 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) } /* end if */ } /* end if */ + /* For VFD SWMR driver, retrieve the underlying vfd for the search in H5F__sfile_search() */ + if(H5FD_is_vfd_swmr_driver(lf)) + underlying_lf = H5FD_vfd_swmr_get_underlying_vfd(lf); + else + underlying_lf = lf; + /* Is the file already open? */ - if((shared = H5F__sfile_search(lf)) != NULL) { + if((shared = H5F__sfile_search(underlying_lf)) != NULL) { /* * The file is already open, so use that one instead of the one we * just opened. We only one one H5FD_t* per file so one doesn't @@ -3652,7 +3660,7 @@ H5F__vfd_swmr_init(H5F_t *f, hbool_t file_create) } /* Update end_of_tick */ - if(H5F__vfd_swmr_update_end_of_tick(f) < 0) + if(H5F__vfd_swmr_update_end_of_tick_and_tick_num(f, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick") done: @@ -3806,11 +3814,10 @@ done: /*------------------------------------------------------------------------- - * Function: H5F__vfd_swmr_update_end_of_tick + * Function: H5F__vfd_swmr_update_end_of_tick_and_tick_num * - * Purpose: Set end_of_tick to the current time + tick length: - * --end_of_tick_g - * --f->shared->end_of_tick + * Purpose: Update end_of_tick (end_of_tick_g, f->shared->end_of_tick) + * Update tick_num (tick_num_g, f->shared->tick_num) * * Return: Success: SUCCEED * Failure: FAIL @@ -3818,27 +3825,55 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__vfd_swmr_update_end_of_tick(H5F_t *f) +H5F__vfd_swmr_update_end_of_tick_and_tick_num(H5F_t *f, hbool_t incr_tick_num) { - struct timespec tmp_end_of_tick; /* end_of_tick */ + struct timespec curr; /* Current time in struct timespec */ + struct timespec new_end_of_tick; /* new end_of_tick in struct timespec */ + long curr_nsecs; /* current time in nanoseconds */ + long tlen_nsecs; /* tick_len in nanoseconds */ + long end_nsecs; /* end_of_tick in nanoseconds */ + long new_end_nsecs; /* new end_of_tick in nanoseconds */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC - /* Get current time */ - if(HDclock_gettime(CLOCK_MONOTONIC, &tmp_end_of_tick) < 0) + /* Get current time in struct timespec */ + if(HDclock_gettime(CLOCK_MONOTONIC, &curr) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get time via clock_gettime") - /* Increment by tick length */ - tmp_end_of_tick.tv_nsec += f->shared->vfd_swmr_config.tick_len * 100000000; - tmp_end_of_tick.tv_sec += tmp_end_of_tick.tv_nsec / 1000000000; - tmp_end_of_tick.tv_nsec = tmp_end_of_tick.tv_nsec % 1000000000; - HDmemcpy(&end_of_tick_g, &tmp_end_of_tick, sizeof(struct timespec)); - HDmemcpy(&f->shared->end_of_tick, &tmp_end_of_tick, sizeof(struct timespec)); + /* Convert curr to nsecs */ + curr_nsecs = curr.tv_sec * SECOND_TO_NANOSECS + curr.tv_nsec; + + /* Convert tick_len to nanosecs */ + tlen_nsecs = f->shared->vfd_swmr_config.tick_len * TENTH_SEC_TO_NANOSECS; + + /* + * Update tick_num_g, f->shared->tick_num + */ + if(incr_tick_num) { + /* Convert end_of_tick_g to nanoseconds */ + end_nsecs = end_of_tick_g.tv_sec * SECOND_TO_NANOSECS + end_of_tick_g.tv_nsec; + + /* Increment tick_num by # of elapsed ticks */ + tick_num_g += (1+ (uint64_t)((curr_nsecs - end_nsecs) / tlen_nsecs)); + f->shared->tick_num = tick_num_g; + } + + /* + * Update end_of_tick_g, f->shared->end_of_tick + */ + /* Calculate new end_of_tick */ + new_end_nsecs = curr_nsecs + tlen_nsecs; + new_end_of_tick.tv_nsec = new_end_nsecs % SECOND_TO_NANOSECS; + new_end_of_tick.tv_sec = new_end_nsecs / SECOND_TO_NANOSECS; + + /* Update end_of_tick */ + HDmemcpy(&end_of_tick_g, &new_end_of_tick, sizeof(struct timespec)); + HDmemcpy(&f->shared->end_of_tick, &new_end_of_tick, sizeof(struct timespec)); done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5F__vfd_swmr_update_end_of_tick() */ +} /* H5F__vfd_swmr_update_end_of_tick_and_tick_num() */ /*------------------------------------------------------------------------- @@ -3865,6 +3900,7 @@ done: static herr_t H5F__vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing) { + H5F_vfd_swmr_dl_entry_t *curr, *next; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -3896,11 +3932,20 @@ H5F__vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing) if(H5MV_close(f) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close the free-space manager for the metadata file") + /* Free the delayed list */ + curr = f->shared->dl_head_ptr; + while(curr != NULL) { + next = curr->next; + curr = H5FL_FREE(H5F_vfd_swmr_dl_entry_t, curr); + curr = next; + } /* end while */ + f->shared->dl_head_ptr = f->shared->dl_tail_ptr = NULL; + vfd_swmr_file_g = NULL; } else { /* For file flush */ /* Update end_of_tick */ - if(H5F__vfd_swmr_update_end_of_tick(f) < 0) + if(H5F__vfd_swmr_update_end_of_tick_and_tick_num(f, TRUE) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick") } @@ -3946,18 +3991,18 @@ H5F__idx_entry_cmp(const void *_entry1, const void *_entry2) /*------------------------------------------------------------------------- * Function: H5F_update_vfd_swmr_metadata_file() * - * Purpose: Updating the metadata file - * --Sort index - * --For each non-null entry_ptr in the index: + * Purpose: Update the metadata file with the input index + * --Sort index + * --For each non-null entry_ptr in the index entries: * --Insert previous image of the entry onto the delayed list * --Allocate space for the entry in the metadata file * --Compute checksum - * --Update index + * --Update index entry * --Write the entry to the metadata file * --Set entry_ptr to NULL * --Construct on disk image of the index and write index to the metadata file * --Construct on disk image of the header and write header to the metadata file - * --Release time out space from the delayed list to the free-space manager + * --Release time out entries from the delayed list to the free-space manager * * Return: SUCCEED/FAIL * @@ -3966,26 +4011,26 @@ H5F__idx_entry_cmp(const void *_entry1, const void *_entry2) herr_t H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t index_len, struct H5FD_vfd_swmr_idx_entry_t index[]) { - H5F_vfd_swmr_dl_entry_t *prev; /* Points to the delayed list */ - H5F_vfd_swmr_dl_entry_t *dl_entry; - haddr_t md_addr; + H5F_vfd_swmr_dl_entry_t *prev; /* Points to the previous entry in the delayed list */ + H5F_vfd_swmr_dl_entry_t *dl_entry; /* Points to an entry in the delayed list */ + haddr_t md_addr; /* Address in the metadata file */ unsigned i; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* Sort index by increasing offset in the HDF5 file */ + /* Sort index entries by increasing offset in the HDF5 file */ HDqsort(index, index_len, sizeof(H5FD_vfd_swmr_idx_entry_t), H5F__idx_entry_cmp); /* For each non-null entry_ptr in the index: - * --If exists, insert previous image of the entry to the delayed list + * --Insert previous image of the entry (if exists) to the beginning of the delayed list * --Allocate space for the entry in the metadata file - * --Compute checksum, update index, write entry to the metadata file + * --Compute checksum, update the index entry, write entry to the metadata file * --Set entry_ptr to NULL */ for(i = 0; i < index_len; i++) { if(index[i].entry_ptr != NULL) { - /* Append previous image of the entry to the delayed list */ + /* Prepend previous image of the entry to the delayed list */ if(index[i].md_file_page_offset) { if(NULL == (dl_entry = H5FL_CALLOC(H5F_vfd_swmr_dl_entry_t))) HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "unable to allocate the delayed entry") @@ -3993,13 +4038,13 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t index_len, struct H5FD_vfd_ dl_entry->md_file_page_offset = index[i].md_file_page_offset; dl_entry->length = index[i].length; dl_entry->tick_num = f->shared->tick_num; - H5F_DC_APPEND(dl_entry, f->shared->dl_head_ptr, f->shared->dl_tail_ptr, f->shared->dl_len); + H5F_DC_PREPEND(dl_entry, f->shared->dl_head_ptr, f->shared->dl_tail_ptr, f->shared->dl_len); } /* Allocate space for the entry in the metadata file */ if((md_addr = H5MV_alloc(f, index[i].length)) == HADDR_UNDEF) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in allocating space from the metadata file") - /* Compute checksum and update the index */ + /* Compute checksum and update the index entry */ index[i].md_file_page_offset = md_addr/f->shared->fs_page_size; index[i].chksum = H5_checksum_metadata(index[i].entry_ptr, (size_t)(index[i].length), 0); @@ -4015,18 +4060,17 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t index_len, struct H5FD_vfd_ } /* end for */ - /* Construct and write index to the md file */ + /* Construct and write index to the metadata file */ if(H5F__vfd_swmr_construct_write_md_idx(f, index_len, index) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write index in md") + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write index to md") /* Construct and write header to the md file */ if(H5F__vfd_swmr_construct_write_md_hdr(f, index_len) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write header in md") + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write header to md") /* - * Release time out entry from the delayed list: - * Scan the delayed list from the bottom up: - * --release to the metadata file free space manager all entries that have + * Release time out entries from the delayed list by scanning the list from the bottom up: + * --release to the metadata file free space manager all index entries that have * resided on the list for more than max_lag ticks * --remove the associated entries from the list */ @@ -4044,8 +4088,8 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t index_len, struct H5FD_vfd_ /* Free the delayed entry struct */ H5FL_FREE(H5F_vfd_swmr_dl_entry_t, dl_entry); - } - /* DO I break from it once it is false ?? */ + } else + break; dl_entry = prev; } /* end while */ @@ -4065,14 +4109,14 @@ done: herr_t H5F_vfd_swmr_writer_end_of_tick(void) { - H5FD_vfd_swmr_idx_entry_t *index; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - /* construct */ if(vfd_swmr_file_g) { - ; + /* Update end_of_tick */ + if(H5F__vfd_swmr_update_end_of_tick_and_tick_num(vfd_swmr_file_g, TRUE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick") } done: @@ -4097,7 +4141,9 @@ H5F_vfd_swmr_reader_end_of_tick(void) /* construct */ if(vfd_swmr_file_g) { - ; + /* Update end_of_tick */ + if(H5F__vfd_swmr_update_end_of_tick_and_tick_num(vfd_swmr_file_g, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to update end of tick") } done: diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 2a5847e..a373285 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -386,8 +386,8 @@ struct H5F_file_t { uint64_t tick_num; /* Number of the current tick */ struct timespec end_of_tick; /* End time of the current tick */ - /* Metadata file */ - int vfd_swmr_md_fd; /* POSIX: file descriptor of the metadata file */ + /* Metadata file for VFD SWMR writer */ + int vfd_swmr_md_fd; /* POSIX: file descriptor for the metadata file */ haddr_t vfd_swmr_md_eoa; /* POSIX: eoa for the metadata file */ /* Free space manager for the metadata file */ @@ -530,6 +530,12 @@ H5_DLL herr_t H5F__get_sohm_mesg_count_test(hid_t fid, unsigned type_id, size_t H5_DLL herr_t H5F__check_cached_stab_test(hid_t file_id); H5_DLL herr_t H5F__get_maxaddr_test(hid_t file_id, haddr_t *maxaddr); H5_DLL herr_t H5F__get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr); + +/* VFD SWMR testing routines */ + +H5_DLL herr_t H5F__vfd_swmr_writer_md_test(hid_t file_id, hbool_t create); +H5_DLL herr_t H5F__vfd_swmr_writer_update_md_test(hid_t file_id, unsigned in_num_entries, + struct H5FD_vfd_swmr_idx_entry_t *in_index, unsigned num_insert_dl, unsigned num_remove_dl); #endif /* H5F_TESTING */ #endif /* _H5Fpkg_H */ diff --git a/src/H5Ftest.c b/src/H5Ftest.c index df9c933..d9ea9b6 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -38,12 +38,14 @@ /* Headers */ /***********/ #include "H5private.h" /* Generic Functions */ +#include "H5FDprivate.h" /* File Drivers */ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ #include "H5Gpkg.h" /* Groups */ #include "H5Iprivate.h" /* IDs */ #include "H5SMpkg.h" /* Shared object header messages */ +#include "H5MMprivate.h" /* Memory management */ /****************/ @@ -64,6 +66,11 @@ /********************/ /* Local Prototypes */ /********************/ +static herr_t H5F__vfd_swmr_decode_md_hdr(int md_fd, H5FD_vfd_swmr_md_header *md_hdr); +static herr_t H5F__vfd_swmr_decode_md_idx(int md_fd, H5FD_vfd_swmr_md_header *md_hdr, H5FD_vfd_swmr_md_index *md_idx); +static herr_t H5F__vfd_swmr_verify_md_hdr_and_idx(H5F_t *f, + H5FD_vfd_swmr_md_header *md_hdr, H5FD_vfd_swmr_md_index *md_idx, + unsigned num_entries, H5FD_vfd_swmr_idx_entry_t *index); /*********************/ @@ -74,6 +81,8 @@ /*****************************/ /* Library Private Variables */ /*****************************/ +/* Declare external the free list for H5FD_vfd_swmr_idx_entry_t */ +H5FL_SEQ_EXTERN(H5FD_vfd_swmr_idx_entry_t); /*******************/ @@ -232,3 +241,355 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__get_sbe_addr_test() */ + +/* + * VFD SWMR tests + */ + +/*------------------------------------------------------------------------- + * Function: H5F__vfd_swmr_writer_md_test + * + * Purpose: Verify info in the header and index when: + * (1) creating an HDF5 file + * (2) opening an existing HDF5 file + * (3) flushing an HDF5 file + * + * Open the metadata file + * Verify the file size is as expected (md_pages_reserved) + * For file create: + * --No header magic is found + * For file open or file flush: + * --Read and decode the header and index in the metadata file + * --Verify info in the header and index read from + * the metadata file is as expected (empty index) + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__vfd_swmr_writer_md_test(hid_t file_id, hbool_t file_create) +{ + H5F_t *f; /* File pointer */ + h5_stat_t stat_buf; /* Buffer for stat info */ + H5FD_vfd_swmr_md_header md_hdr; /* Header for the metadata file */ + H5FD_vfd_swmr_md_index md_idx; /* Indedx for the metadata file */ + int md_fd = -1; /* The metadata file descriptor */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments */ + if(NULL == (f = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + + /* Open the metadata file */ + if((md_fd = HDopen(f->shared->vfd_swmr_config.md_file_path, O_RDONLY)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "error opening metadata file") + + /* Verify the minimum size for the metadata file */ + if(HDstat(f->shared->vfd_swmr_config.md_file_path, &stat_buf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to stat the metadata file") + if(stat_buf.st_size < (HDoff_t)((hsize_t)f->shared->vfd_swmr_config.md_pages_reserved * f->shared->fs_page_size)) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect metadata file size") + + if(file_create) { /* Creating file */ + uint32_t hdr_magic; + + /* Seek to the beginning of the file */ + if(HDlseek(md_fd, (HDoff_t)0, SEEK_SET) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "error seeking metadata file") + + /* Try to read the magic for header */ + if(HDread(md_fd, &hdr_magic, H5_SIZEOF_MAGIC) < 0) + HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "error reading metadata file") + + /* Verify that there is no header magic in the metadata file */ + if(HDmemcmp(&hdr_magic, H5FD_MD_HEADER_MAGIC, (size_t)H5_SIZEOF_MAGIC) == 0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "error finding header magic in the metadata file") + + } else { /* Opening or flushing the file */ + + HDmemset(&md_hdr, 0, sizeof(H5FD_vfd_swmr_md_header)); + HDmemset(&md_idx, 0, sizeof(H5FD_vfd_swmr_md_index)); + + /* Decode the header */ + if(H5F__vfd_swmr_decode_md_hdr(md_fd, &md_hdr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "error decoding header in the metadata file") + + /* Decode the index */ + if(H5F__vfd_swmr_decode_md_idx(md_fd, &md_hdr, &md_idx) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "error decoding index in the metadata file") + + /* Verify info in header and index read from the metadata file */ + if(H5F__vfd_swmr_verify_md_hdr_and_idx(f, &md_hdr, &md_idx, 0, NULL) < 0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect info found in header and index of the metadata file") + } + +done: + /* Free the index entries */ + if(!file_create && md_idx.entries) { + HDassert(md_idx.num_entries); + H5MM_free(md_idx.entries); + } + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__vfd_swmr_writer_md_test() */ + + + +/*------------------------------------------------------------------------- + * Function: H5F__vfd_swmr_decode_md_hdr + * + * Purpose: Decode header and verify header magic + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__vfd_swmr_decode_md_hdr(int md_fd, H5FD_vfd_swmr_md_header *md_hdr) +{ + uint8_t image[H5FD_MD_HEADER_SIZE]; /* Buffer for the header image */ + uint8_t *p = NULL; /* Points to the image */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + p = image; + + /* Seek to the beginning of the file */ + if(HDlseek(md_fd, (HDoff_t)0, SEEK_SET) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "error seeking metadata file") + + /* Read the header */ + if(HDread(md_fd, image, H5FD_MD_HEADER_SIZE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "error reading metadata file") + + /* Verify magic for header */ + if(HDmemcmp(p, H5FD_MD_HEADER_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "does not find header magic in the metadata file") + + p += H5_SIZEOF_MAGIC; + + /* Deserialize fs_page_size, tick_num, index_offset, index_length */ + UINT32DECODE(p, md_hdr->fs_page_size); + UINT64DECODE(p, md_hdr->tick_num); + UINT64DECODE(p, md_hdr->index_offset); + UINT64DECODE(p, md_hdr->index_length); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__vfd_swmr_decode_md_hdr() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__vfd_swmr_decode_md_idx + * + * Purpose: Decode index and verify index magic + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__vfd_swmr_decode_md_idx(int md_fd, H5FD_vfd_swmr_md_header *md_hdr, H5FD_vfd_swmr_md_index *md_idx) +{ + uint8_t *image = NULL; /* Points to the buffer for the index image */ + uint8_t *p = NULL; /* Points to the image */ + unsigned i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Allocate buffer for the index image */ + if(NULL == (image = (uint8_t *)H5MM_malloc(md_hdr->index_length))) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "memory allocation failed for index on disk buffer") + + p = image; + + /* Seek to the position of the index */ + if(HDlseek(md_fd, (HDoff_t)md_hdr->index_offset, SEEK_SET) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "unable to seek in metadata file") + + /* Read the index */ + if(HDread(md_fd, image, md_hdr->index_length) < (int64_t)md_hdr->index_length) + HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "error in reading the header in metadata file") + + /* Verify magic for index */ + if(HDmemcmp(p, H5FD_MD_INDEX_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "no header magic in the metadata file") + + p += H5_SIZEOF_MAGIC; + + /* Deserialize tick_num and num_entries */ + UINT64DECODE(p, md_idx->tick_num); + UINT32DECODE(p, md_idx->num_entries); + + /* Deserialize index entries */ + if(md_idx->num_entries) { + + /* Allocate memory for the index entries */ + if(NULL == (md_idx->entries = (H5FD_vfd_swmr_idx_entry_t *)H5MM_calloc(md_idx->num_entries * sizeof(H5FD_vfd_swmr_idx_entry_t)))) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "memory allocation failed for index entries") + + /* Decode index entries */ + for(i = 0; i < md_idx->num_entries; i++) { + UINT32DECODE(p, md_idx->entries[i].hdf5_page_offset); + UINT32DECODE(p, md_idx->entries[i].md_file_page_offset); + UINT32DECODE(p, md_idx->entries[i].length); + UINT32DECODE(p, md_idx->entries[i].chksum); + } /* end for */ + + } /* end if */ + +done: + /* Free the buffer */ + if(image) + H5MM_free(image); + if(ret_value < 0) { + /* Free the index entries */ + if(md_idx->entries) { + HDassert(md_idx->num_entries); + H5MM_free(md_idx->entries); + } + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__vfd_swmr_decode_md_idx() */ + + + +/*------------------------------------------------------------------------- + * Function: H5F__vfd_swmr_verify_md_hdr_idx_test + * + * Purpose: Verify the header and index in the metadata file: + * --fs_page_size in md header is the same as that stored in "f" + * --index_length in md header is as indicated by num_entries + * --index_offset in md header is right after the header + * --number of entries in md index is num_entries + * --entries in md index is as indicated by num_entries and index + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__vfd_swmr_verify_md_hdr_and_idx(H5F_t *f, + H5FD_vfd_swmr_md_header *md_hdr, H5FD_vfd_swmr_md_index *md_idx, + unsigned num_entries, H5FD_vfd_swmr_idx_entry_t *index) +{ + unsigned i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Verify fs_page_size read from header in the metadata file is fs_page_size in f */ + if(md_hdr->fs_page_size != f->shared->fs_page_size) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect fs_page_size read from metadata file") + + /* Verify index_length read from header in the metadata file is the size of num_entries index */ + if(md_hdr->index_length != H5FD_MD_INDEX_SIZE(num_entries)) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect index_length read from metadata file") + + /* Verify index_offset read from header in the metadata file is the size of md header */ + if(md_hdr->index_offset != H5FD_MD_HEADER_SIZE) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect index_offset read from metadata file") + + /* Verify num_entries read from index in the metadata file is num_entries */ + if(md_idx->num_entries != num_entries) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect num_entries read from metadata file") + + /* Verify empty/non-empty index entries */ + if(num_entries == 0) { + /* Verify the index is empty */ + if(md_idx->entries != NULL) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect entries in index") + } else { + /* Verify entries */ + for(i = 0; i < num_entries; i++) { + if(md_idx->entries[i].length != index[i].length) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect length read from metadata file") + + if(md_idx->entries[i].hdf5_page_offset != index[i].hdf5_page_offset) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect md_file_page_offset read from metadata file") + + if(md_idx->entries[i].md_file_page_offset != index[i].md_file_page_offset) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect md_file_page_offset read from metadata file") + + if(md_idx->entries[i].chksum != index[i].chksum) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect chksum read from metadata file") + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__vfd_swmr_verify_md_hdr_and_idx() */ + + + +/*------------------------------------------------------------------------- + * Function: H5F__vfd_swmr_update_md_test + * + * Purpose: Update the metadata file with the input index + * Verify the info read from the metadata file is as indicated by + * the input: num_entries and index + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__vfd_swmr_writer_update_md_test(hid_t file_id, unsigned num_entries, H5FD_vfd_swmr_idx_entry_t *index, + unsigned num_insert_dl, unsigned num_remove_dl) +{ + H5F_t *f; /* File pointer */ + int md_fd = -1; /* The metadata file descriptor */ + H5FD_vfd_swmr_md_header md_hdr; /* Header for the metadata file */ + H5FD_vfd_swmr_md_index md_idx; /* Indedx for the metadata file */ + unsigned save_dl_len = 0; /* # of entries in the delayed list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + HDmemset(&md_hdr, 0, sizeof(H5FD_vfd_swmr_md_header)); + HDmemset(&md_idx, 0, sizeof(H5FD_vfd_swmr_md_index)); + + /* Check arguments */ + if(NULL == (f = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + + /* Save the number of entries in the delayed list */ + save_dl_len = f->shared->dl_len; + + /* Update the metadata file with the input index */ + if(H5F_update_vfd_swmr_metadata_file(f, num_entries, index) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, "error updating the md file with the index") + + /* Verify the number of entries in the delayed list is as expected */ + if(f->shared->dl_len != (save_dl_len + num_insert_dl - num_remove_dl)) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect # of entries in the delayed list") + + /* Open the metadata file */ + if((md_fd = HDopen(f->shared->vfd_swmr_config.md_file_path, O_RDONLY)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "error opening metadata file") + + /* Decode the header in the metadata file */ + if(H5F__vfd_swmr_decode_md_hdr(md_fd, &md_hdr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "error decoding header in the metadata file") + + /* Decode the index in the metadata file */ + if(H5F__vfd_swmr_decode_md_idx(md_fd, &md_hdr, &md_idx) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "error decoding index in the metadata file") + + /* Verify info read from the metadata file is the same as the input index */ + if(H5F__vfd_swmr_verify_md_hdr_and_idx(f, &md_hdr, &md_idx, num_entries, index) < 0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect info found in header and index of the metadata file") + +done: + /* Free index entries */ + if(md_idx.entries) { + HDassert(md_idx.num_entries); + H5MM_free(md_idx.entries); + } + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__vfd_swmr_writer_update_md_test() */ diff --git a/src/H5private.h b/src/H5private.h index 842ba87..fb01d06 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -374,6 +374,10 @@ /* Raise an integer to a power of 2 */ # define H5_EXP2(n) (1 << (n)) +/* VFD SWMR */ +#define SECOND_TO_NANOSECS 1000000000 /* Second to nanoseconds */ +#define TENTH_SEC_TO_NANOSECS 100000000 /* Tenth of a second to nanoseconds */ + /* * HDF Boolean type. */ @@ -2070,16 +2074,18 @@ H5_DLL herr_t H5CX_pop(void); /* Initialize the library */ \ if(vfd_swmr_g) { \ struct timespec curr_time; \ + long curr_nsecs, end_nsecs; \ if(HDclock_gettime(CLOCK_MONOTONIC, &curr_time) < 0) \ HGOTO_ERROR(H5E_FUNC, H5E_CANTGET, err, "can't get time via clock_gettime") \ - if( (curr_time.tv_sec >= end_of_tick_g.tv_sec) && \ - (curr_time.tv_nsec >= end_of_tick_g.tv_nsec) ) { \ + curr_nsecs = curr_time.tv_sec * 1000000000 + curr_time.tv_nsec; \ + end_nsecs = end_of_tick_g.tv_sec * 1000000000 + end_of_tick_g.tv_nsec; \ + if(curr_nsecs > end_nsecs) { \ if(vfd_swmr_writer_g) { \ - if(H5F_vfd_swmr_writer_end_of_tick() < 0) \ + if(H5F_vfd_swmr_writer_end_of_tick() < 0) \ HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, err, "end of tick error for VFD SWMR writer") \ } \ else if(!swmr_reader_exit) { \ - if(H5F_vfd_swmr_reader_end_of_tick() < 0) \ + if(H5F_vfd_swmr_reader_end_of_tick() < 0) \ HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, err, "end of tick error for VFD SWMR reader") \ } \ } \ diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c index 4d1695a..0cf865a 100644 --- a/test/vfd_swmr.c +++ b/test/vfd_swmr.c @@ -24,38 +24,24 @@ /* * This file needs to access private information from the H5F package. */ -#define H5MF_FRIEND /*suppress error about including H5MFpkg */ -#include "H5MFpkg.h" -#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ #define H5F_TESTING +#include "H5FDprivate.h" #include "H5Fpkg.h" -#include "H5CXprivate.h" /* API Contexts */ -#include "H5Iprivate.h" -#include "H5PBprivate.h" +#include "H5CXprivate.h" /* API Contexts */ - -#define FILENAME_LEN 1024 -#define NUM_DSETS 5 -#define NX 100 -#define NY 50 - -static unsigned open_file(char *filename, hid_t fapl, hsize_t page_size, size_t page_buffer_size); +#define FS_PAGE_SIZE 512 +#define FILENAME "vfd_swmr_file.h5" +#define MD_FILENAME "vfd_swmr_metadata_file" /* test routines for VFD SWMR */ -static unsigned test_fapl(); -static unsigned test_file_end_tick(); -static unsigned test_file_fapl(); - -const char *FILENAME[] = { - "filepaged", - NULL -}; - - -/*********************/ -/*********************/ +static unsigned test_fapl(void); +static unsigned test_file_end_tick(void); +static unsigned test_file_fapl(void); +static unsigned test_writer_md(void); +static unsigned test_writer_update_md(void); /*------------------------------------------------------------------------- @@ -78,7 +64,7 @@ const char *FILENAME[] = { *------------------------------------------------------------------------- */ static unsigned -test_fapl() +test_fapl(void) { hid_t fapl = -1; /* File access property list */ H5F_vfd_swmr_config_t *my_config = NULL; /* Configuration for VFD SWMR */ @@ -149,7 +135,7 @@ test_fapl() TEST_ERROR; /* Set md_file_path */ - HDstrcpy(my_config->md_file_path, "my_md_file"); + HDstrcpy(my_config->md_file_path, MD_FILENAME); my_config->vfd_swmr_writer = TRUE; /* Should succeed in setting the configuration info */ @@ -168,7 +154,7 @@ test_fapl() TEST_ERROR; if(my_config->md_pages_reserved != 2) TEST_ERROR; - if(HDstrcmp(my_config->md_file_path, "my_md_file") != 0) + if(HDstrcmp(my_config->md_file_path, MD_FILENAME) != 0) TEST_ERROR; /* Close the file access property list */ @@ -200,6 +186,13 @@ error: * B) Verify the VFD SWMR configuration set in fapl * used to create/open the file is the same as the * configuration retrieved from the file's fapl. + * C) Verify the following when configured as VFD SWMR reader: + * (1) there is an existing file opened as writer: + * --same process open as reader: will just increment the + * file reference count and use the same shared struct + * (2) there is no existing file opened as writer: + * --opening the file as reader will fail + * because there is no metadata file * * Return: 0 if test is sucessful * 1 if test fails @@ -209,25 +202,22 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_file_fapl() +test_file_fapl(void) { hid_t fid = -1; /* File ID */ + hid_t fid_read = -1; /* File ID */ hid_t fcpl = -1; /* File creation property list ID */ hid_t fapl1 = -1; /* File access property list ID */ hid_t fapl2 = -1; /* File access property list ID */ hid_t file_fapl = -1; /* File access property list ID associated with the file */ - H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ - H5F_vfd_swmr_config_t *config2 = NULL; /* Configuration for VFD SWMR */ - H5F_vfd_swmr_config_t *config3 = NULL; /* Configuration for VFD SWMR */ - herr_t ret; /* Return value */ -hid_t fid_read = -1; /* File ID for VFD SWMR reader */ -hid_t sid = -1; -hid_t did = -1; + H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *config2 = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *file_config = NULL; /* Configuration for VFD SWMR */ TESTING("VFD SWMR configuration for the file and fapl"); /* Should succeed without VFD SWMR configured */ - if((fid = H5Fcreate("myfile", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; /* Close the file */ @@ -239,35 +229,36 @@ hid_t did = -1; FAIL_STACK_ERROR; if((config2 = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) FAIL_STACK_ERROR; - if((config3 = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + if((file_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) FAIL_STACK_ERROR; HDmemset(config1, 0, sizeof(H5F_vfd_swmr_config_t)); HDmemset(config2, 0, sizeof(H5F_vfd_swmr_config_t)); - HDmemset(config3, 0, sizeof(H5F_vfd_swmr_config_t)); + HDmemset(file_config, 0, sizeof(H5F_vfd_swmr_config_t)); /* Create a copy of the file access property list */ if((fapl1 = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR; + /* Configured as VFD SWMR writer */ config1->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; config1->tick_len = 4; config1->max_lag = 6; config1->vfd_swmr_writer = TRUE; config1->md_pages_reserved = 2; - HDstrcpy(config1->md_file_path, "my_md_file"); + HDstrcpy(config1->md_file_path, MD_FILENAME); /* Should succeed in setting the VFD SWMR configuration */ if(H5Pset_vfd_swmr_config(fapl1, config1) < 0) TEST_ERROR; - /* Should fail to configure VFD SWMR: page buffering and paged aggregation not enabled */ + /* Should fail to create: page buffering and paged aggregation not enabled */ H5E_BEGIN_TRY { - fid = H5Fcreate("myfile", H5F_ACC_TRUNC, H5P_DEFAULT, fapl1); + fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, fapl1); } H5E_END_TRY; if(fid >= 0) TEST_ERROR; - /* Create a copy of file creation property list */ + /* Create a copy of the file creation property list */ if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) FAIL_STACK_ERROR @@ -275,9 +266,9 @@ hid_t did = -1; if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) FAIL_STACK_ERROR; - /* Should fail to configure VFD SWMR: no page buffering */ + /* Should fail to create: no page buffering */ H5E_BEGIN_TRY { - fid = H5Fcreate("myfile", H5F_ACC_TRUNC, fcpl, fapl1); + fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1); } H5E_END_TRY; if(fid >= 0) TEST_ERROR; @@ -286,8 +277,8 @@ hid_t did = -1; if(H5Pset_page_buffer_size(fapl1, 4096, 0, 0) < 0) FAIL_STACK_ERROR; - /* Should succeed to configure VFD SWMR: paged aggregation and page buffering enabled */ - if((fid = H5Fcreate("myfile", H5F_ACC_TRUNC, fcpl, fapl1)) < 0) + /* Should succeed to create the file: paged aggregation and page buffering enabled */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) TEST_ERROR; /* Get the file's file access property list */ @@ -295,11 +286,11 @@ hid_t did = -1; FAIL_STACK_ERROR; /* Retrieve the VFD SWMR configuration from file_fapl */ - if(H5Pget_vfd_swmr_config(file_fapl, config2) < 0) + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) TEST_ERROR; /* Verify the retrieved info is the same as config1 */ - if(HDmemcmp(config1, config2, sizeof(H5F_vfd_swmr_config_t)) != 0) + if(HDmemcmp(config1, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) TEST_ERROR; /* Closing */ @@ -309,23 +300,23 @@ hid_t did = -1; FAIL_STACK_ERROR; - /* Should succeed: VFD SWMR writer */ - if((fid = H5Fopen("myfile", H5F_ACC_RDWR, fapl1)) < 0) + /* Should succeed to open the file as VFD SWMR writer */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) TEST_ERROR; /* Get the file's file access property list */ if((file_fapl = H5Fget_access_plist(fid)) < 0) FAIL_STACK_ERROR; - /* Clear info in config2 */ - HDmemset(config2, 0, sizeof(H5F_vfd_swmr_config_t)); + /* Clear info in file_config */ + HDmemset(file_config, 0, sizeof(H5F_vfd_swmr_config_t)); /* Retrieve the VFD SWMR configuration from file_fapl */ - if(H5Pget_vfd_swmr_config(file_fapl, config2) < 0) + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) TEST_ERROR; /* Verify the retrieved info is the same as config1 */ - if(HDmemcmp(config1, config2, sizeof(H5F_vfd_swmr_config_t)) != 0) + if(HDmemcmp(config1, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) TEST_ERROR; /* Closing */ @@ -334,97 +325,130 @@ hid_t did = -1; if(H5Pclose(file_fapl) < 0) FAIL_STACK_ERROR; - /* Set a different VFD SWMR configuration */ /* Create a copy of the file access property list */ if((fapl2 = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR; - config3->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; - config3->tick_len = 4; - config3->max_lag = 10; - config3->vfd_swmr_writer = TRUE; - config3->md_pages_reserved = 2; - HDstrcpy(config3->md_file_path, "my_md_file"); + /* Set up different VFD SWMR configuration */ + config2->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config2->tick_len = 4; + config2->max_lag = 10; + config2->vfd_swmr_writer = TRUE; + config2->md_pages_reserved = 2; + HDstrcpy(config2->md_file_path, MD_FILENAME); /* Should succeed in setting the VFD SWMR configuration */ - if(H5Pset_vfd_swmr_config(fapl2, config3) < 0) + if(H5Pset_vfd_swmr_config(fapl2, config2) < 0) TEST_ERROR; /* Enable page buffering */ if(H5Pset_page_buffer_size(fapl2, 4096, 0, 0) < 0) FAIL_STACK_ERROR; - /* Should succeed: VFD SWMR writer */ - if((fid = H5Fopen("myfile", H5F_ACC_RDWR, fapl2)) < 0) + /* Should succeed to open the file as VFD SWMR writer */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl2)) < 0) TEST_ERROR; /* Get the file's file access property list */ if((file_fapl = H5Fget_access_plist(fid)) < 0) FAIL_STACK_ERROR; - /* Clear info in config2 */ - HDmemset(config2, 0, sizeof(H5F_vfd_swmr_config_t)); + /* Clear info in file_config */ + HDmemset(file_config, 0, sizeof(H5F_vfd_swmr_config_t)); /* Retrieve the VFD SWMR configuration from file_fapl */ - if(H5Pget_vfd_swmr_config(file_fapl, config2) < 0) + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) TEST_ERROR; /* Verify the retrieved info is NOT the same as config1 */ - if(HDmemcmp(config1, config2, sizeof(H5F_vfd_swmr_config_t)) == 0) + if(HDmemcmp(config1, file_config, sizeof(H5F_vfd_swmr_config_t)) == 0) TEST_ERROR; - /* Verify the retrieved info is the same as config3 */ - if(HDmemcmp(config2, config3, sizeof(H5F_vfd_swmr_config_t)) != 0) + /* Verify the retrieved info is the same as config2 */ + if(HDmemcmp(config2, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) TEST_ERROR; - /* Closing */ - if(H5Fclose(fid) < 0) - FAIL_STACK_ERROR; - if(H5Pclose(file_fapl) < 0) - FAIL_STACK_ERROR; - - /* - * VDF SWMR READER + /* + * The file previously opened as writer is not closed. */ /* Create a copy of the file access property list */ if((fapl2 = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR; - config3->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; - config3->tick_len = 4; - config3->max_lag = 10; - config3->vfd_swmr_writer = FALSE; - config3->md_pages_reserved = 2; - HDstrcpy(config3->md_file_path, "my_md_file"); + /* Set up as VFD SWMR reader */ + config2->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + config2->tick_len = 4; + config2->max_lag = 10; + config2->vfd_swmr_writer = FALSE; + config2->md_pages_reserved = 2; + HDstrcpy(config2->md_file_path, MD_FILENAME); - /* Should succeed in setting the VFD SWMR configuration */ - if(H5Pset_vfd_swmr_config(fapl2, config3) < 0) + /* Should succeed in setting the VFD SWMR configuration in fapl2 */ + if(H5Pset_vfd_swmr_config(fapl2, config2) < 0) TEST_ERROR; /* Enable page buffering */ if(H5Pset_page_buffer_size(fapl2, 4096, 0, 0) < 0) FAIL_STACK_ERROR; + /* Should succeed in opening the file */ + /* Same process open: even though opened with reader configuration, + * it just increments the file reference count and uses the writer's + * shared file struct */ + if((fid_read = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl2)) < 0) + TEST_ERROR; + + /* Clear info in file_config */ + HDmemset(file_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Get the file's file access property list */ + if((file_fapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Retrieve the VFD SWMR configuration from file_fapl */ + if(H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) + TEST_ERROR; + + /* Verify that the retrieved config is a writer */ + if(file_config->vfd_swmr_writer == FALSE) + TEST_ERROR; + /* Verify that the retrieved config is not the same as the initial configuration */ + if(file_config->vfd_swmr_writer == config2->vfd_swmr_writer) + TEST_ERROR; + + /* Closing */ + if(H5Fclose(fid_read) < 0) + FAIL_STACK_ERROR; + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(file_fapl) < 0) + FAIL_STACK_ERROR; - /* Open the file for reading only */ + /* + * The file opened as writer is closed. + */ + /* Should fail to open the file as VFD SWMR reader: no metadata file */ H5E_BEGIN_TRY { - fid_read = H5Fopen("myfile", H5F_ACC_RDONLY, fapl2); + fid_read = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl2); } H5E_END_TRY; if(fid_read >= 0) TEST_ERROR; + /* Closing */ if(H5Pclose(fapl1) < 0) FAIL_STACK_ERROR; if(H5Pclose(fapl2) < 0) FAIL_STACK_ERROR; if(H5Pclose(fcpl) < 0) FAIL_STACK_ERROR; + + /* Free buffers */ if(config1) HDfree(config1); if(config2) HDfree(config2); - if(config3) - HDfree(config3); + if(file_config) + HDfree(file_config); PASSED() return 0; @@ -440,8 +464,8 @@ error: HDfree(config1); if(config2) HDfree(config2); - if(config3) - HDfree(config3); + if(file_config) + HDfree(file_config); return 1; } /* test_file_fapl() */ @@ -463,7 +487,7 @@ error: *------------------------------------------------------------------------- */ static unsigned -test_file_end_tick() +test_file_end_tick(void) { hid_t fid = -1; /* File ID */ hid_t fapl = -1; /* File access property list */ @@ -474,7 +498,7 @@ test_file_end_tick() TESTING("H5Fvfd_swmr_end_tick() for VFD SWMR"); /* Should succeed without VFD SWMR configured */ - if((fid = H5Fcreate("myfile", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; /* Should fail */ @@ -503,7 +527,7 @@ test_file_end_tick() my_config->max_lag = 3; my_config->vfd_swmr_writer = TRUE; my_config->md_pages_reserved = 2; - HDstrcpy(my_config->md_file_path, "my_md_file"); + HDstrcpy(my_config->md_file_path, MD_FILENAME); /* Should succeed in setting the VFD SWMR configuration */ if(H5Pset_vfd_swmr_config(fapl, my_config) < 0) @@ -522,7 +546,7 @@ test_file_end_tick() FAIL_STACK_ERROR; /* Create the file with VFD SWMR configured */ - if((fid = H5Fcreate("myfile", H5F_ACC_TRUNC, fcpl, fapl)) < 0) + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR; /* Should succeed */ @@ -534,7 +558,7 @@ test_file_end_tick() FAIL_STACK_ERROR; /* Open the file as VFD SWMR writer */ - if((fid = H5Fopen("myfile", H5F_ACC_RDWR, fapl)) < 0) + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; /* Should succeed */ @@ -545,72 +569,431 @@ test_file_end_tick() if(H5Fclose(fid) < 0) FAIL_STACK_ERROR; -#ifdef NOTYET - /* Open the file as VFD SWMR reader */ - if((fid = H5Fopen("myfile", H5F_ACC_RDONLY, fapl)) < 0) + /* Open the file as reader without VFD SWMR configured */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR; - /* Should succeed */ - if(H5Fvfd_swmr_end_tick(fid) < 0) + /* Should fail */ + H5E_BEGIN_TRY { + ret = H5Fvfd_swmr_end_tick(fid); + } H5E_END_TRY; + if(ret >= 0) TEST_ERROR; /* Close the file */ if(H5Fclose(fid) < 0) FAIL_STACK_ERROR; -#endif - /* Open the file as writer without VFD SWMR configured */ - if((fid = H5Fopen("myfile", H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + if(H5Pclose(fapl) < 0) FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + if(my_config) + HDfree(my_config); - /* Should fail */ + PASSED() + return 0; + +error: H5E_BEGIN_TRY { - ret = H5Fvfd_swmr_end_tick(fid); + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fid); } H5E_END_TRY; - if(ret >= 0) - TEST_ERROR; + + if(my_config) + HDfree(my_config); + + return 1; +} /* test_file_end_tick() */ + + +/*------------------------------------------------------------------------- + * Function: test_writer_md() + * + * Purpose: Verify info in the metadata file when: + * --creating the HDF5 file + * --flushing the HDF5 file + * --opening an existing HDF5 file + * It will call the internal testing routine + * H5F__vfd_swmr_writer_md_test() to do the following: + * --Open the metadata file + * --Verify the file size is as expected (md_pages_reserved) + * --For file create: + * --No header magic is found + * --For file open or file flush: + * --Read and decode the header and index in the metadata file + * --Verify info in the header and index read from + * the metadata file is as expected (empty index) + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2018 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_writer_md(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + hid_t fcpl = -1; /* File creation property list */ + H5F_vfd_swmr_config_t *my_config = NULL; /* Configuration for VFD SWMR */ + + TESTING("Create/Open/Flush an HDF5 file for VFD SWMR"); + + /* Allocate memory for the configuration structure */ + if((my_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + HDmemset(my_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Create a copy of the file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set up the VFD SWMR configuration */ + my_config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + my_config->tick_len = 1; + my_config->max_lag = 3; + my_config->vfd_swmr_writer = TRUE; + my_config->md_pages_reserved = 1; + HDstrcpy(my_config->md_file_path, MD_FILENAME); + + /* Set the VFD SWMR configuration in fapl */ + if(H5Pset_vfd_swmr_config(fapl, my_config) < 0) + FAIL_STACK_ERROR; + + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, 4096, 0, 0) < 0) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + + /* Create an HDF5 file with VFD SWMR configured */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Verify info in metadata file when creating the HDF5 file */ + if(H5F__vfd_swmr_writer_md_test(fid, TRUE) < 0) + TEST_ERROR + + /* Flush the HDF5 file */ + if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) + FAIL_STACK_ERROR + + /* Verify info in metadata file when flushing the HDF5 file */ + if(H5F__vfd_swmr_writer_md_test(fid, FALSE) < 0) + TEST_ERROR /* Close the file */ if(H5Fclose(fid) < 0) FAIL_STACK_ERROR; - /* Open the file as reader without VFD SWMR configured */ - if((fid = H5Fopen("myfile", H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + /* Re-open the file as VFD SWMR writer */ + if((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR; + + /* Verify info in metadata file when reopening the HDF5 file */ + if(H5F__vfd_swmr_writer_md_test(fid, FALSE) < 0) + TEST_ERROR + + /* Closing */ + if(H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fcpl) < 0) FAIL_STACK_ERROR; - /* Should fail */ + if(my_config) + HDfree(my_config); + + PASSED() + return 0; + +error: + if(my_config) + HDfree(my_config); + H5E_BEGIN_TRY { - ret = H5Fvfd_swmr_end_tick(fid); + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fid); } H5E_END_TRY; - if(ret >= 0) - TEST_ERROR; + + return 1; +} /* test_writer_md() */ + + +/*------------------------------------------------------------------------- + * Function: test_writer_update_md() + * + * Purpose: Verify info in the metadata file after update with the + * constructed index: (A), (B), (C), (D) + * It will call the internal testing routine + * H5F__vfd_swmr_writer_update_md_test() to do the following: + * --Update the metadata file with the input index via the + * internal library routine H5F_update_vfd_swmr_metadata_file() + * --Verify the entries in the delayed list is as expected + * (input: num_insert_dl, num_remove_dl) + * --Open the metadata file, read and decode the header and index + * --Verify info in the header and index just read from the + * metadatea file is as expected (input: num_entries and index) + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2018 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_writer_update_md(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fid_read = -1; /* File ID for the reader */ + hid_t fapl = -1; /* File access property list */ + hid_t fapl2 = -1; /* File access property list */ + hid_t fcpl = -1; /* File creation property list */ + unsigned num_entries = 10; /* Number of entries in the index */ + unsigned i = 0; /* Local index variables */ + uint8_t *buf = NULL; /* Data page from the page buffer */ + hid_t dcpl = -1; /* Dataset creation property list */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1; /* Dataset ID */ + int *rwbuf = NULL; /* Data buffer for writing */ + H5O_info_t oinfo; /* Object metadata information */ + char dname[50]; /* Name of dataset */ + hsize_t dims[2] = {50, 20}; /* Dataset dimension sizes */ + hsize_t max_dims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* Dataset maximum dimension sizes */ + hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ + H5FD_vfd_swmr_idx_entry_t *index = NULL; /* Pointer to the index entries */ + H5F_vfd_swmr_config_t *my_config = NULL; /* Configuration for VFD SWMR */ + + TESTING("Updating the metadata file for VFD SWMR writer"); + + /* Allocate memory for the configuration structure */ + if((my_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + HDmemset(my_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Create a copy of the file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Set up the VFD SWMR configuration */ + my_config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; + my_config->tick_len = 1; + my_config->max_lag = 3; + my_config->vfd_swmr_writer = TRUE; + my_config->md_pages_reserved = 2; + HDstrcpy(my_config->md_file_path, MD_FILENAME); + + /* Set the VFD SWMR configuration in fapl */ + if(H5Pset_vfd_swmr_config(fapl, my_config) < 0) + FAIL_STACK_ERROR; + + /* Enable page buffering */ + if(H5Pset_page_buffer_size(fapl, FS_PAGE_SIZE, 0, 0) < 0) + FAIL_STACK_ERROR; + + /* Create a copy of the file creation property list */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set file space strategy */ + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + if(H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + FAIL_STACK_ERROR; + + /* Create an HDF5 file with VFD SWMR configured */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Verify info in the metadata file when creating an HDF5 file */ + if(H5F__vfd_swmr_writer_md_test(fid, TRUE) < 0) + TEST_ERROR + + /* Allocate num_entries for the data buffer */ + if((buf = (uint8_t *)HDmalloc((num_entries * FS_PAGE_SIZE * sizeof(uint8_t)))) == NULL) + FAIL_STACK_ERROR; + + /* Allocate memory for num_entries index */ + if(NULL == (index = (H5FD_vfd_swmr_idx_entry_t *)HDcalloc(num_entries, sizeof(H5FD_vfd_swmr_idx_entry_t)))) + FAIL_STACK_ERROR; + + /* (A) Construct index for updating the metadata file */ + for(i = 0; i < num_entries; i++) { + index[i].hdf5_page_offset = (uint64_t)my_config->md_pages_reserved; + index[i].md_file_page_offset = 0; + index[i].length = (uint32_t)FS_PAGE_SIZE; + index[i].entry_ptr = (void *)&buf[i]; + } + + /* Update with index and verify info in the metadata file */ + /* Also verify that 0/0 entries are inserted/removed to/from the delayed list */ + if(H5F__vfd_swmr_writer_update_md_test(fid, num_entries, index, 0, 0) < 0) + TEST_ERROR + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + + /* Set to use chunked dataset */ + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + FAIL_STACK_ERROR + + /* Create dataspace */ + if((sid = H5Screate_simple(2, dims, max_dims)) < 0) + FAIL_STACK_ERROR + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < 500; i++) { + + /* Create a chunked dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dcreate2(fid, dname, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Get dataset object header address */ + if(H5Oget_info2(did, &oinfo, H5O_INFO_BASIC) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* (B) Update every other entry in the index */ + for(i = 0; i < num_entries; i+= 2) + index[i].entry_ptr = (void *)&buf[i]; + + /* Update with index and verify info in the metadata file */ + /* Also verify that 5/0 entries are inserted/removed to/from the delayed list */ + if(H5F__vfd_swmr_writer_update_md_test(fid, num_entries, index, 5, 0) < 0) + TEST_ERROR + + /* Allocate memory for the read/write buffer */ + if((rwbuf = (int *)HDmalloc(sizeof(int) * (50 * 20))) == NULL) + FAIL_STACK_ERROR; + for(i = 0; i < (50 * 20); i++) + rwbuf[i] = (int)i; + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < 500; i++) { + /* Open the dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dopen2(fid, dname, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Write to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) + FAIL_STACK_ERROR + + /* Get dataset object info */ + if(H5Oget_info2(did, &oinfo, H5O_INFO_BASIC) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* (C) Update every 3 entry in the index */ + for(i = 0; i < num_entries; i+= 3) + index[i].entry_ptr = (void *)&buf[i]; + + /* Update with index and verify info in the metadata file */ + /* Also verify that 4/5 entries are inserted/removed to/from the delayed list */ + if(H5F__vfd_swmr_writer_update_md_test(fid, num_entries, index, 4, 5) < 0) + TEST_ERROR + + /* Clear the read/write buffer */ + HDmemset(rwbuf, 0, sizeof(sizeof(int) * (50 * 20))); + + /* Perform activities to ensure that max_lag ticks elapse */ + for(i = 0; i < 500; i++) { + /* Open the dataset */ + sprintf(dname, "dset %d", i); + if((did = H5Dopen2(fid, dname, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Read from the dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) + FAIL_STACK_ERROR + + /* Get dataset object info */ + if(H5Oget_info2(did, &oinfo, H5O_INFO_BASIC) < 0) + FAIL_STACK_ERROR + + /* Close the dataset */ + if(H5Dclose(did) < 0) + FAIL_STACK_ERROR + } + + /* (D) Update two entries in the index */ + index[1].entry_ptr = (void *)&buf[1]; + index[5].entry_ptr = (void *)&buf[5]; + + /* Update with index and verify info in the metadata file */ + /* Also verify that 2/4 entries are inserted/removed to/from the delayed list */ + if(H5F__vfd_swmr_writer_update_md_test(fid, num_entries, index, 2, 4) < 0) + TEST_ERROR /* Close the file */ if(H5Fclose(fid) < 0) FAIL_STACK_ERROR; + if(H5Sclose(sid) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(dcpl) < 0) + FAIL_STACK_ERROR; + if(H5Pclose(fapl) < 0) FAIL_STACK_ERROR; if(H5Pclose(fcpl) < 0) FAIL_STACK_ERROR; + if(my_config) HDfree(my_config); + if(buf) + HDfree(buf); + if(rwbuf) + HDfree(rwbuf); + PASSED() return 0; error: + if(my_config) + HDfree(my_config); + if(buf) + HDfree(buf); + if(rwbuf) + HDfree(rwbuf); + if(index) + HDfree(index); + H5E_BEGIN_TRY { + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); H5Pclose(fapl); H5Pclose(fcpl); H5Fclose(fid); } H5E_END_TRY; - if(my_config) - HDfree(my_config); - return 1; -} /* test_file_end_tick() */ +} /* test_writer_update_md() */ /*------------------------------------------------------------------------- @@ -663,7 +1046,8 @@ main(void) nerrors += test_file_fapl(); nerrors += test_file_end_tick(); - h5_clean_files(FILENAME, fapl); + nerrors += test_writer_md(); + nerrors += test_writer_update_md(); if(nerrors) goto error; |