diff options
| -rw-r--r-- | MANIFEST | 4 | ||||
| -rwxr-xr-x | bin/trace | 1 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/H5FDonion.c | 3483 | ||||
| -rw-r--r-- | src/H5FDonion.h | 149 | ||||
| -rw-r--r-- | src/H5FDonion_priv.h | 237 | ||||
| -rw-r--r-- | src/Makefile.am | 6 | ||||
| -rw-r--r-- | src/hdf5.h | 1 | ||||
| -rw-r--r-- | test/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | test/Makefile.am | 3 | ||||
| -rw-r--r-- | test/onion.c | 3493 | ||||
| -rw-r--r-- | tools/lib/h5tools.c | 2 | ||||
| -rw-r--r-- | tools/lib/h5tools.h | 1 | ||||
| -rw-r--r-- | tools/src/h5dump/h5dump.c | 21 |
14 files changed, 7398 insertions, 6 deletions
@@ -810,6 +810,9 @@ ./src/H5FDmpio.h ./src/H5FDmulti.c ./src/H5FDmulti.h +./src/H5FDonion.c +./src/H5FDonion.h +./src/H5FDonion_priv.h ./src/H5FDros3.c ./src/H5FDros3.h ./src/H5FDpkg.h @@ -1289,6 +1292,7 @@ ./test/objcopy.c ./test/objcopy_ref.c ./test/ohdr.c +./test/onion.c ./test/page_buffer.c ./test/paged_nopersist.h5 ./test/paged_persist.h5 @@ -197,6 +197,7 @@ $Source = ""; "H5FD_t" => "#", "H5FD_hdfs_fapl_t" => "#", "H5FD_mirror_fapl_t" => "#", + "H5FD_onion_fapl_t" => "#", "H5FD_ros3_fapl_t" => "#", "H5FD_splitter_vfd_config_t" => "#", "H5L_class_t" => "#", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d1d848d..2c83cb6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -240,6 +240,7 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDmpi.c ${HDF5_SRC_DIR}/H5FDmpio.c ${HDF5_SRC_DIR}/H5FDmulti.c + ${HDF5_SRC_DIR}/H5FDonion.c ${HDF5_SRC_DIR}/H5FDros3.c ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.c @@ -261,6 +262,7 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDmpi.h ${HDF5_SRC_DIR}/H5FDmpio.h ${HDF5_SRC_DIR}/H5FDmulti.h + ${HDF5_SRC_DIR}/H5FDonion.h ${HDF5_SRC_DIR}/H5FDpublic.h ${HDF5_SRC_DIR}/H5FDros3.h ${HDF5_SRC_DIR}/H5FDs3comms.h diff --git a/src/H5FDonion.c b/src/H5FDonion.c new file mode 100644 index 0000000..f56c808 --- /dev/null +++ b/src/H5FDonion.c @@ -0,0 +1,3483 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: Provide in-file provenance and revision/version control. + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDonion.h" /* Onion file driver */ +#include "H5FDonion_priv.h" /* Onion file driver internals */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ + +/* The driver identification number, initialized at runtime + */ +static hid_t H5FD_ONION_g = 0; + +/****************************************************************************** + * + * Structure: H5FD_onion_t + * + * Purpose: Store information required to manage an onionized file. + * This structure is created when such a file is "opened" and + * discarded when it is "closed". + * + * Programmer: Jacob "Jake" Smith + * 6 July 2020 + * + * `pub` (H5FD_t) + * + * Instance of H5FD_t which contains all fields common to all VFDs. + * It must be the first item in this structure, since at higher levels, + * this structure will be treated as an instance of H5FD_t. + * + * `fa` (H5FD_onion_fapl_info_t) + * + * Instance of `H5FD_onion_fapl_info_t` containing the configuration data + * needed to "open" the HDF5 file. + * + * `backing_canon` (H5FD_t *) + * + * Virtual file handle for the canonical (i.e., logical HDF5) file in the + * backing store. + * + * `backing_onion` (H5FD_t *) + * + * Virtual file handle for the onion file in the backing store. + * NULL if not set to use the single, separate storage target. (TODO) + * + * `backing_recov` (H5FD_t *) + * + * Virtual file handle for the whole-history recovery file. + * + * `name_recov` (char *) + * + * String allocated and populated on file-open in write mode and freed on + * file-close, stores the path/name of the 'recovery' file. The file + * created at this location is to be removed upon succesful file-close + * from write mode. + * + * `is_open_rw` (hbool_t) + * + * Remember whether the file was opened in a read-write mode. + * + * `page_align_history` (hbool_t) + * + * Remember whether onion-writes must be aligned to page boundaries. + * + * `header` (struct H5FD__onion_history_header) + * + * In-memory copy of the onion history data header. + * + * `summary` (struct H5FD__onion_whole_history) + * + * In-memory copy of the onion history "whole-history". + * + * `rev_record` (struct H5FD__onion_revision_record) + * + * `history_eof` (haddr_t) + * + * Last byte in the onion history backing file. + * + * `rev_index` (struct H5FD__onion_revision_index *) + * + * Index for maintaining modified pages. + * Pointer is NULL when the file is not opened in write mode. + * Pointer is allocated on open and must be freed on close. + * Contents must be merged with the revision record's archival index prior + * to commitment of history to backing store. + * + * `history_eof` (haddr_t) + * + * Address of first byte past in-use onion history data. + * + * `origin_eof` (haddr_t) + * + * Size of the origin canonical file. + * + * `logi_eoa` (haddr_t) + * + * Address of first byte past addressed space in logical 'canonical' file. + * + * `logi_eof` (haddr_t) + * + * Address of first byte past Last byte in the logical 'canonical' file. + * Must be copied into the revision record on close to write onion data. + * + ****************************************************************************** + */ +typedef struct H5FD_onion_t { + H5FD_t pub; + H5FD_onion_fapl_info_t fa; + H5FD_t *backing_canon; + H5FD_t *backing_onion; + H5FD_t *backing_recov; + char *name_recov; + hbool_t is_open_rw; + hbool_t page_align_history; + struct H5FD__onion_history_header header; + struct H5FD__onion_whole_history summary; + struct H5FD__onion_revision_record rev_record; + struct H5FD__onion_revision_index *rev_index; + haddr_t history_eof; + haddr_t origin_eof; + haddr_t logi_eoa; + haddr_t logi_eof; +} H5FD_onion_t; + +H5FL_DEFINE_STATIC(H5FD_onion_t); + +#define MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1) + +/* 2^n for uint64_t types -- H5_EXP2 unsafe past 32 bits */ +#define U64_EXP2(n) ((uint64_t)1 << (n)) + +/* Prototypes */ +static herr_t H5FD__onion_close(H5FD_t *); +static haddr_t H5FD__onion_get_eoa(const H5FD_t *, H5FD_mem_t); +static haddr_t H5FD__onion_get_eof(const H5FD_t *, H5FD_mem_t); +static H5FD_t *H5FD__onion_open(const char *, unsigned int, hid_t, haddr_t); +static herr_t H5FD__onion_read(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, + void *); +static herr_t H5FD__onion_set_eoa(H5FD_t *, H5FD_mem_t, haddr_t); +static herr_t H5FD__onion_term(void); +static herr_t H5FD__onion_write(H5FD_t *, H5FD_mem_t, hid_t, haddr_t, size_t, + const void *); + +static int H5FD__onion_archival_index_list_sort_compar(const void *, + const void *); +static herr_t H5FD__onion_injest_whole_history( + struct H5FD__onion_whole_history *whs_out, H5FD_t *raw_file, + haddr_t addr, haddr_t size); +static herr_t H5FD__onion_open_rw(H5FD_onion_t *, unsigned int, haddr_t); +static herr_t H5FD__onion_revision_index_resize( + H5FD__onion_revision_index_t *); +static uint64_t H5FD__onion_whole_history_write( + struct H5FD__onion_whole_history *whs, H5FD_t *file_dest, + haddr_t off_start, haddr_t filesize_curr); + +#if 0 +static void *H5FD_ros3_fapl_get(H5FD_t *_file); +static void *H5FD_ros3_fapl_copy(const void *_old_fa); +static herr_t H5FD_ros3_fapl_free(void *_fa); +static int H5FD_ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD_ros3_query(const H5FD_t *_f1, unsigned long *flags); +static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, + void **file_handle); +static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, + hbool_t closing); +static herr_t H5FD_ros3_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD_ros3_unlock(H5FD_t *_file); +static herr_t H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa); +#endif + +static const H5FD_class_t H5FD_onion_g = { + "onion", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__onion_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_onion_fapl_info_t), /* fapl_size */ + NULL, /*H5FD_ros3_fapl_get,*/ /* fapl_get */ + NULL, /*H5FD_ros3_fapl_copy,*/ /* fapl_copy */ + NULL, /*H5FD_ros3_fapl_free,*/ /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__onion_open, /* open */ + H5FD__onion_close, /* close */ + NULL, /*H5FD_ros3_cmp,*/ /* cmp */ + NULL, /*H5FD_ros3_query,*/ /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD__onion_get_eoa, /* get_eoa */ + H5FD__onion_set_eoa, /* set_eoa */ + H5FD__onion_get_eof, /* get_eof */ + NULL, /*H5FD_ros3_get_handle,*/ /* get_handle */ + H5FD__onion_read, /* read */ + H5FD__onion_write, /* write */ + NULL, /* flush */ + NULL, /*H5FD_ros3_truncate,*/ /* truncate */ + NULL, /*H5FD_ros3_lock,*/ /* lock */ + NULL, /*H5FD_ros3_unlock,*/ /* unlock */ + NULL, /* del */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/*----------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Non-negative on success/Negative on failure + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + if (H5FD_onion_init() < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + "unable to initialize Onion VFD"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD__init_package() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD_onion_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the onion driver. + * Failure: Negative + * + *----------------------------------------------------------------------------- + */ +hid_t +H5FD_onion_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + + if (H5I_VFL != H5I_get_type(H5FD_ONION_g)) { + H5FD_ONION_g = H5FD_register(&H5FD_onion_g, sizeof(H5FD_class_t), + FALSE); + } + + /* Set return value */ + ret_value = H5FD_ONION_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_onion_init() */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__onion_term + * + * Purpose: Shut down the Onion VFD. + * + * Returns: SUCCEED (Can't fail) + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_term(void) +{ + FUNC_ENTER_STATIC_NOERR; + + /* Reset VFL ID */ + H5FD_ONION_g = 0; + + FUNC_LEAVE_NOAPI(SUCCEED); + +} /* end H5FD__onion_term() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5Pget_fapl_onion + * + * Purpose: Copy the Onion configuraiton information from the FAPL at + * `fapl_id` to the destination pointer `fa_out`. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out) +{ + const H5FD_onion_fapl_info_t *info_ptr = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*!", fapl_id, fa_out); + + if (NULL == fa_out) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info-out pointer"); + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (NULL == plist) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID"); + + if (H5FD_ONION != H5P_peek_driver(plist)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Incorrect VFL driver"); + + info_ptr = (const H5FD_onion_fapl_info_t *)H5P_peek_driver_info(plist); + if (NULL == info_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad VFL driver info"); + if (H5FD_ONION_FAPL_INFO_MAGIC != info_ptr->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "bad VFL driver info"); + + HDmemcpy(fa_out, info_ptr, sizeof(H5FD_onion_fapl_info_t)); + +done: + FUNC_LEAVE_API(ret_value) + +} /* end H5Pget_fapl_onion() */ + +/*----------------------------------------------------------------------------- + * Function: H5Pset_fapl_onion + * + * Purpose Set the file access property list at `fapl_id` to use the + * Onion virtual file driver with the given configuration. + * The info structure may be modified or deleted after this call, + * as its contents are copied into the FAPL. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_onion(hid_t fapl_id, const H5FD_onion_fapl_info_t *fa) +{ + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*!", fapl_id, fa); + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (NULL == plist) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Not a valid FAPL ID"); + + if (NULL == fa) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL info pointer"); + + if (H5FD_ONION_FAPL_INFO_MAGIC != fa->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info magic"); + + if (H5FD_ONION_FAPL_INFO_VERSION_CURR != fa->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info version"); + + if (!POWER_OF_TWO(fa->page_size)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size"); + + if (fa->page_size < 1) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid info page size"); + + if (H5P_DEFAULT != fa->backing_fapl_id) { + H5P_genplist_t *_plist_ret = NULL; + + H5E_BEGIN_TRY { + _plist_ret = H5P_object_verify(fa->backing_fapl_id, + H5P_FILE_ACCESS); + } H5E_END_TRY; + if (_plist_ret == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid backing fapl id"); + } + } + + ret_value = H5P_set_driver(plist, H5FD_ONION, (const void *)fa); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_onion() */ + +/*----------------------------------------------------------------------------- + * + * Write in-memory history header to appropriate backing file. + * Overwrites existing header data. + * + * 11 August 2020 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_update_and_write_header(H5FD_onion_t *file) +{ + uint32_t _sum = 0; /* required */ + uint64_t size = 0; + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + /* unset write-lock flag */ + if (file->is_open_rw) + file->header.flags &= (uint32_t)~H5FD__ONION_HEADER_FLAG_WRITE_LOCK; + + buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_HEADER); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate buffer for updated history header"); + } + size = H5FD_onion_history_header_encode(&file->header, buf, &_sum); + if (0 == size) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "problem encoding updated history header"); + } + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, 0, + (haddr_t)size, buf) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "can't write updated history header"); + } + +done: + if (buf != NULL) + HDfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_update_and_write_header()*/ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_whole_history_write() + * + * Purpose: Encode and write whole-history to file at the given address. + * + * Returns: Success: Number of bytes written to destination file. (nonzero) + * Failure: Zero. (0) + * + *----------------------------------------------------------------------------- + */ +static uint64_t +H5FD__onion_whole_history_write(struct H5FD__onion_whole_history *whs, + H5FD_t *file_dest, haddr_t off_start, haddr_t filesize_curr) +{ + uint32_t _sum = 0; /* required */ + uint64_t size = 0; + unsigned char *buf = NULL; + uint64_t ret_value = 0; + + FUNC_ENTER_STATIC; + + buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY + + (H5FD__ONION_ENCODED_SIZE_RECORD_POINTER * whs->n_revisions)); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, 0, + "can't allocate buffer for updated whole-history"); + } + size = H5FD_onion_whole_history_encode(whs, buf, &_sum); + if (0 == size) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, 0, + "problem encoding updated whole-history"); + } + if ((size + off_start > filesize_curr) + && (H5FD_set_eoa(file_dest, H5FD_MEM_DRAW, off_start + size) < 0)) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, 0, + "can't modify EOA for updated whole-history"); + } + if (H5FDwrite(file_dest, H5FD_MEM_DRAW, H5P_DEFAULT, off_start, size, buf) + < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, 0, + "can't write whole-history as intended"); + } + + ret_value = size; + +done: + if (buf != NULL) + HDfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_whole_history_write() */ + +/*----------------------------------------------------------------------------- + * + * Write in-memory whole-history summary to appropriate backing file. + * Update information in other in-memory components. + * + * 11 August 2020 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_update_and_write_whole_history(H5FD_onion_t *file) +{ + uint64_t size = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + /* TODO: history EOF may not be correct (under what circumstances?) */ + + size = H5FD__onion_whole_history_write(&file->summary, file->backing_onion, + file->history_eof, file->history_eof); + if (0 == size) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "can't write updated whole-history"); + } + if (size != file->header.whole_history_size) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "written whole-history differed from expected size"); + } + + /* Is last write operation to history file; no need to extend to page + * boundary if set to page-align. + */ + file->history_eof += size; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_update_and_write_whole_history() */ + + +/*----------------------------------------------------------------------------- + * + * Write in-memory revision record to appropriate backing file. + * Update information in other in-memory components. + * + * 11 August 2020 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_commit_new_revision_record(H5FD_onion_t *file) +{ + uint32_t _sum = 0; /* required */ + uint64_t size = 0; + uint64_t phys_addr = 0; /* offset in history file to record start */ + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + struct H5FD__onion_revision_record *rec_p = &file->rev_record; + struct H5FD__onion_whole_history *whs_p = &file->summary; + struct H5FD__onion_record_pointer *new_list = NULL; + + FUNC_ENTER_STATIC; + + time_t rawtime; + struct tm *info; + time(&rawtime); + info = gmtime(&rawtime); + strftime(rec_p->time_of_creation, sizeof(rec_p->time_of_creation), "%Y%m%dT%H%M%SZ", info); + //HDmemcpy(rec_p->time_of_creation, "19411207T190643Z", 16); + + rec_p->logi_eof = file->logi_eof; + + if ((TRUE == file->is_open_rw) + && (H5FD_onion_merge_revision_index_into_archival_index(file->rev_index, + &file->rev_record.archival_index) < 0)) + { + HGOTO_ERROR(H5E_VFL, H5E_INTERNAL, FAIL, + "unable to update index to write"); + } + + buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_REVISION_RECORD + + (size_t)rec_p->comment_size + + (size_t)rec_p->username_size + + (H5FD__ONION_ENCODED_SIZE_INDEX_ENTRY + * rec_p->archival_index.n_entries)); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate buffer for encoded revision record"); + } + size = H5FD_onion_revision_record_encode(rec_p, buf, &_sum); + if (0 == size) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "problem encoding revision record"); + } + phys_addr = file->history_eof; + if (H5FD_set_eoa(file->backing_onion, H5FD_MEM_DRAW, + phys_addr + size) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, + "can't modify EOA for new revision record"); + } + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, + phys_addr, size, buf) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "can't write new revision record"); + } + + file->history_eof = phys_addr + size; + if (TRUE == file->page_align_history) + file->history_eof = (file->history_eof + (file->header.page_size - 1)) + & (~(file->header.page_size - 1)); + + /* Update whole-history info to accommodate new revision + */ + + if (whs_p->n_revisions == 0) { + unsigned char *ptr = buf; /* re-use buffer space to compute checksum */ + + HDassert(whs_p->record_pointer_list == NULL); + whs_p->n_revisions = 1; + whs_p->record_pointer_list = + (struct H5FD__onion_record_pointer *)HDcalloc(1, + sizeof(struct H5FD__onion_record_pointer)); + if (NULL == whs_p->record_pointer_list) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate temporary record pointer list"); + } + whs_p->record_pointer_list[0].phys_addr = phys_addr; + whs_p->record_pointer_list[0].record_size = size; + UINT64ENCODE(ptr, phys_addr); + UINT64ENCODE(ptr, size); + whs_p->record_pointer_list[0].checksum = H5_checksum_fletcher32(buf, + (size_t)(ptr - buf)); + /* TODO: size-reset belongs where? */ + file->header.whole_history_size += + H5FD__ONION_ENCODED_SIZE_RECORD_POINTER; + } /* end if no extant revisions in history */ + else { + unsigned char *ptr = buf; /* re-use buffer space to compute checksum */ + + HDassert(whs_p->record_pointer_list != NULL); + + new_list = (struct H5FD__onion_record_pointer *)HDcalloc( + whs_p->n_revisions + 1, + sizeof(struct H5FD__onion_record_pointer)); + if (NULL == new_list) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to resize record pointer list"); + } + HDmemcpy(new_list, whs_p->record_pointer_list, + sizeof(struct H5FD__onion_record_pointer) + * whs_p->n_revisions); + HDfree(whs_p->record_pointer_list); + whs_p->record_pointer_list = new_list; + new_list = NULL; + whs_p->record_pointer_list[whs_p->n_revisions].phys_addr = phys_addr; + whs_p->record_pointer_list[whs_p->n_revisions].record_size = size; + UINT64ENCODE(ptr, phys_addr); + UINT64ENCODE(ptr, size); + whs_p->record_pointer_list[whs_p->n_revisions].checksum = + H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + file->header.whole_history_size += + H5FD__ONION_ENCODED_SIZE_RECORD_POINTER; + whs_p->n_revisions += 1; + } /* end if one or more revisions present in history */ + + file->header.whole_history_addr = file->history_eof; + +done: + if (buf != NULL) + HDfree(buf); + if (new_list != NULL) + HDfree(new_list); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_commit_new_revision_record() */ + + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_close + * + * Purpose: Close an onionized file. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_close(H5FD_t *_file) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + HDassert(file != NULL); + + if (H5FD_ONION_STORE_TARGET_H5 == file->fa.store_target) { + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "hdf5 store-target not supported"); + } + else + if (H5FD_ONION_STORE_TARGET_ONION == file->fa.store_target) { + HDassert(file->backing_onion != NULL); + if (TRUE == file->is_open_rw) { + HDassert(file->backing_recov != NULL); + + if (H5FD__onion_commit_new_revision_record(file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "Can't write revision record to backing store"); + + if (H5FD__onion_update_and_write_whole_history(file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "Can't write whole-history to backing store"); + + if (H5FD__onion_update_and_write_header(file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "Can't write updated header to backing store"); + + /* Clean up loose ends. + */ + + if (H5FD_close(file->backing_recov) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "can't close backing recovery file"); + } + file->backing_recov = NULL; + HDremove(file->name_recov); + } /* end if opened in write-mode */ + if (H5FD_close(file->backing_onion) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "can't close backing onion file"); + } + file->backing_onion = NULL; + } + else { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid history target"); + } + + if (file->rev_index != NULL) { + if (H5FD_onion_revision_index_destroy(file->rev_index) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, + "can't close revision index"); + } + file->rev_index = NULL; + } + + if (H5FD_close(file->backing_canon) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "can't close backing canonical file"); + } + +done: + if (FAIL == ret_value) { + if (file->backing_canon != NULL) + (void)H5FD_close(file->backing_canon); + if (file->backing_onion != NULL) + (void)H5FD_close(file->backing_onion); + if (file->backing_recov != NULL) + (void)H5FD_close(file->backing_recov); + if (file->rev_index != NULL) + (void)H5FD_onion_revision_index_destroy(file->rev_index); + } + + /* common, can't-fail releasing of resources */ + if (file->name_recov != NULL) + HDfree(file->name_recov); + if (file->summary.record_pointer_list != NULL) + HDfree(file->summary.record_pointer_list); + if (file->rev_record.username != NULL) + HDfree(file->rev_record.username); + if (file->rev_record.comment != NULL) + HDfree(file->rev_record.comment); + if (file->rev_record.archival_index.list != NULL) + HDfree(file->rev_record.archival_index.list); + file = H5FL_FREE(H5FD_onion_t, file); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_close() */ + + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_get_eoa + * + * Purpose: Get end-of-address address. + * + * Return: Address of first byte past the addressed space + * + *----------------------------------------------------------------------------- + */ +static haddr_t +H5FD__onion_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_onion_t *file = (const H5FD_onion_t *)_file; + + FUNC_ENTER_STATIC_NOERR; + + FUNC_LEAVE_NOAPI(file->logi_eoa) +} /* end H5FD__onion_get_eoa() */ + + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_get_eof + * + * Purpose: Get end-of-file address. + * + * Return: Address of first byte past the file-end. + * + *----------------------------------------------------------------------------- + */ +static haddr_t +H5FD__onion_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_onion_t *file = (const H5FD_onion_t *)_file; + + FUNC_ENTER_STATIC_NOERR; + + FUNC_LEAVE_NOAPI(file->logi_eof) +} /* end H5FD__onion_get_eof() */ + +/*----------------------------------------------------------------------------- + * + * Sanitize the backing FAPL ID + * + *----------------------------------------------------------------------------- + */ +static hid_t +get_legit_fapl_id(hid_t fapl_id) +{ + if (H5P_DEFAULT == fapl_id) + return H5P_FILE_ACCESS_DEFAULT; + else + if (TRUE == H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + return fapl_id; + else + return H5I_INVALID_HID; +} + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_history_header_decode + * + * Purpose: Attempt to read a buffer and store it as a history-header + * structure. + * + * Implementation must correspond with + * H5FD_onion_history_header_encode(). + * + * Return: Success: Number of bytes read from buffer. + * Failure: Zero (0). + * + *----------------------------------------------------------------------------- + */ +uint64_t +H5FD_onion_history_header_decode(unsigned char *buf, + struct H5FD__onion_history_header *header) +{ + uint32_t ui32 = 0; + uint32_t sum = 0; + uint64_t ui64 = 0; + uint8_t *ui8p = NULL; + unsigned char *ptr = NULL; + uint64_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT; + + HDassert(buf != NULL); + HDassert(header != NULL); + HDassert(H5FD__ONION_HEADER_MAGIC == header->magic); + HDassert(H5FD__ONION_HEADER_VERSION_CURR == header->version); + + if (HDstrncmp((const char *)buf, H5FD__ONION_HEADER_SIGNATURE, 4)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header signature"); + + if (buf[4] != H5FD__ONION_HEADER_VERSION_CURR) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid header version"); + + ptr = buf + 5; + ui32 = 0; + HDmemcpy(&ui32, ptr, 3); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, header->flags); + ptr += 3; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, header->page_size); + ptr += 4; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT32DECODE(ui8p, header->origin_eof); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT32DECODE(ui8p, header->whole_history_addr); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT32DECODE(ui8p, header->whole_history_size); + ptr += 8; + + sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, header->checksum); + ptr += 4; + + if (sum != header->checksum) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch"); + + ret_value = (uint64_t)(ptr - buf); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_history_header_decode() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_history_header_encode + * + * Purpose: Write history-header structure to the given buffer. + * All multi-byte elements are stored in little-endian word order. + * + * Implementation must correspond with + * H5FD_onion_history_header_decode(). + * + * The destination buffer must be sufficiently large to hold the + * encoded contents (H5FD__ONION_ENCODED_SIZE_HEADER). + * + * Return: Number of bytes written to buffer. + * The checksum of the generated buffer contents (excluding the + * checksum itself) is stored in the pointer `sum_out`). + * + *----------------------------------------------------------------------------- + */ +uint64_t +H5FD_onion_history_header_encode(struct H5FD__onion_history_header *header, + unsigned char *buf, uint32_t *sum_out) +{ + unsigned char *ptr = buf; + uint64_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR; + + HDassert(buf != NULL); + HDassert(sum_out != NULL); + HDassert(header != NULL); + HDassert(H5FD__ONION_HEADER_MAGIC == header->magic); + HDassert(H5FD__ONION_HEADER_VERSION_CURR == header->version); + HDassert(0 == (header->flags & 0xFF000000)); /* max three bits long */ + + HDmemcpy(ptr, H5FD__ONION_HEADER_SIGNATURE, 4); + ptr += 4; + HDmemcpy(ptr, (unsigned char *)&header->version, 1); + ptr += 1; + UINT32ENCODE(ptr, header->flags); + ptr -= 1; /* truncate to three bytes */ + UINT32ENCODE(ptr, header->page_size); + UINT64ENCODE(ptr, header->origin_eof); + UINT64ENCODE(ptr, header->whole_history_addr); + UINT64ENCODE(ptr, header->whole_history_size); + *sum_out = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + UINT32ENCODE(ptr, *sum_out); + ret_value = (uint64_t)(ptr - buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_history_header_encode() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_revision_record_decode + * + * Purpose: Attempt to read a buffer and store it as a revision record + * structure. + * + * Implementation must correspond with + * H5FD_onion_revision_record_encode(). + * + * MUST BE CALLED TWICE: + * On the first call, n_entries, comment_size, and username_size + * in the destination structure must all all be zero, and their + * respective variable-length components (index_entry_list, + * comment, username) must all be NULL. + * + * If the buffer is well-formed, the destinatino structure is + * tentatively populated with fixed-size values, and the number of + * bytes read are returned. + * + * Prior to the second call, the user must allocate space for the + * variable-length components, in accordance with the associated + * indicators (array of index-entry structures for + * index_entry_list, of size n_entries; character arrays for + * username and comment, allocated with the *_size number of + * bytes -- space for NULL-terminator is included in _size). + * + * Then the decode operation is called a second time, and all + * components will be populated (and again number of bytes read is + * returned). + * + * Return: Success: Number of bytes read from buffer. + * Failure: Zero (0). + * + *----------------------------------------------------------------------------- + */ +uint64_t +H5FD_onion_revision_record_decode(unsigned char *buf, + struct H5FD__onion_revision_record *record) +{ + uint32_t ui32 = 0; + uint32_t page_size = 0; + uint32_t sum = 0; + uint64_t ui64 = 0; + uint64_t n_entries = 0; + uint32_t username_size = 0; + uint32_t comment_size = 0; + uint8_t *ui8p = NULL; + unsigned char *ptr = NULL; + uint64_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT; + + HDassert(buf != NULL); + HDassert(record != NULL); + HDassert(H5FD__ONION_REVISION_RECORD_MAGIC == record->magic); + HDassert(H5FD__ONION_REVISION_RECORD_VERSION_CURR == record->version); + HDassert(H5FD__ONION_ARCHIVAL_INDEX_MAGIC == record->archival_index.magic); + HDassert(H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version); + + if (HDstrncmp((const char *)buf, H5FD__ONION_REVISION_RECORD_SIGNATURE, 4)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature"); + + if (H5FD__ONION_REVISION_RECORD_VERSION_CURR != buf[4]) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid record version"); + + ptr = buf + 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, record->revision_id); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, record->parent_revision_id); + ptr += 8; + + HDmemcpy(record->time_of_creation, ptr, 16); + ptr += 16; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, record->logi_eof); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, page_size); + ptr += 4; + + if (page_size == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size is zero"); + if (!POWER_OF_TWO(page_size)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "page size not power of two"); + + for (record->archival_index.page_size_log2 = 0; + (((uint32_t)1 << record->archival_index.page_size_log2) + & page_size) == 0; + record->archival_index.page_size_log2++); + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, record->user_id); + ptr += 4; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, n_entries); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, username_size); + ptr += 4; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, comment_size); + ptr += 4; + + if (record->archival_index.n_entries == 0) { + record->archival_index.n_entries = n_entries; + ptr += H5FD__ONION_ENCODED_SIZE_INDEX_ENTRY * n_entries; + } + else + if (n_entries != record->archival_index.n_entries) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "n_entries in archival index does not match decoded"); + } + else { + struct H5FD__onion_index_entry *entry_p = NULL; + size_t i = 0; + if (record->archival_index.list == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "no archival index entry list"); + } + for (i = 0; i < n_entries; i++) { + entry_p = &record->archival_index.list[i]; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, entry_p->logi_page); + ptr += 8; + /* logi_page actually encoded as address; check and convert */ + if (entry_p->logi_page & (page_size - 1)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "logical address does not align with page size"); + } + entry_p->logi_page = entry_p->logi_page >> + record->archival_index.page_size_log2; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, entry_p->phys_addr); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, sum); + ptr += 4; + + ui32 = H5_checksum_fletcher32((ptr - 20), 16); + if (ui32 != sum) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "index entry checksum mismatch"); + } + } + } + + if (record->username_size == 0) { + if (record->username != NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "username pointer prematurely allocated"); + } + record->username_size = username_size; + } + else { + if (record->username == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no username pointer"); + HDmemcpy(record->username, ptr, username_size); + } + ptr += username_size; + + if (record->comment_size == 0) { + if (record->comment != NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "comment pointer prematurely allocated"); + } + record->comment_size = comment_size; + } + else { + if (record->comment == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "no comment pointer"); + HDmemcpy(record->comment, ptr, comment_size); + } + ptr += comment_size; + + sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, record->checksum); + ptr += 4; + + if (sum != record->checksum) +{ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch"); +} + + ret_value = (uint64_t)(ptr - buf); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_revision_record_decode() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_revision_record_encode + * + * Purpose: Write revision-record structure to the given buffer. + * All multi-byte elements are stored in little-endian word order. + * + * Implementation must correspond with + * H5FD_onion_revision_record_decode(). + * + * The destination buffer must be sufficiently large to hold the + * encoded contents. + * (Hint: `sizeof(revision-record-struct) + comment-size + + * username-size + sizeof(index-entry-struct) * n_entries)` + * guarantees ample/excess space.) + * + * Return: Number of bytes written to buffer. + * The checksum of the generated buffer contents (excluding the + * checksum itself) is stored in the pointer `sum_out`). + * + *----------------------------------------------------------------------------- + */ +uint64_t +H5FD_onion_revision_record_encode(struct H5FD__onion_revision_record *record, + unsigned char *buf, uint32_t *sum_out) +{ + unsigned char *ptr = buf; /* original pointer */ + uint32_t vers_u32 = (uint32_t)record->version; /* pad out unused bytes */ + uint32_t page_size = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR; + + HDassert(sum_out != NULL); + HDassert(buf != NULL); + HDassert(record != NULL); + HDassert(vers_u32 < 0x100); + HDassert(H5FD__ONION_REVISION_RECORD_MAGIC == record->magic); + HDassert(H5FD__ONION_REVISION_RECORD_VERSION_CURR == record->version); + HDassert(H5FD__ONION_ARCHIVAL_INDEX_MAGIC == record->archival_index.magic); + HDassert(H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR == record->archival_index.version); + + page_size = (uint32_t)(1 << record->archival_index.page_size_log2); + + HDmemcpy(ptr, H5FD__ONION_REVISION_RECORD_SIGNATURE, 4); + ptr += 4; + UINT32ENCODE(ptr, vers_u32); + UINT64ENCODE(ptr, record->revision_id); + UINT64ENCODE(ptr, record->parent_revision_id); + HDmemcpy(ptr, record->time_of_creation, 16); + ptr += 16; + UINT64ENCODE(ptr, record->logi_eof); + UINT32ENCODE(ptr, page_size); + UINT32ENCODE(ptr, record->user_id); + UINT64ENCODE(ptr, record->archival_index.n_entries); + UINT32ENCODE(ptr, record->username_size); + UINT32ENCODE(ptr, record->comment_size); + + if (record->archival_index.n_entries > 0) { + uint64_t i = 0; + uint64_t page_size_log2 = record->archival_index.page_size_log2; + + HDassert(record->archival_index.list != NULL); + for (i = 0; i < record->archival_index.n_entries; i++) { + uint32_t sum = 0; + struct H5FD__onion_index_entry *entry_p = NULL; + uint64_t logi_addr = 0; + + entry_p = &record->archival_index.list[i]; + logi_addr = entry_p->logi_page << page_size_log2; + + UINT64ENCODE(ptr, logi_addr); + UINT64ENCODE(ptr, entry_p->phys_addr); + sum = H5_checksum_fletcher32((ptr - 16), 16); + UINT32ENCODE(ptr, sum); + } + } + + if (record->username_size > 0) { + HDassert(record->username != NULL && *record->username != '\0'); + HDmemcpy(ptr, record->username, record->username_size); + ptr += record->username_size; + } + + if (record->comment_size > 0) { + HDassert(record->comment != NULL && *record->comment != '\0'); + HDmemcpy(ptr, record->comment, record->comment_size); + ptr += record->comment_size; + } + + *sum_out = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + UINT32ENCODE(ptr, *sum_out); + + FUNC_LEAVE_NOAPI((uint64_t)(ptr - buf)); +} /* end H5FD_onion_revision_record_encode() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_whole_history_decode + * + * Purpose: Attempt to read a buffer and store it as a whole-history + * structure. + * + * Implementation must correspond with + * H5FD_onion_whole_history_encode(). + * + * MUST BE CALLED TWICE: + * On the first call, n_records in the destination structure must + * be zero, and record_pointer_list be NULL. + * + * If the buffer is well-formed, the destination structure is + * tentatively populated with fixed-size values, and the number of + * bytes read are returned. + * + * Prior to the second call, the user must allocate space for + * record_pointer_list to hold n_records record-pointer structs. + * + * Then the decode operation is called a second time, and all + * components will be populated (and again number of bytes read is + * returned). + * + * Return: Success: Number of bytes read from buffer. + * Failure: Zero (0). + * + *----------------------------------------------------------------------------- + */ +uint64_t +H5FD_onion_whole_history_decode(unsigned char *buf, + struct H5FD__onion_whole_history *summary) +{ + uint32_t ui32 = 0; + uint32_t sum = 0; + uint64_t ui64 = 0; + uint64_t n_revisions = 0; + uint8_t *ui8p = NULL; + unsigned char *ptr = NULL; + uint64_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT; + + HDassert(buf != NULL); + HDassert(summary != NULL); + HDassert(H5FD__ONION_WHOLE_HISTORY_MAGIC == summary->magic); + HDassert(H5FD__ONION_WHOLE_HISTORY_VERSION_CURR == summary->version); + + if (HDstrncmp((const char *)buf, "OWHS", 4)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid signature"); + + if (H5FD__ONION_WHOLE_HISTORY_VERSION_CURR != buf[4]) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid version"); + + ptr = buf + 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, n_revisions); + ptr += 8; + + if (0 == summary->n_revisions) { + summary->n_revisions = n_revisions; + ptr += H5FD__ONION_ENCODED_SIZE_RECORD_POINTER * n_revisions; + } + else { + uint64_t i = 0; + + if (summary->n_revisions != n_revisions) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "summary argument suggests different revision count than encoded buffer"); + if (NULL == summary->record_pointer_list) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, + "list is NULL -- cannot populate"); + + for (i = 0; i < n_revisions; i++) { + struct H5FD__onion_record_pointer *rpp = + &summary->record_pointer_list[i]; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, rpp->phys_addr); + ptr += 8; + + HDmemcpy(&ui64, ptr, 8); + ui8p = (uint8_t *)&ui64; + UINT64DECODE(ui8p, rpp->record_size); + ptr += 8; + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT64DECODE(ui8p, rpp->checksum); + ptr += 4; + } + } + + sum = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + + HDmemcpy(&ui32, ptr, 4); + ui8p = (uint8_t *)&ui32; + UINT32DECODE(ui8p, summary->checksum); + ptr += 4; + + if (sum != summary->checksum) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "checksum mismatch"); + + ret_value = (uint64_t)(ptr - buf); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_whole_history_decode() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_whole_history_encode + * + * Purpose: Write whole-history structure to the given buffer. + * All multi-byte elements are stored in little-endian word order. + * + * Implementation must correspond with + * H5FD_onion_whole_history_decode(). + * + * The destination buffer must be sufficiently large to hold the + * encoded contents. + * (Hint: `sizeof(whole-history-struct) + + * sizeof(record-pointer-struct) * n_records)` guarantees + * ample/excess space.) + * + * Return: Number of bytes written to buffer. + * The checksum of the generated buffer contents (excluding the + * checksum itself) is stored in the pointer `sum_out`). + * + *----------------------------------------------------------------------------- + */ +uint64_t +H5FD_onion_whole_history_encode(struct H5FD__onion_whole_history *summary, + unsigned char *buf, uint32_t *sum_out) +{ + unsigned char *ptr = buf; + uint32_t vers_u32 = (uint32_t)summary->version; /* pad out unused bytes */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR; + + HDassert(summary != NULL); + HDassert(H5FD__ONION_WHOLE_HISTORY_MAGIC == summary->magic); + HDassert(H5FD__ONION_WHOLE_HISTORY_VERSION_CURR == summary->version); + HDassert(buf != NULL); + HDassert(sum_out != NULL); + + HDmemcpy(ptr, H5FD__ONION_WHOLE_HISTORY_SIGNATURE, 4); + ptr += 4; + UINT32ENCODE(ptr, vers_u32); + UINT64ENCODE(ptr, summary->n_revisions); + if (summary->n_revisions > 0) { + uint64_t i = 0; + + HDassert(summary->record_pointer_list != NULL); /* TODO: error? */ + for (i = 0; i < summary->n_revisions; i++) { + UINT64ENCODE(ptr, summary->record_pointer_list[i].phys_addr); + UINT64ENCODE(ptr, summary->record_pointer_list[i].record_size); + UINT32ENCODE(ptr, summary->record_pointer_list[i].checksum); + } + } + *sum_out = H5_checksum_fletcher32(buf, (size_t)(ptr - buf)); + UINT32ENCODE(ptr, *sum_out); + + FUNC_LEAVE_NOAPI((uint64_t)(ptr - buf)); +} /* end H5FD_onion_whole_history_encode() */ + +/*----------------------------------------------------------------------------- + * + * Populate user_id and username (string) in revision record pointer. + * Assumes that the username string pointer arrives as NULL; + * Allocated username string must be manually freed when done. + * + * 11 August 2020 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_set_userinfo_in_record(struct H5FD__onion_revision_record *rec_p) +{ + uid_t uid = 0; + struct passwd *user_info = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + uid = HDgetuid(); + + HDassert(0 == ((uint64_t)uid & 0xFFFFFFFF00000000)); /* fits uint32_t */ + rec_p->user_id = (uint32_t)uid; + + user_info = HDgetpwuid(uid); + if (NULL == user_info) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't get user info"); + + rec_p->username_size = (uint32_t)HDstrlen(user_info->pw_name) + 1; + + rec_p->username = HDmalloc(sizeof(char) * rec_p->username_size); + if (NULL == rec_p->username) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate space for username string"); + } + + HDmemcpy(rec_p->username, user_info->pw_name, rec_p->username_size); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_set_userinfo_in_record() */ + +/*----------------------------------------------------------------------------- + * + * Create/truncate HDF5 and onion data for a fresh file. + * + * Speical open operation required to instantiate the canonical file and + * history simultaneously. If successful, the required backing files are + * craeated and given initial population on the backing store, and the Onion + * virtual file handle is set; open effects a write-mode open. + * + * Cannot create 'template' history and proceeed with normal write-mode open, + * as this would in effect create an empty first revision, making the history + * unintuitive. (create file -> initialize and commit empty first revision + * (revision 0); any data written to file during the 'create' open, as seen by + * the user, would be in the second revision (revision 1).) + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_create_truncate_onion(H5FD_onion_t *file, const char *filename, + const char *name_onion, const char *name_recovery, unsigned int flags, + haddr_t maxaddr) +{ + hid_t backing_fapl_id = H5I_INVALID_HID; + struct H5FD__onion_history_header *hdr_p = NULL; + struct H5FD__onion_whole_history *whs_p = NULL; + struct H5FD__onion_revision_record *rec_p = NULL; + unsigned char *buf = NULL; + uint64_t size = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + HDassert(file != NULL); + + hdr_p = &file->header; + whs_p = &file->summary; + rec_p = &file->rev_record; + + hdr_p->flags = H5FD__ONION_HEADER_FLAG_WRITE_LOCK; + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY & + file->fa.creation_flags) + hdr_p->flags |= H5FD__ONION_HEADER_FLAG_DIVERGENT_HISTORY; + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & + file->fa.creation_flags) + hdr_p->flags |= H5FD__ONION_HEADER_FLAG_PAGE_ALIGNMENT; + + hdr_p->origin_eof = 0; + + if (H5FD__onion_set_userinfo_in_record(rec_p) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Can't record user info"); + + backing_fapl_id = get_legit_fapl_id(file->fa.backing_fapl_id); + if (H5I_INVALID_HID == backing_fapl_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid backing FAPL ID"); + + /* Create backing files for onion history + */ + + file->backing_canon = H5FD_open(filename, flags, backing_fapl_id, maxaddr); + if (NULL == file->backing_canon) { + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, + "cannot open the backing file"); + } + + file->backing_onion = H5FD_open(name_onion, flags, backing_fapl_id, + maxaddr); + if (NULL == file->backing_onion) { + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, + "cannot open the backing onion file"); + } + + file->backing_recov = H5FD_open(name_recovery, flags, backing_fapl_id, + maxaddr); + if (NULL == file->backing_recov) { + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, + "cannot open the backing file"); + } + + /* Write "empty" .h5 file contents (signature ONIONEOF) + */ + + if (H5FD_set_eoa(file->backing_canon, H5FD_MEM_DRAW, 8) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA"); + /* must use public API to correclty set DXPL context :( */ + if (H5FDwrite(file->backing_canon, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 8, + "ONIONEOF") < 0) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, + "cannot write header to the backing h5 file"); + } + + /* Write nascent whole-history summary (with no revisions) to "recovery". + */ + + buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY); + if (NULL == buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer"); + size = H5FD_onion_whole_history_encode(whs_p, buf, &whs_p->checksum); + if (H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY != size) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode whole-history"); + if (H5FD_set_eoa(file->backing_recov, H5FD_MEM_DRAW, size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA"); + /* must use public API to correclty set DXPL context :( */ + if (H5FDwrite(file->backing_recov, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, + buf) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, + "cannot write summary to the backing recovery file"); + } + hdr_p->whole_history_size = size; /* record for later use */ + HDfree(buf); + buf = NULL; + + /* Write history header with "no" whole-history summary to history. + * Size of the "recovery" history recorded for later use on close. + */ + + buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_HEADER); + if (NULL == buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer"); + size = H5FD_onion_history_header_encode(hdr_p, buf, + &hdr_p->checksum); + if (H5FD__ONION_ENCODED_SIZE_HEADER != size) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history header"); + if (H5FD_set_eoa(file->backing_onion, H5FD_MEM_DRAW, size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA"); + /* must use public API to correclty set DXPL context :( */ + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, + buf) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, + "cannot write header to the backing onion file"); + } + file->history_eof = (haddr_t)size; + if (TRUE == file->page_align_history) + file->history_eof = (file->history_eof + (hdr_p->page_size - 1)) + & (~(hdr_p->page_size - 1)); + + /* list must be allocated */ + rec_p->archival_index.list = (struct H5FD__onion_index_entry *)HDmalloc(0); + + file->rev_index = H5FD_onion_revision_index_init(file->fa.page_size); + if (NULL == file->rev_index) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + "can't initialize revision index"); + } + +done: + if (buf != NULL) + HDfree(buf); + + if (FAIL == ret_value) { + HDremove(name_recovery); /* destroy new temp file, if 'twas created */ + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_create_truncate_onion() */ + +/*----------------------------------------------------------------------------- + * + * Read and decode the history header information from `raw_file` at `addr`, + * and store the decoded information in the structure at `hdr_out`. + * + * 12 August 2020 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_injest_history_header(struct H5FD__onion_history_header *hdr_out, + H5FD_t *raw_file, haddr_t addr) +{ + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + haddr_t size = (haddr_t)H5FD__ONION_ENCODED_SIZE_HEADER; + uint32_t sum = 0; + + FUNC_ENTER_STATIC; + + if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "header indicates whole-history beyond EOF"); + } + + buf = (unsigned char *)HDmalloc(sizeof(char) * size); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate buffer space"); + } + + if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, + "can't modify EOA"); + } + + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, addr, size, buf) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't read history header from file"); + } + + if (H5FD_onion_history_header_decode(buf, hdr_out) == 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, + "can't decode history header"); + } + + sum = H5_checksum_fletcher32(buf, size - 4); + if (hdr_out->checksum != sum) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "checksum mismatch between buffer and stored"); + } + +done: + HDfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_injest_history_header() */ + +/*----------------------------------------------------------------------------- + * + * Read and decode the revision_record information from `raw_file` at + * `addr` .. `addr + size` (taken from whole-history), and store the decoded + * information in the structure at `r_out`. + * + * If successful, `r_out->index_entry_list` is always allocated, even if + * there is zero entries, and `r_out->username` is always allocated. + * `r_out->comment` is allocated and populated iff comment_size for the target + * revision is not zero. + * + * 13 August 2020 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_injest_revision_record(struct H5FD__onion_revision_record *r_out, + H5FD_t *raw_file, const struct H5FD__onion_whole_history *whs, + uint64_t revision_id) +{ + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + uint64_t n = 0; + uint64_t high = 0; + uint64_t low = 0; + uint64_t range = 0; + uint32_t sum = 0; + haddr_t addr = 0; + haddr_t size = 0; + + FUNC_ENTER_STATIC; + + HDassert(r_out != NULL); + HDassert(raw_file != NULL); + HDassert(whs != NULL); + HDassert(whs->record_pointer_list != NULL); + HDassert(whs->n_revisions > 0); + + high = whs->n_revisions - 1; + range = high; + addr = (haddr_t)whs->record_pointer_list[high].phys_addr; + size = (haddr_t)whs->record_pointer_list[high].record_size; + + if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "at least one record extends beyond EOF"); + } + +#if 0 /* TODO: recovery-open */ +/* recovery-open may have EOA below revision record */ + if ((H5FD_get_eoa(raw_file, H5FD_MEM_DRAW) < (addr + size)) + && (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0)) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, + "can't modify EOA"); + } +#endif + + /* Perform binary search on records to find target revision by ID. + * As IDs are added sequentially, they are "guaranteed" to be sorted. + */ + while (range > 0) { + n = (range / 2) + low; + addr = (haddr_t)whs->record_pointer_list[n].phys_addr; + size = (haddr_t)whs->record_pointer_list[n].record_size; + + buf = (unsigned char *)HDmalloc(sizeof(char) * size); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate buffer space"); + } + + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, addr, size, buf) + < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't read revision record from file"); + } + + if (H5FD_onion_revision_record_decode(buf, r_out) != size) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, + "can't decode revision record (initial)"); + } + + sum = H5_checksum_fletcher32(buf, size - 4); + if (r_out->checksum != sum) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "checksum mismatch between buffer and stored"); + } + + if (revision_id == r_out->revision_id) + break; + + HDfree(buf); + buf = NULL; + + r_out->archival_index.n_entries = 0; + r_out->comment_size = 0; + r_out->username_size = 0; + + if (r_out->revision_id < revision_id) + low = (n == high) ? high : n + 1; + else + high = (n == low) ? low : n - 1; + range = high - low; + } /* end while 'non-leaf' binary search */ + + if (range == 0) { + n = low; + addr = (haddr_t)whs->record_pointer_list[n].phys_addr; + size = (haddr_t)whs->record_pointer_list[n].record_size; + + buf = (unsigned char *)HDmalloc(sizeof(char) * size); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate buffer space"); + } + + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, addr, size, buf) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't read revision record from file"); + } + + if (H5FD_onion_revision_record_decode(buf, r_out) != size) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, + "can't decode revision record (initial)"); + } + + sum = H5_checksum_fletcher32(buf, size - 4); + if (r_out->checksum != sum) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "checksum mismatch between buffer and stored"); + } + + if (revision_id != r_out->revision_id) { + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, + "could not find target revision!"); /* TODO: corrupted? */ + } + } /* end if revision ID at 'leaf' in binary search */ + + + r_out->username = (char *)HDmalloc(sizeof(char) * r_out->username_size); + if (NULL == r_out->username) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate username space"); + } + + if (r_out->comment_size > 0) { + r_out->comment = (char *)HDmalloc(sizeof(char) * r_out->comment_size); + if (NULL == r_out->comment) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate comment space"); + } + } + + r_out->archival_index.list = (struct H5FD__onion_index_entry *) + HDcalloc(r_out->archival_index.n_entries, + sizeof(struct H5FD__onion_index_entry)); + if (NULL == r_out->archival_index.list) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate index entry list"); + } + + if (H5FD_onion_revision_record_decode(buf, r_out) != size) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, + "can't decode revision record (final)"); + } + +done: + if (buf != NULL) + HDfree(buf); + if (ret_value == FAIL) { + if (r_out->comment != NULL) + HDfree(r_out->comment); + if (r_out->username != NULL) + HDfree(r_out->username); + if (r_out->archival_index.list != NULL) + HDfree(r_out->archival_index.list); + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_injest_revision_record() */ + +/*----------------------------------------------------------------------------- + * + * Read and decode the whole-history information from `raw_file` at + * `addr` .. `addr + size` (taken from history header), and store the decoded + * information in the structure at `whs_out`. + * + * If successful, `whs_out->record_pointer_list` is always allocated, even if + * there is zero revisions. + * + * 12 August 2020 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_injest_whole_history(struct H5FD__onion_whole_history *whs_out, + H5FD_t *raw_file, haddr_t addr, haddr_t size) +{ + unsigned char *buf = NULL; + herr_t ret_value = SUCCEED; + uint32_t sum = 0; + + FUNC_ENTER_STATIC; + + if (H5FD_get_eof(raw_file, H5FD_MEM_DRAW) < (addr + size)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "header indicates whole-history beyond EOF"); + } + + buf = (unsigned char *)HDmalloc(sizeof(char) * size); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate buffer space"); + } + + if (H5FD_set_eoa(raw_file, H5FD_MEM_DRAW, (addr + size)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, + "can't modify EOA"); + } + + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, addr, size, buf) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't read whole-history from file"); + } + + if (H5FD_onion_whole_history_decode(buf, whs_out) != size) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, + "can't decode whole-history (initial)"); + } + + sum = H5_checksum_fletcher32(buf, size - 4); + if (whs_out->checksum != sum) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "checksum mismatch between buffer and stored"); + } + + whs_out->record_pointer_list = (struct H5FD__onion_record_pointer *) + HDcalloc(whs_out->n_revisions, + sizeof(struct H5FD__onion_record_pointer)); + if (NULL == whs_out->record_pointer_list) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate record pointer list"); + } + + if (H5FD_onion_whole_history_decode(buf, whs_out) != size) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, + "can't decode whole-history (final)"); + } + +done: + HDfree(buf); + if (ret_value == FAIL) { + if (whs_out->record_pointer_list != NULL) + HDfree(whs_out->record_pointer_list); + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_injest_whole_history() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_open + * + * Purpose: Open an onionized file. + * + * Return: Success: Virtual file pointer. + * Failure: NULL. + * + *----------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD__onion_open(const char *filename, unsigned flags, hid_t fapl_id, + haddr_t maxaddr) +{ + H5FD_onion_t *file = NULL; + H5FD_onion_fapl_info_t fa; + H5FD_t *ret_value = NULL; + hid_t backing_fapl_id = H5I_INVALID_HID; + char *name_onion = NULL; + char *name_recovery = NULL; + + FUNC_ENTER_STATIC; + + if (H5Pget_fapl_onion(fapl_id, &fa) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get FAPL info"); + + if (H5FD_ONION_STORE_TARGET_H5 == fa.store_target) { + HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, + "same-file storage not implemented"); + } + else + if (H5FD_ONION_STORE_TARGET_ONION != fa.store_target ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid store target"); + } + + file = H5FL_CALLOC(H5FD_onion_t); + if (NULL == file) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, + "unable to allocate file struct"); + } + + name_onion = (char *)HDmalloc(sizeof(char) * (HDstrlen(filename) + 7)); + if (NULL == name_onion) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, + "unable to allocate onion name string"); + } + HDsnprintf(name_onion, HDstrlen(filename) + 7, "%s.onion", filename); + + name_recovery = (char *)HDmalloc(sizeof(char) * (HDstrlen(name_onion) + 10)); + file->name_recov = (char *)HDmalloc(sizeof(char) * (HDstrlen(name_onion) + 10)); + if (NULL == name_recovery) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, + "unable to allocate recovery name string"); + } + HDsnprintf(name_recovery, HDstrlen(name_onion) + 10, "%s.recovery", name_onion); + HDsnprintf(file->name_recov, HDstrlen(name_onion) + 10, "%s.recovery", name_onion); + + backing_fapl_id = get_legit_fapl_id(file->fa.backing_fapl_id); + if (H5I_INVALID_HID == backing_fapl_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid backing FAPL ID"); + + /* Initialize internal comonents + */ + + HDmemcpy(&(file->fa), &fa, sizeof(H5FD_onion_fapl_info_t)); + file->is_open_rw = FALSE; + + file->header.magic = H5FD__ONION_HEADER_MAGIC; + file->header.version = H5FD__ONION_HEADER_VERSION_CURR; + file->header.page_size = file->fa.page_size; /* guarded on FAPL-set */ + + file->summary.magic = H5FD__ONION_WHOLE_HISTORY_MAGIC; + file->summary.version = H5FD__ONION_WHOLE_HISTORY_VERSION_CURR; + + file->rev_record.magic = H5FD__ONION_REVISION_RECORD_MAGIC; + file->rev_record.version = H5FD__ONION_REVISION_RECORD_VERSION_CURR; + file->rev_record.archival_index.magic = H5FD__ONION_ARCHIVAL_INDEX_MAGIC; + file->rev_record.archival_index.version = + H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR; + /* compute and store log2(page_size) */ + for (file->rev_record.archival_index.page_size_log2 = 0; + (((uint32_t)1 << file->rev_record.archival_index.page_size_log2) + & fa.page_size) == 0; + file->rev_record.archival_index.page_size_log2++); + + /* Proceed with open. */ + + if ((H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) { + if (fa.creation_flags + & H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT) + { + file->header.flags |= H5FD__ONION_HEADER_FLAG_PAGE_ALIGNMENT; + file->page_align_history = TRUE; + } + + /* Truncate and create everything as necessary */ + if (H5FD__onion_create_truncate_onion(file, filename, name_onion, + file->name_recov, flags, maxaddr) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTCREATE, NULL, + "unable to create/truncate onionized files."); + } + file->is_open_rw = TRUE; + } /* end if creating onion from scratch */ + else { + + file->backing_canon = H5FD_open(filename, flags, backing_fapl_id, + maxaddr); + if (NULL == file->backing_canon) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, + "unable to open canonical file (does not exist?)."); + } + + H5E_BEGIN_TRY { + file->backing_onion = H5FD_open(name_onion, flags, backing_fapl_id, + maxaddr); + } H5E_END_TRY; + if (NULL == file->backing_onion) { + if (H5F_ACC_RDWR & flags) { + hid_t backing_fapl_id = H5I_INVALID_HID; + struct H5FD__onion_history_header *hdr_p = NULL; + struct H5FD__onion_whole_history *whs_p = NULL; + struct H5FD__onion_revision_record *rec_p = NULL; + unsigned char *head_buf = NULL; + unsigned char *wh_buf = NULL; + uint64_t size = 0; + uint64_t saved_size = 0; + herr_t ret_value = SUCCEED; + + //HDassert(file != NULL); + + hdr_p = &file->header; + whs_p = &file->summary; + rec_p = &file->rev_record; + + //hdr_p->flags = H5FD__ONION_HEADER_FLAG_WRITE_LOCK; + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY & + file->fa.creation_flags) + hdr_p->flags |= H5FD__ONION_HEADER_FLAG_DIVERGENT_HISTORY; + if (H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT & + file->fa.creation_flags) + hdr_p->flags |= H5FD__ONION_HEADER_FLAG_PAGE_ALIGNMENT; + + // TODO: is this right? + //hdr_p->origin_eof = 0; + + if (H5FD__onion_set_userinfo_in_record(rec_p) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Can't record user info"); + + backing_fapl_id = get_legit_fapl_id(file->fa.backing_fapl_id); + if (H5I_INVALID_HID == backing_fapl_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid backing FAPL ID"); + + /* Create backing files for onion history + */ + + file->backing_onion = H5FD_open(name_onion, (H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC), backing_fapl_id, + maxaddr); + if (NULL == file->backing_onion) { + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, + "AAAAA cannot open the backing onion file"); + } + + file->backing_recov = H5FD_open(name_recovery, (H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC), backing_fapl_id, + maxaddr); + if (NULL == file->backing_recov) { + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, + "AAAAA cannot open the backing file"); + } + + ///snipped from here + + /* Write history header with "no" whole-history summary to history. + * Size of the "recovery" history recorded for later use on close. + */ + hdr_p->whole_history_size = H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY; /* record for later use */ + hdr_p->whole_history_addr = H5FD__ONION_ENCODED_SIZE_HEADER + 1; /* TODO: comment these 2 or do some other way */ + head_buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_HEADER); + if (NULL == head_buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer"); + size = H5FD_onion_history_header_encode(hdr_p, head_buf, + &hdr_p->checksum); + if (H5FD__ONION_ENCODED_SIZE_HEADER != size) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode history header"); + //if (H5FD_set_eoa(file->backing_onion, H5FD_MEM_DRAW, size) < 0) + // HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA"); + /* must use public API to correclty set DXPL context :( */ + + + wh_buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY); + if (NULL == wh_buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate buffer"); + saved_size = size; + whs_p->n_revisions = 0; + //whs_p->n_revisions = 1; + //whs_p->record_pointer_list = (struct H5FD__onion_record_pointer *)HDmalloc(0); + //whs_p->record_pointer_list = (struct H5FD__onion_record_pointer *)HDcalloc(whs_p->n_revisions, sizeof(struct H5FD__onion_record_pointer)); + size = H5FD_onion_whole_history_encode(whs_p, wh_buf, &whs_p->checksum); + // MOVED HERE + file->header.whole_history_size = size; /* record for later use */ + // TODO: double check + if (H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY != size) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't encode whole-history"); + } + if (H5FD_set_eoa(file->backing_onion, H5FD_MEM_DRAW, saved_size + size + 1) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't extend EOA"); + /* must use public API to correclty set DXPL context :( */ + //if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, + + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, 0, saved_size, + head_buf) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, + "cannot write header to the backing onion file"); + } + file->history_eof = (haddr_t)saved_size; + if (TRUE == file->page_align_history) + file->history_eof = (file->history_eof + (hdr_p->page_size - 1)) + & (~(hdr_p->page_size - 1)); + + /* list must be allocated */ + rec_p->archival_index.list = (struct H5FD__onion_index_entry *)HDmalloc(0); + + file->rev_index = H5FD_onion_revision_index_init(file->fa.page_size); + if (NULL == file->rev_index) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + "can't initialize revision index"); + } + + ///to here + //// + file->header.whole_history_addr = file->history_eof; + /* Write nascent whole-history summary (with no revisions) to "recovery". + */ + + // TODO: JUST SETUP file->summary???????? and return????? + + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, saved_size + 1, size, + wh_buf) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, + "cannot write summary to the backing recovery file"); + } + //file->header.whole_history_size = size; /* record for later use */ + //hdr_p->whole_history_size = size; /* record for later use */ + //hdr_p->whole_history_addr = file->history_eof; + // TODO: Free stuff? + //HDfree(buf); + //buf = NULL; + + + + // TODO: move + /*if (buf != NULL) + HDfree(buf); + + if (FAIL == ret_value) { + HDremove(name_recovery); /* destroy new temp file, if 'twas created */ + //} + } + else { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, + "unable to open onion file (does not exist?)."); + } + } + + if (H5FD__onion_injest_history_header(&file->header, + file->backing_onion, 0) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, + "can't get history header from backing store"); + } + file->page_align_history = + (file->header.flags & H5FD__ONION_HEADER_FLAG_PAGE_ALIGNMENT) + ? TRUE : FALSE; + + if (H5FD__ONION_HEADER_FLAG_WRITE_LOCK & file->header.flags) { + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, + "Can't open file already opened in write-mode"); + } /* end if file is already opened in write-mode */ + else { + if (H5FD__onion_injest_whole_history(&file->summary, + file->backing_onion, file->header.whole_history_addr, + file->header.whole_history_size) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, + "can't get whole-history from backing store"); + } + + if (fa.revision_id >= file->summary.n_revisions + && fa.revision_id != H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "target revision ID out of range"); + } + + //if (H5FD__onion_injest_revision_record(&file->rev_record, + if (file->summary.n_revisions > 0 && H5FD__onion_injest_revision_record(&file->rev_record, + file->backing_onion, &file->summary, + MIN(fa.revision_id, (file->summary.n_revisions - 1))) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, NULL, + "can't get revision record from backing store"); + } + + if (H5F_ACC_RDWR & flags) { + if (H5FD__onion_open_rw(file, flags, maxaddr) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, + "can't write-open write-locked file"); + } + } + } /* end if file not already opened in write-mode */ + + } /* end if opening existing file */ + + /* Copy comment from FAPL info, if one is given */ + if ((H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC) & flags) { + if (*fa.comment != '\0') { + size_t comment_size = HDstrlen(fa.comment) + 1; + file->rev_record.comment = (char *)HDmalloc(sizeof(char) + * comment_size); + if (NULL == file->rev_record.comment) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, + "unable to allocate comment string"); + } + HDmemcpy(file->rev_record.comment, fa.comment, comment_size); + file->rev_record.comment_size = (uint32_t)comment_size; + } /* end if comment present in FAPL info */ + } /* end if flags indicate write-mode */ + + file->origin_eof = file->header.origin_eof; + file->logi_eof = file->rev_record.logi_eof; + file->logi_eoa = 0; + + file->history_eof = H5FD_get_eoa(file->backing_onion, H5FD_MEM_DRAW); + if (TRUE == file->page_align_history) + file->history_eof = (file->history_eof + (file->header.page_size - 1)) + & (~(file->header.page_size - 1)); + + ret_value = (H5FD_t *)file; + +done: + if (name_onion != NULL) + HDfree(name_onion); + + if (ret_value == NULL) { + + if (file->backing_canon != NULL) { + (void)H5FD_close(file->backing_canon); + file->backing_canon = NULL; + } + if (file->backing_onion != NULL) { + (void)H5FD_close(file->backing_onion); + file->backing_onion = NULL; + } + if (file->backing_recov != NULL) { + (void)H5FD_close(file->backing_recov); + file->backing_recov = NULL; + } + if (file->name_recov != NULL) { + HDfree(file->name_recov); + file->name_recov = NULL; + } + if ((file->rev_index != NULL) + && (H5FD_onion_revision_index_destroy(file->rev_index) < 0)) + { + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, NULL, + "can't destroy revision index"); + } + if (file->rev_record.comment != NULL) { + HDfree(file->rev_record.comment); + file->rev_record.comment = NULL; + } + if (file->rev_record.username != NULL) { + HDfree(file->rev_record.username); + file->rev_record.username = NULL; + } + if (file != NULL) + H5FL_FREE(H5FD_onion_t, file); + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_open() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_open_rw() + * + * Purpose: Complete onion file-open, handling process for write mode. + * + * Creates recovery file if one does not exist. + * Initializes 'live' revision index. + * Force write-open is not yet supported (recovery provision) TODO + * Establishes write-lock in history header (sets lock flag). + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_open_rw(H5FD_onion_t *file, unsigned int flags, haddr_t maxaddr) +{ + unsigned char *buf = NULL; + uint64_t size = 0; + uint32_t _sum = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + /* Guard against simultaneous write-open. + * TODO: support recovery open with force-write-open flag in FAPL info. + */ + + if (file->header.flags & H5FD__ONION_HEADER_FLAG_WRITE_LOCK) { + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "can't write-open write-locked file"); + } + + /* Copy whole-history to recovery file. + */ + + file->backing_recov = H5FD_open(file->name_recov, + (flags | H5F_ACC_CREAT | H5F_ACC_TRUNC), file->fa.backing_fapl_id, + maxaddr); + if (NULL == file->backing_recov) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, + "unable to create recovery file"); + } + + size = H5FD__onion_whole_history_write(&file->summary, + file->backing_recov, 0, 0); + if (0 == size) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "can't write whole-history to recovery file"); + } + if (size != file->header.whole_history_size) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "written whole-history differed from expected size"); + } + + /* Set write-lock flag in Onion header. + */ + + buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_HEADER); + if (NULL == buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate space for encoded buffer"); + } + + file->header.flags |= H5FD__ONION_HEADER_FLAG_WRITE_LOCK; + + size = H5FD_onion_history_header_encode(&file->header, buf, &_sum); + if (0 == size) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "problem encoding history header"); + } + + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, 0, + (haddr_t)size, buf) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "can't write updated history header"); + } + + /* Prepare revision index and finalize write-mode open. + */ + + file->rev_index = H5FD_onion_revision_index_init(file->fa.page_size); + if (NULL == file->rev_index) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + "can't initialize revision index"); + } + file->rev_record.parent_revision_id = file->rev_record.revision_id; + file->rev_record.revision_id += 1; + file->is_open_rw = TRUE; + +done: + if (FAIL == ret_value) { + if (file->backing_recov != NULL) { + if (H5FD_close(file->backing_recov) < 0) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "can't close recovery file"); + } + else { + file->backing_recov = NULL; + } + } + + if (file->rev_index != NULL) { + if (H5FD_onion_revision_index_destroy(file->rev_index) < 0) { + HDONE_ERROR(H5E_VFL, H5E_CANTRELEASE, FAIL, + "can't destroy revision index"); + } + else { + file->rev_index = NULL; + } + } + } /* end if failure */ + + if (buf != NULL) + HDfree(buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_open_rw() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_read + * + * Purpose: Read bytes from an onionized file. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_read(H5FD_t *_file, H5FD_mem_t type, + hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len, + void *_buf_out) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + uint64_t page_0 = 0; + size_t n_pages = 0; + uint32_t page_size = 0; + uint32_t page_size_log2 = 0; + size_t i = 0; + size_t j = 0; + size_t bytes_to_read = len; + unsigned char *buf_out = (unsigned char *)_buf_out; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + HDassert(file != NULL); + HDassert(buf_out != NULL); + + if ((uint64_t)(offset + len) > file->logi_eoa) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Read extends beyond addressed space"); + } + + if (0 == len) + goto done; + + page_size = file->header.page_size; + page_size_log2 = file->rev_record.archival_index.page_size_log2; + page_0 = offset >> page_size_log2; + n_pages = (len + page_size - 1) >> page_size_log2; + + /* Read, page-by-page */ + for (i = 0; i < n_pages; i++) { + const struct H5FD__onion_index_entry *entry_out_p = NULL; + haddr_t page_gap_head = 0; /* start of page to start of buffer */ + haddr_t page_gap_tail = 0; /* end of buffer to end of page */ + size_t page_readsize = 0; + uint64_t page_i = page_0 + i; + + if (0 == i) + page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1); + if (n_pages - 1 == i) + page_gap_tail = page_size - bytes_to_read - page_gap_head; + page_readsize = (size_t)page_size - page_gap_head - page_gap_tail; + + + if (TRUE == file->is_open_rw + && H5FD_onion_revision_index_find(file->rev_index, page_i, + &entry_out_p)) + { + if (H5FDread(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, + (haddr_t)entry_out_p->phys_addr + page_gap_head, + page_readsize, buf_out) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't get working file data"); + } + } /* end if page exists in 'live' revision index */ + else + if (H5FD_onion_archival_index_find(&file->rev_record.archival_index, + page_i, &entry_out_p)) + { + if (H5FDread(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, + (haddr_t)entry_out_p->phys_addr + page_gap_head, + page_readsize, buf_out) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't get previously-amended file data"); + } + } /* end if page exists in 'dead' archival index */ + else + { + /* casts prevent truncation */ + haddr_t addr_start = (haddr_t)page_i * (haddr_t)page_size + + (haddr_t)page_gap_head; + haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 + : file->origin_eof - addr_start; + haddr_t read_size = MIN(overlap_size, page_readsize); +#if 0 +HDputs("READING from original file"); +HDprintf("page_size: %llu, addr_start: %llu page_gap_head: %llu, page_gap_tail: %llu overlap_size: %llu page_readsize:%llu read_size: %llu\n", page_size, addr_start, page_gap_head, page_gap_tail, overlap_size, page_readsize, read_size); +#endif + /* Get all original bytes in page range */ + if ((read_size > 0) + && H5FDread(file->backing_canon, type, H5P_DEFAULT, + addr_start, read_size, buf_out) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't get original file data"); + } + + /* Fill with 0s any gaps after end of original bytes + * and before end of page. + */ + for (j = read_size; j < page_readsize; j++) + buf_out[j] = 0; + } /* end if page exists in neither index */ + + buf_out += page_readsize; + bytes_to_read -= page_readsize; + } /* end for each page in range */ + + HDassert(0 == bytes_to_read); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_read() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_set_eoa + * + * Purpose: Set end-of-address marker of the logical file. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + + FUNC_ENTER_STATIC_NOERR; + + file->logi_eoa = addr; + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* end H5FD__onion_set_eoa() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_write + * + * Purpose: Write bytes to an onionized file. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_write(H5FD_t *_file, H5FD_mem_t type, + hid_t H5_ATTR_UNUSED dxpl_id, haddr_t offset, size_t len, + const void *_buf) +{ + H5FD_onion_t *file = (H5FD_onion_t *)_file; + uint64_t page_0 = 0; + size_t n_pages = 0; + unsigned char *page_buf = NULL; + uint32_t page_size = 0; + uint32_t page_size_log2 = 0; + size_t i = 0; + size_t j = 0; + size_t bytes_to_write = len; + const unsigned char *buf = (const unsigned char *)_buf; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + HDassert(file != NULL); + HDassert(buf != NULL); + HDassert(file->rev_index != NULL); + HDassert((uint64_t)(offset + len) <= file->logi_eoa); + + if (FALSE == file->is_open_rw) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Write not allowed if file not opened in write mode"); + } + + if (0 == len) + goto done; + + page_size = file->header.page_size; + page_size_log2 = file->rev_record.archival_index.page_size_log2; + page_0 = offset >> page_size_log2; + n_pages = (len + page_size - 1) >> page_size_log2; + + page_buf = (unsigned char *)HDcalloc(page_size, sizeof(unsigned char)); + if (NULL == page_buf) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "cannot allocate temporary buffer"); + } + + /* Write, page-by-page */ + for (i = 0; i < n_pages; i++) { + const unsigned char *write_buf = buf; + struct H5FD__onion_index_entry new_entry; + const struct H5FD__onion_index_entry *entry_out_p = NULL; + haddr_t page_gap_head = 0; /* start of page to start of buffer */ + haddr_t page_gap_tail = 0; /* end of buffer to end of page */ + size_t page_n_used = 0; /* nbytes from buffer for this page-write */ + uint64_t page_i = page_0 + i; + + if (0 == i) + page_gap_head = offset & (((uint32_t)1 << page_size_log2) - 1); + if (n_pages - 1 == i) + page_gap_tail = page_size - bytes_to_write - page_gap_head; + page_n_used = page_size - page_gap_head - page_gap_tail; + + /* Modify page in revision index, if present */ + if (H5FD_onion_revision_index_find(file->rev_index, page_i, &entry_out_p)) { + if (page_gap_head | page_gap_tail) { + /* Copy existing page verbatim. */ + if (H5FDread(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, + (haddr_t)entry_out_p->phys_addr, page_size, page_buf) + < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't get working file data"); + } + /* Overlay delta from input buffer onto page buffer. */ + HDmemcpy(page_buf, buf, page_n_used); + write_buf = page_buf; + } /* end if partial page */ + + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, + (haddr_t)entry_out_p->phys_addr, page_size, write_buf) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "write amended page data to backing file"); + } + + buf += page_n_used; /* overflow never touched */ + bytes_to_write -= page_n_used; + + continue; + } /* end if page exists in 'live' revision index */ + + if (page_gap_head || page_gap_tail) { + /* Fill gaps with existing data or zeroes. */ + if (H5FD_onion_archival_index_find( + &file->rev_record.archival_index, page_i, &entry_out_p)) + { + /* Copy existing page verbatim. */ + if (H5FDread(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, + (haddr_t)entry_out_p->phys_addr, page_size, page_buf) + < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't get previously-amended data"); + } + } /* end if page exists in 'dead' archival index */ + else + { + haddr_t addr_start = (haddr_t)(page_i * page_size); + haddr_t overlap_size = (addr_start > file->origin_eof) ? 0 + : file->origin_eof - addr_start; + haddr_t read_size = MIN(overlap_size, page_size); + + /* Get all original bytes in page range */ + if ((read_size > 0) + && H5FDread(file->backing_canon, type, H5P_DEFAULT, + addr_start, read_size, page_buf) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, + "can't get original file data"); + } + + /* Fill with 0s any gaps after end of original bytes + * or start of page and before start of new data. + */ + for (j = read_size; j < page_gap_head; j++) + page_buf[j] = 0; + + /* Fill with 0s any gaps after end of original bytes + * or end of new data and before end of page. + */ + for (j = MAX(read_size, page_size - page_gap_tail); + j < page_size; j++) + { + page_buf[j] = 0; + } + } /* end if page exists in neither index */ + + /* Copy input buffer to temporary page buffer */ + HDmemcpy(page_buf + page_gap_head, buf, page_n_used); + write_buf = page_buf; + + } /* end if data range does not span entire page */ + + new_entry.logi_page = page_i; + new_entry.phys_addr = file->history_eof; + + if (H5FD_set_eoa(file->backing_onion, H5FD_MEM_DRAW, + file->history_eof + page_size) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, + "can't modify EOA for new page amendment"); + } + if (H5FDwrite(file->backing_onion, H5FD_MEM_DRAW, H5P_DEFAULT, + file->history_eof, page_size, write_buf) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "write amended page data to backing file"); + } + if (H5FD_onion_revision_index_insert(file->rev_index, &new_entry) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINSERT, FAIL, + "can't insert new index entry into revision index"); + } + + file->history_eof += page_size; + buf += page_n_used; /* possible overflow never touched */ + bytes_to_write -= page_n_used; + + } /* end for each page to write */ + + HDassert(0 == bytes_to_write); + + file->logi_eof = MAX(file->logi_eof, (offset + len)); + +done: + if (page_buf != NULL) + HDfree(page_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_write() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_archival_index_is_valid + * + * Purpose: Determine whether an archival index structure is valid. + * + * + Verify magic number and version (sanity checking). + * + Verify page size (power of two). + * + Verify list exists. + * + Verify list contents: + * + Sorted by increasing logical address (no duplicates) + * + Logical addresses are multiples of page size. + * + * Return: 1 if above creteria are met. + * 0 otherwise. + * + *----------------------------------------------------------------------------- + */ +int +H5FD_onion_archival_index_is_valid( + const struct H5FD__onion_archival_index *aix) +{ + hbool_t ret_value = 1; + + FUNC_ENTER_NOAPI_NOINIT; + + if (NULL == aix) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "pointer is null"); + if (H5FD__ONION_ARCHIVAL_INDEX_MAGIC != aix->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid magic"); + if (H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR != aix->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid version"); + if (NULL == aix->list) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "list is NULL"); + + if (aix->n_entries > 1) { + uint64_t i = 1; + uint64_t prev_page = aix->list[0].logi_page; + uint64_t logi_page = aix->list[1].logi_page; + + for (i = 1; + i < aix->n_entries; + prev_page = logi_page, i++, logi_page = aix->list[i].logi_page) + { + if (logi_page <= prev_page) { + ret_value = 0; + break; + } + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_archival_index_is_valid() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_archival_index_find + * + * Purpose: Retrieve the archival index entry by logical page ID. + * + * The archival index pointer must point to a valid index entry. + * The entry out pointer-pointer cannot be null. + * + * Return: Success: Positive value (1) -- entry found. + * Entry out pointer-pointer is set to point to entry. + * Failure: Zero (0) -- entry not found. + * Entry out pointer-pointer is unmodified. + * + *----------------------------------------------------------------------------- + */ +int +H5FD_onion_archival_index_find( + const struct H5FD__onion_archival_index *aix, uint64_t logi_page, + const struct H5FD__onion_index_entry **entry_out_p) +{ + uint64_t low = 0; + uint64_t high = 0; + uint64_t n = 0; + uint64_t range = 0; + struct H5FD__onion_index_entry *x = NULL; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR; + + HDassert(aix != NULL); + HDassert(H5FD__ONION_ARCHIVAL_INDEX_MAGIC == aix->magic); + HDassert(H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR == aix->version); + HDassert(entry_out_p != NULL); + if (aix->n_entries != 0) + HDassert(aix->list != NULL); + + high = aix->n_entries - 1; + range = high; + + if (aix->n_entries == 0 + || logi_page > aix->list[high].logi_page + || logi_page < aix->list[0].logi_page) + { + ret_value = 0; /* pre-empt address out of range */ + goto done; + } + + /* + * Binary search on sorted list + */ + + /* winnow down to first of found or one element */ + while (range > 0) { + HDassert(high < aix->n_entries); + n = low + (range / 2); + x = &(aix->list[n]); + if (x->logi_page == logi_page) { + *entry_out_p = x; /* element found at fence */ + ret_value = 1; + goto done; + } + else + if (x->logi_page < logi_page) { + low = (n == high) ? high : n + 1; + } + else { + high = (n == low) ? low : n - 1; + } + range = high - low; + } + + HDassert(high == low); /* one element */ + /* n==low/high check because we may have tested it already above */ + if ((n != low || n != high) && (aix->list[low].logi_page == logi_page)) { + *entry_out_p = &aix->list[low]; + ret_value = 1; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_archival_index_find() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_revision_index_destroy + * + * Purpose: Release all resources of a revision index. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +herr_t +H5FD_onion_revision_index_destroy(H5FD__onion_revision_index_t *rix) +{ + size_t i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + if (NULL == rix) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, -1, "null index pointer"); + if (H5FD__ONION_REVISION_INDEX_MAGIC != rix->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, -1, "invalid index magic"); + if (H5FD__ONION_REVISION_INDEX_VERSION_CURR != rix->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, -1, "invalid index version"); + + for (i = 0; + 0 < rix->_hash_table_n_keys_populated && i < rix->_hash_table_size; + i++) + { + struct H5FD__onion_revision_index_hash_chain_node *next_p = NULL; + struct H5FD__onion_revision_index_hash_chain_node *node_p = + rix->_hash_table[i]; + + if (node_p != NULL) + rix->_hash_table_n_keys_populated -= 1; + + while (node_p != NULL) { + HDassert(H5FD__ONION_REVISION_INDEX_HASH_CHAIN_NODE_MAGIC + == node_p->magic); + HDassert(H5FD__ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR + == node_p->version); + + next_p = node_p->next; + HDfree(node_p); + node_p = next_p; + } + } + HDfree(rix->_hash_table); + rix->magic++; + HDfree(rix); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_revision_index_destroy() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_revision_index_init + * + * Purpose: Initialize a revision index structure with a default starting + * size. A new structure is allocated and populated with initial + * values. + * + * Return: Success: Pointer to newly-allocated structure. + * Failure: NULL + * + *----------------------------------------------------------------------------- + */ +H5FD__onion_revision_index_t * +H5FD_onion_revision_index_init(uint32_t page_size) +{ + uint64_t table_size = + U64_EXP2(H5FD__ONION_REVISION_INDEX_STARTING_SIZE_LOG2); + H5FD__onion_revision_index_t *rix = NULL; + H5FD__onion_revision_index_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT; + + if (0 == page_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "page size must be greater than zero (0)"); + } + if (!POWER_OF_TWO(page_size)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "page size must be a power of 2"); + } + + rix = (H5FD__onion_revision_index_t *)HDmalloc( + sizeof(H5FD__onion_revision_index_t)); + if (NULL == rix) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, + "cannot allocate index"); + } + + rix->_hash_table = NULL; /* initialize to catch error */ + rix->_hash_table = (struct H5FD__onion_revision_index_hash_chain_node **) + HDcalloc(table_size, + sizeof(struct H5FD__onion_revision_index_hash_chain_node *)); + + if (NULL == rix->_hash_table) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, + "cannot allocate hash table"); + } + + rix->magic = H5FD__ONION_REVISION_INDEX_MAGIC; + rix->version = H5FD__ONION_REVISION_INDEX_VERSION_CURR; + rix->n_entries = 0; + /* compute and store log2(page_size) */ + for (rix->page_size_log2 = 0; + (((uint32_t)1 << rix->page_size_log2) & page_size) == 0; + rix->page_size_log2++); + rix->_hash_table_size = table_size; + rix->_hash_table_size_log2 = H5FD__ONION_REVISION_INDEX_STARTING_SIZE_LOG2; + rix->_hash_table_n_keys_populated = 0; + + ret_value = rix; + +done: + if (NULL == ret_value && NULL != rix) + HDfree(rix); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_revision_index_init() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__onion_revision_index_resize() + * + * Purpose: Replace the hash table in the revision index. + * + * Doubles the available number of keys, re-hashes table contents, + * and updates relevant components in the index structure. + * + * Fails if unable to allocate space for larger hash table. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__onion_revision_index_resize(H5FD__onion_revision_index_t *rix) +{ + struct H5FD__onion_revision_index_hash_chain_node **new_table = NULL; + uint64_t i = 0; + uint64_t new_size_log2 = rix->_hash_table_size_log2 + 1; + uint64_t new_size = U64_EXP2(new_size_log2); + uint64_t new_n_keys_populated = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC; + + /* function not user-exposed: asserts suffice for sanity-checks */ + HDassert(rix != NULL); + HDassert(H5FD__ONION_REVISION_INDEX_MAGIC == rix->magic); + HDassert(H5FD__ONION_REVISION_INDEX_VERSION_CURR == rix->version); + HDassert(rix->_hash_table != NULL); + + new_table = (struct H5FD__onion_revision_index_hash_chain_node **) + HDcalloc(new_size, + sizeof(struct H5FD__onion_revision_index_hash_chain_node *)); + if (NULL == new_table) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "cannot allocate new hash table"); + } + + for (i = 0; i < rix->_hash_table_size; i++) { + while (rix->_hash_table[i] != NULL) { + struct H5FD__onion_revision_index_hash_chain_node *node = NULL; + uint64_t key = 0; + + /* pop entry off of bucket stack and re-hash */ + node = rix->_hash_table[i]; + rix->_hash_table[i] = node->next; + node->next = NULL; + key = node->entry_data.logi_page & (new_size - 1); + + if (NULL == new_table[key]) { + new_table[key] = node; + new_n_keys_populated++; + } + else { + node->next = new_table[i]; + new_table[i] = node; + } + } + } + + HDfree(rix->_hash_table); + rix->_hash_table_size = new_size; + rix->_hash_table_size_log2 = new_size_log2; + rix->_hash_table_n_keys_populated = new_n_keys_populated; + rix->_hash_table = new_table; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__onion_revision_index_resize() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_revision_index_insert() + * + * Purpose: Add an entry to the revision index, or update an existing + * entry. Must be used to update entries as well as add -- + * checksum value will change. + * + * Entry data is copied into separate memory region; user pointer + * can be safley re-used or discarded after operation. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +herr_t +H5FD_onion_revision_index_insert(H5FD__onion_revision_index_t *rix, + const struct H5FD__onion_index_entry *entry) +{ + uint64_t key = 0; + struct H5FD__onion_revision_index_hash_chain_node *node = NULL; + struct H5FD__onion_revision_index_hash_chain_node **append_dest = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + if (NULL == rix) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null index pointer"); + if (H5FD__ONION_REVISION_INDEX_MAGIC != rix->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index magic"); + if (H5FD__ONION_REVISION_INDEX_VERSION_CURR != rix->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid index version"); + if (NULL == entry) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null entry pointer"); + + /* Resize and re-hash table if necessary. + */ + if (rix->n_entries >= (rix->_hash_table_size * 2) + || rix->_hash_table_n_keys_populated >= (rix->_hash_table_size / 2)) + { + if (H5FD__onion_revision_index_resize(rix) < 0) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NONE_MINOR, FAIL, + "unable to resize and hash table"); + } + } + + key = entry->logi_page & (rix->_hash_table_size - 1); + HDassert(key < rix->_hash_table_size); + + if (NULL == rix->_hash_table[key]) { + append_dest = &rix->_hash_table[key]; + rix->_hash_table_n_keys_populated++; + } /* end if insert into unoccupied bucket */ + else { + for (node = rix->_hash_table[key]; node != NULL; node = node->next) { + append_dest = &node->next; /* look for bucket tail */ + if (entry->logi_page == node->entry_data.logi_page) { + if (entry->phys_addr != node->entry_data.phys_addr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "physical address mismatch"); + } + HDmemcpy(&node->entry_data, entry, /* TODO: can fail? */ + sizeof(struct H5FD__onion_index_entry)); + append_dest = NULL; /* node updated; do not append */ + break; + } /* end if page ID match with chain node; update node */ + } + } /* end if key maps to populated bucket */ + + if (append_dest != NULL) { + node = (struct H5FD__onion_revision_index_hash_chain_node *)HDmalloc( + sizeof(struct H5FD__onion_revision_index_hash_chain_node)); + if (NULL == node) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "cannot allocate new ash chain node"); + } + node->magic = H5FD__ONION_REVISION_INDEX_HASH_CHAIN_NODE_MAGIC; + node->version = H5FD__ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR; + node->next = NULL; + HDmemcpy(&node->entry_data, entry, + sizeof(struct H5FD__onion_index_entry)); + *append_dest = node; + rix->n_entries++; + } /* end if new entry to bucket chain */ + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_revision_index_insert() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_revision_index_find() + * + * + * Purpose: Get pointer to revision index entry with the given page number, + * if it exists in the index. + * + * Return: Success: Positive value (1) -- entry found. + * Entry out pointer-pointer is set to point to entry. + * Failure: Zero (0) -- entry not found. + * Entry out pointer-pointer is unmodified. + * + *----------------------------------------------------------------------------- + */ +int +H5FD_onion_revision_index_find(const H5FD__onion_revision_index_t *rix_p, + uint64_t logi_page, + const struct H5FD__onion_index_entry **entry_out_p) +{ + uint64_t key = 0; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR; + + HDassert(rix_p != NULL); + HDassert(H5FD__ONION_REVISION_INDEX_MAGIC == rix_p->magic); + HDassert(H5FD__ONION_REVISION_INDEX_VERSION_CURR == rix_p->version); + HDassert(rix_p->_hash_table != NULL); + HDassert(entry_out_p != NULL); + + key = logi_page & (rix_p->_hash_table_size - 1); + HDassert(key < rix_p->_hash_table_size); + + if (rix_p->_hash_table[key] != NULL) { + struct H5FD__onion_revision_index_hash_chain_node *node = NULL; + + for (node = rix_p->_hash_table[key]; node != NULL; node = node->next) { + if (logi_page == node->entry_data.logi_page) { + *entry_out_p = &node->entry_data; + ret_value = 1; + break; + } + } + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_revision_index_find() */ + +/*----------------------------------------------------------------------------- + * + * Callback for comparisons in sorting archival index entries by logi_page. + * + *----------------------------------------------------------------------------- + */ +static int +H5FD__onion_archival_index_list_sort_compar(const void *_a, const void *_b) +{ + const struct H5FD__onion_index_entry *a = + (const struct H5FD__onion_index_entry *)_a; + const struct H5FD__onion_index_entry *b = + (const struct H5FD__onion_index_entry *)_b; + + if (a->logi_page < b->logi_page) + return -1; + else + if (a->logi_page > b->logi_page) + return 1; + return 0; +} /* end H5FD__onion_archival_index_list_sort_compar() */ + +/*----------------------------------------------------------------------------- + * + * Function: H5FD_onion_merge_revision_index_into_archival_index() + * + * Purpose: Merge index entries from revision index into archival index. + * + * If successful, the archival index is expanded 'behind the + * scenes' and new entries from the revision index are inserted. + * The archival index remains sorted in ascending order of logical + * address. + * + * The conversion to archival index changes logical pages in + * revision index entries to their logical addresses in-file. + * + * No resources are created or destroyed as part of this routine. + * The archival index MUST have its list pointer allocated, even + * if there are zero entries. + * + * Return: Success: Non-negative value (SUCCEED). + * Failure: Negative value (FAIL). + * + *----------------------------------------------------------------------------- + */ +herr_t +H5FD_onion_merge_revision_index_into_archival_index( + const H5FD__onion_revision_index_t *rix, + struct H5FD__onion_archival_index *aix) +{ + uint64_t i = 0; + uint64_t n_kept = 0; + struct H5FD__onion_index_entry *kept_list = NULL; + struct H5FD__onion_archival_index new_aix = { + H5FD__ONION_ARCHIVAL_INDEX_MAGIC, + H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR, + 0, /* page_size_log2 tbd */ + 0, /* n_entries */ + NULL, /* list tbd */ + }; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + if (NULL == rix) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "null revision index pointer"); + } + if (NULL == aix) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "null archival index pointer"); + } + + if (H5FD__ONION_REVISION_INDEX_MAGIC != rix->magic) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid revision index magic"); + } + if (H5FD__ONION_ARCHIVAL_INDEX_MAGIC != aix->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null archival index"); + + if (H5FD__ONION_REVISION_INDEX_VERSION_CURR != rix->version) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid revision index version"); + } + if (H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR != aix->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null archival version"); + + if (NULL == aix->list) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null archival index list"); + + if (aix->page_size_log2 != rix->page_size_log2) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "page size mismatch"); + + /* Short-circuit degenerate case */ + if (rix->n_entries == 0) + goto done; + + /* Add all 'live' revision index entries to new list (unsorted) */ + new_aix.page_size_log2 = aix->page_size_log2; + new_aix.list = (struct H5FD__onion_index_entry *)HDcalloc(rix->n_entries, + sizeof(struct H5FD__onion_index_entry)); + if (NULL == new_aix.list) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to allocate larger archival index list"); + } + for (i = 0; i < rix->_hash_table_size; i++) { + const struct H5FD__onion_revision_index_hash_chain_node *node_p = NULL; + + for (node_p = rix->_hash_table[i]; + node_p != NULL; + node_p = node_p->next) + { + HDmemcpy(&new_aix.list[new_aix.n_entries], &node_p->entry_data, + sizeof(struct H5FD__onion_index_entry)); + new_aix.n_entries++; + } + } + HDqsort(new_aix.list, new_aix.n_entries, + sizeof(struct H5FD__onion_index_entry), + H5FD__onion_archival_index_list_sort_compar); + + /* Add any remaining 'dead' archival index entries to list (sorted) */ + /* supplementary new archival index space for possible archived entries */ + n_kept = 0; + kept_list = (struct H5FD__onion_index_entry *)HDcalloc(aix->n_entries, + sizeof(struct H5FD__onion_index_entry)); + if (NULL == kept_list) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to allocate larger archival index list"); + } + for (i = 0; i < aix->n_entries; i++) { + const struct H5FD__onion_index_entry *_p = NULL; + + /* Add only if page not already added from live index */ + if (H5FD_onion_archival_index_find(&new_aix, aix->list[i].logi_page, + &_p) == 0) + { + HDmemcpy(&kept_list[n_kept], &aix->list[i], + sizeof(struct H5FD__onion_index_entry)); + n_kept++; + } + } + + /* destroy prev list and replace with exact-sized buffer w/ new contents */ + HDfree(aix->list); + aix->list = (struct H5FD__onion_index_entry *)HDcalloc( + new_aix.n_entries + n_kept, + sizeof(struct H5FD__onion_index_entry)); + if (NULL == aix->list) { + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "unable to allocate exact-size archival index list"); + } + /* copy new entries to replacement list */ + HDmemcpy(aix->list, new_aix.list, + sizeof(struct H5FD__onion_index_entry) * new_aix.n_entries); + aix->n_entries = new_aix.n_entries; + /* copy old entries to replacement list */ + HDmemcpy(&aix->list[aix->n_entries], kept_list, + sizeof(struct H5FD__onion_index_entry) * n_kept); + aix->n_entries += n_kept; + + /* cleanup temporary allocs */ + HDfree(kept_list); + kept_list = NULL; + HDfree(new_aix.list); + new_aix.list = NULL; + new_aix.magic++; /* new_aix allocated on the stack; no free */ + + HDqsort(aix->list, aix->n_entries, sizeof(struct H5FD__onion_index_entry), + H5FD__onion_archival_index_list_sort_compar); + +done: + if (kept_list != NULL) + HDfree(kept_list); + if (new_aix.list != NULL) + HDfree(new_aix.list); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_onion_merge_revision_index_into_entry_list() */ + diff --git a/src/H5FDonion.h b/src/H5FDonion.h new file mode 100644 index 0000000..696eb66 --- /dev/null +++ b/src/H5FDonion.h @@ -0,0 +1,149 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: The public header file for the Onion VFD. + */ +#ifndef H5FDonion_H +#define H5FDonion_H + +#define H5FD_ONION (H5FD_onion_init()) + +#define H5FD_ONION_ENABLE_INDEX_STATS 0 + +#define H5FD_ONION_FAPL_INFO_MAGIC 0x10101010 /* TODO */ +#define H5FD_ONION_FAPL_INFO_VERSION_CURR 1 +#define H5FD_ONION_FAPL_INFO_FLAG_FORCE_OPEN 1 +#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY 1 +#define H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT 2 +#define H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN 255 +#define H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST (uint64_t)(-1) + +enum H5FD_onion_target_file_constant { + H5FD_ONION_STORE_TARGET_H5, /* onion history as part of H5 file */ + H5FD_ONION_STORE_TARGET_ONION, /* separate, single "onion" file */ + /* TODO: other storage location/scheme? */ +}; + +/*----------------------------------------------------------------------------- + * Structure H5FD_onion_fapl_info_t + * + * Purpose: Encapsulate info for the Onion driver FAPL entry. + * + * magic: "Magic number" identifying struct. + * Must equal H5FD_ONION_FAPL_MAGIC to be considered valid. + * + * version: Future-proofing identifier. Informs struct membership. + * Must equal H5FD_ONION_FAPL_VERSION_CURR to be considered valid. + * + * backing_fapl_id: + * Backing or 'child' FAPL ID to handle I/O with the + * underlying backing store. If the onion data is stored as a + * separate file, it must use the same backing driver as the + * original file. + * + * page_size: Size of the amended data pages. If opening an existing file, + * must equal the existing page size or zero (0). If creating a + * new file or an initial revision of an existing file, must be a + * power of 2. + * + * store_target: + * Enumerated/defined value identifying where the history data is + * stored, either in the same file (appended to HDF5 data) or a + * separate file. Other options may be added in later versions. + * + * + H5FD_ONION_FAPL_STORE_MODE_SEPARATE_SINGLE (1) + * Onion history is stored in a single, separate "onion + * file". Shares filename and path as hdf5 file (if any), + * with only a different filename extension. + * + * revision_id: Which revision to open. Must be 0 (the original file) or the + * revision number of an existing revision. + * Revision ID -1 is reserved to open the most recently-created + * revision in history. + * + * force_write_open: + * Flag to ignore the write-lock flag in the onion data + * and attempt to open the file write-only anyway. + * This may be relevant if, for example, the library crashed + * while the file was open in write mode and the write-lock + * flag was not cleared. + * Must equal H5FD_ONION_FAPL_FLAG_FORCE_OPEN to enable. + * + * creation_flags: + * Flag used only when instantiating an Onion file. + * If the relevant bit is set to a nonzero value, its feature + * will be enabled. + * + * + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_DIVERGENT_HISTORY + * (1, bit 1) + * User will be allowed to open arbitrary revisions + * in write mode. + * If disabled (0), only the most recent revision may be + * opened for amendment. + * + * + H5FD_ONION_FAPL_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT (2, bit 2) + * Onion history metadata will align to page_size. + * Partial pages of unused space will occur in the file, + * but may improve read performance from the backing store + * on some systems. + * If disabled (0), padding will not be inserted to align + * to page boundaries. + * + * + <Remaining bits reserved> + * + * comment: User-supplied NULL-terminated comment for a revision to be + * written. + * Cannot be longer than H5FD_ONION_FAPL_COMMENT_MAX_LEN. + * Ignored if part of a FAPL used to open in read mode. + * + * The comment for a revision may be modified prior to committing + * to the revision (closing the file and writing the record) + * with a call to H5FDfctl(). + * This H5FDfctl overwrite may be used to exceed constraints of + * maximum string length and the NULL-terminator requirement. + * + *----------------------------------------------------------------------------- + */ +typedef struct H5FD_onion_fapl_info_t { + uint32_t magic; + uint8_t version; + hid_t backing_fapl_id; + uint32_t page_size; + enum H5FD_onion_target_file_constant store_target; + uint64_t revision_id; + uint8_t force_write_open; + uint8_t creation_flags; + char comment[H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN + 1]; +} H5FD_onion_fapl_info_t; + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * PUBLIC PROTOTYPES + */ + +H5_DLL hid_t H5FD_onion_init(void); +H5_DLL herr_t H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out); +H5_DLL herr_t H5Pset_fapl_onion(hid_t fapl_id, + const H5FD_onion_fapl_info_t *fa); + +#ifdef __cplusplus +} +#endif + +#endif /* H5FDonion_H */ + diff --git a/src/H5FDonion_priv.h b/src/H5FDonion_priv.h new file mode 100644 index 0000000..a4cff71 --- /dev/null +++ b/src/H5FDonion_priv.h @@ -0,0 +1,237 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) Internals. + * + * Purpose: The private header file for the Onion VFD. + * Contains definitions and declarations used internallay and by + * tests. + */ + +#ifndef H5FDonion_priv_H +#define H5FDonion_priv_H + +/* + * INTERNAL MACROS AND DEFINITIONS + */ + +#define H5FD__ONION_ARCHIVAL_INDEX_MAGIC 0x11111111 /* TODO */ +#define H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR 1 + +/* Number of bytes to encode fixed-size components */ +#define H5FD__ONION_ENCODED_SIZE_HEADER 40 +#define H5FD__ONION_ENCODED_SIZE_INDEX_ENTRY 20 +#define H5FD__ONION_ENCODED_SIZE_RECORD_POINTER 20 +#define H5FD__ONION_ENCODED_SIZE_REVISION_RECORD 76 +#define H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY 20 + +/* Flags must align exactly one per bit, up to 24 bits */ +#define H5FD__ONION_HEADER_FLAG_WRITE_LOCK 0x1 +#define H5FD__ONION_HEADER_FLAG_DIVERGENT_HISTORY 0x2 +#define H5FD__ONION_HEADER_FLAG_PAGE_ALIGNMENT 0x4 +#define H5FD__ONION_HEADER_MAGIC 0x433421fa /* TODO */ +#define H5FD__ONION_HEADER_SIGNATURE "OHDH" +#define H5FD__ONION_HEADER_VERSION_CURR (uint8_t)1 + +#define H5FD__ONION_REVISION_INDEX_HASH_CHAIN_NODE_MAGIC 0x33333333 /* TODO */ +#define H5FD__ONION_REVISION_INDEX_HASH_CHAIN_NODE_VERSION_CURR 1 +#define H5FD__ONION_REVISION_INDEX_MAGIC 0x22222222 /* TODO */ +#define H5FD__ONION_REVISION_INDEX_STARTING_SIZE_LOG2 10 /* 2^n slots */ +#define H5FD__ONION_REVISION_INDEX_VERSION_CURR (uint8_t)1 + +#define H5FD__ONION_REVISION_RECORD_MAGIC 0x54672381 /* TODO */ +#define H5FD__ONION_REVISION_RECORD_SIGNATURE "ORRS" +#define H5FD__ONION_REVISION_RECORD_VERSION_CURR (uint8_t)1 + +#define H5FD__ONION_WHOLE_HISTORY_MAGIC 0xb38a0921 /* TODO */ +#define H5FD__ONION_WHOLE_HISTORY_SIGNATURE "OWHS" +#define H5FD__ONION_WHOLE_HISTORY_VERSION_CURR (uint8_t)1 + +/* + * INTERNAL STRUCTURE DEFINITIONS + */ + +/*----------------------------------------------------------------------------- + * + * Structure H5FD__onion_index_entry + * + * Purpose: Map a page in the logical file to a 'physical address' in the + * backing store. + * + * logi_page: Page 'id' in the logical file. + * + * phys_addr: Address/offset of start of page in the backing store. + * + *----------------------------------------------------------------------------- + */ +struct H5FD__onion_index_entry { + uint64_t logi_page; + uint64_t phys_addr; +}; + +/*----------------------------------------------------------------------------- + * + * Structure H5FD__onion_archival_index + * + * Purpose: Encapsulate archival index and associated data. + * Convenience structure with sanity-checking components. + * + * magic: "Magic number" identifying struct. + * Must equal H5FD__ONION_ARCHIVAL_INDEX_MAGIC to be considered + * valid. + * + * version: Future-proofing identifier. Informs struct membership. + * Must equal H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR to be + * considered valid. + * + * page_size: Interval to which the `logi_page` component of each list + * entry must align. + * Value is taken from the onion history data; must not change + * following onionization or file or creation of onion file. + * + * n_entries: Number of entries in the list. + * + * list: Pointer to array of archival index entries. + * Cannot be NULL. + * Entries must be sorted by `logi_page_id` in ascending order. + * + *----------------------------------------------------------------------------- + */ +struct H5FD__onion_archival_index { + uint32_t magic; + uint8_t version; + uint32_t page_size_log2; + uint64_t n_entries; + struct H5FD__onion_index_entry *list; +}; + +/* data structure for storing index entries at a hash key collision */ +/* version 1 implements a singly-linked list */ +struct H5FD__onion_revision_index_hash_chain_node { + uint32_t magic; + uint8_t version; + struct H5FD__onion_index_entry entry_data; + struct H5FD__onion_revision_index_hash_chain_node *next; +}; + +typedef struct H5FD__onion_revision_index { + uint32_t magic; + uint8_t version; + uint32_t page_size_log2; + uint64_t n_entries; /* count of all entries in table */ + uint64_t _hash_table_size; /* 'slots' in hash table */ + uint64_t _hash_table_size_log2; /* 2^(n) -> 'slots' in hash table */ + uint64_t _hash_table_n_keys_populated; /* count of slots not NULL */ + struct H5FD__onion_revision_index_hash_chain_node **_hash_table; +} H5FD__onion_revision_index_t; + +/* In-memory representation of the on-store onion history file header. + */ +struct H5FD__onion_history_header { + uint32_t magic; + uint8_t version; + uint32_t flags; /* at most three bytes used! */ + uint32_t page_size; + uint64_t origin_eof; /* size of the 'original' canonical file */ + uint64_t whole_history_addr; + uint64_t whole_history_size; + uint32_t checksum; +}; + +/* In-memory representation of the on-store revision record. + */ +struct H5FD__onion_revision_record { + uint32_t magic; + uint8_t version; + uint64_t revision_id; + uint64_t parent_revision_id; + char time_of_creation[16]; + uint64_t logi_eof; + uint32_t user_id; + uint32_t username_size; + uint32_t comment_size; + struct H5FD__onion_archival_index archival_index; + char *username; + char *comment; + uint32_t checksum; +}; + +/* In-memory representation of the on-store revision record pointer. + * Used in the whole-history. + */ +struct H5FD__onion_record_pointer { + uint64_t phys_addr; + uint64_t record_size; + uint32_t checksum; +}; + +/* In-memory representation of the on-store whole-history record/summary. + */ +struct H5FD__onion_whole_history { + uint32_t magic; + uint8_t version; + uint64_t n_revisions; + struct H5FD__onion_record_pointer *record_pointer_list; + uint32_t checksum; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * INTERNAL FUNCTION DECLARATIONS + */ + +H5_DLL int H5FD_onion_archival_index_is_valid( + const struct H5FD__onion_archival_index *); +H5_DLL int H5FD_onion_archival_index_find( + const struct H5FD__onion_archival_index *, uint64_t, + const struct H5FD__onion_index_entry **); + +H5_DLL struct H5FD__onion_revision_index * H5FD_onion_revision_index_init( + uint32_t page_size); +H5_DLL herr_t H5FD_onion_revision_index_destroy( + H5FD__onion_revision_index_t *); +H5_DLL herr_t H5FD_onion_revision_index_insert( + H5FD__onion_revision_index_t *, + const struct H5FD__onion_index_entry *); +H5_DLL int H5FD_onion_revision_index_find( + const H5FD__onion_revision_index_t *, uint64_t, + const struct H5FD__onion_index_entry **); + +H5_DLL herr_t H5FD_onion_merge_revision_index_into_archival_index( + const H5FD__onion_revision_index_t *, + struct H5FD__onion_archival_index *); + +H5_DLL uint64_t H5FD_onion_history_header_decode(unsigned char *, + struct H5FD__onion_history_header *); +H5_DLL uint64_t H5FD_onion_history_header_encode( + struct H5FD__onion_history_header *, unsigned char *, uint32_t *); + +H5_DLL uint64_t H5FD_onion_revision_record_decode(unsigned char *, + struct H5FD__onion_revision_record *); +H5_DLL uint64_t H5FD_onion_revision_record_encode( + struct H5FD__onion_revision_record *, unsigned char *, uint32_t *); + +H5_DLL uint64_t H5FD_onion_whole_history_decode(unsigned char *, + struct H5FD__onion_whole_history *); +H5_DLL uint64_t H5FD_onion_whole_history_encode( + struct H5FD__onion_whole_history *, unsigned char *, uint32_t *); + +#ifdef __cplusplus +} +#endif + +#endif /* H5FDonion_priv_H */ + diff --git a/src/Makefile.am b/src/Makefile.am index ce6e3b1..afbe0c2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -62,7 +62,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c H5FDfamily.c H5FDint.c H5FDlog.c \ - H5FDmulti.c H5FDsec2.c H5FDspace.c \ + H5FDmulti.c H5FDonion.c H5FDsec2.c H5FDspace.c \ H5FDsplitter.c H5FDstdio.c H5FDtest.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ H5FSstat.c H5FStest.c \ @@ -145,8 +145,8 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers H5Cpublic.h H5Dpublic.h \ H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \ H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \ - H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \ - H5FDsec2.h H5FDsplitter.h H5FDstdio.h H5FDwindows.h \ + H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDonion.h \ + H5FDros3.h H5FDsec2.h H5FDsplitter.h H5FDstdio.h H5FDwindows.h \ H5Gpublic.h H5Ipublic.h H5Lpublic.h \ H5Mpublic.h H5MMpublic.h H5Opublic.h H5Ppublic.h \ H5PLextern.h H5PLpublic.h \ @@ -64,6 +64,7 @@ #include "H5FDmirror.h" /* Mirror VFD and IPC definitions */ #include "H5FDmpi.h" /* MPI-based file drivers */ #include "H5FDmulti.h" /* Usage-partitioned file family */ +#include "H5FDonion.h" /* Onion file I/O */ #include "H5FDros3.h" /* R/O S3 "file" I/O */ #include "H5FDsec2.h" /* POSIX unbuffered file I/O */ #include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 56427f5..f849dae 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -333,6 +333,7 @@ set (H5_TESTS timer cmpd_dtransform event_set + onion ) macro (ADD_H5_EXE file) diff --git a/test/Makefile.am b/test/Makefile.am index d465664..0821ab9 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -66,7 +66,8 @@ TEST_PROG= testhdf5 \ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ - unregister cache_logging cork swmr thread_id vol timer event_set + unregister cache_logging cork swmr thread_id vol timer event_set \ + onion # List programs to be built when testing here. # error_test and err_compat are built at the same time as the other tests, but executed by testerror.sh. diff --git a/test/onion.c b/test/onion.c new file mode 100644 index 0000000..ae492ba --- /dev/null +++ b/test/onion.c @@ -0,0 +1,3493 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Onion Virtual File Driver (VFD) + * + * Purpose: + * + * Verify Onion VFD behavior that is not involved with operations on the + * backing store. + */ + +#include "h5test.h" + +#include "H5Fprivate.h" /* encode/decode macros */ +#include "H5FDonion.h" /* This file driver's utilities */ +#include "H5FDonion_priv.h" /* Onion file driver internals */ + +/* 2^n for uint64_t types -- H5_EXP2 unsafe past 32 bits */ +#define U64_EXP2(n) ((uint64_t)1 << (n)) + +#define ONION_TEST_PAGE_SIZE_1 (uint32_t)4 +#define ONION_TEST_PAGE_SIZE_5 (uint32_t)32 +#define ONION_TEST_BASENAME_SIZE 32 +#define ONION_TEST_FIXNAME_SIZE 1024 +#define ONION_TEST_EXPECTED_HISTORY_REVISIONS_MAX 16 +#define ONION_TEST_REV_REV_WRITES_MAX 8 +#define ONION_TEST_REV_REV_MAGIC 0xDEADBEEF + +#define WIP 1 /* development toggle */ + +/* Structure to collect the onion filepaths in one place. */ +struct onion_filepaths { + char *canon; + char *onion; + char *recovery; +}; + +struct expected_revision { + uint64_t revision_id; + uint64_t parent_revision_id; + uint64_t logi_eof; + uint64_t n_index_entries; + const char *comment; +}; +struct expected_history { + uint64_t page_size; + uint64_t n_revisions; + uint64_t origin_eof; + struct expected_revision revisions[ONION_TEST_EXPECTED_HISTORY_REVISIONS_MAX]; +}; + +struct write_info { + haddr_t offset; + haddr_t size; + const unsigned char *buf; +}; +struct revise_revision { + uint64_t magic; + hbool_t truncate; /* onion-create, truncating any existing data */ + uint64_t revision_id; + size_t n_writes; + struct write_info writes[ONION_TEST_REV_REV_WRITES_MAX]; + const char *comment; +}; + +static int compare_file_bytes_exactly(const char *, hid_t, size_t, + const unsigned char *); +static int do_onion_open_and_writes(const char *filename, + H5FD_onion_fapl_info_t *onion_info_p, size_t n_ops, + struct revise_revision *about); +static void onion_filepaths_destroy(struct onion_filepaths *); +static struct onion_filepaths *onion_filepaths_init(const char *, + H5FD_onion_fapl_info_t *); +#if 0 +static int is_onion_data_page_aligned( + const struct H5FD__onion_history_header *); +static uint32_t up_size_to_page_boundary(uint64_t, uint32_t); +#endif + + + +/* set at runtime in main() */ +static unsigned int flags_create_s = 0; + + +/* NOTE: b_list must be longer than a_list. + * Sizes must match respective buffer lengths. + */ + +/* twenty-six four-character words beginning with 'a' -> 104 bytes */ +const unsigned char *a_list_s = (const unsigned char *) \ + "abetableacedacesacheacidacneadzeafaragedagesaidsairsajarallyalum" \ + "amokantsapesarcsareaartsasksaspsavidaxes"; +uint64_t a_list_size_s = 104; + +/* fifty-three four-character words beginning with 'b' -> 212 bytes */ +const unsigned char *b_list_s = (const unsigned char *) \ + "badebailbaitbalebanebarebaskbeambeanbearbeenbeerbeltbentbestbide" \ + "bikebilebindbirdbiteblipblueboarboatbobsbodyboilboldbollboltbond" \ + "boneboobboorboosbootbradbragbratbraybrewbritbrowbuckbudsbunkbunt" \ + "buoyburnburybustbuys"; +uint64_t b_list_size_s = 212; + + +/* Allocate and populate filepaths with h5_fixname'd strings as appropriate. + * Should be released with onion_filepaths_destroy() when done. + */ +static struct onion_filepaths * +onion_filepaths_init(const char *basename, H5FD_onion_fapl_info_t *fa_info) +{ + struct onion_filepaths *paths = NULL; + + paths = (struct onion_filepaths *)HDmalloc(sizeof(struct onion_filepaths)); + if (NULL == paths) + TEST_ERROR; + paths->canon = NULL; + paths->onion = NULL; + paths->recovery = NULL; + + paths->canon = (char *)HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE); + if (NULL == paths->canon) + TEST_ERROR; + if (!h5_fixname_no_suffix(basename, fa_info->backing_fapl_id, \ + paths->canon, ONION_TEST_FIXNAME_SIZE)) + TEST_ERROR; + + paths->onion = (char *)HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE); + HDsnprintf(paths->onion, ONION_TEST_FIXNAME_SIZE, "%s.onion", \ + paths->canon); + + paths->recovery = (char *)HDmalloc(sizeof(char) * ONION_TEST_FIXNAME_SIZE); + HDsnprintf(paths->recovery, ONION_TEST_FIXNAME_SIZE, "%s.onion.recovery", \ + paths->canon); + + return paths; + +error: + if (paths != NULL) { + if (paths->canon != NULL) + HDfree(paths->canon); + if (paths->onion != NULL) + HDfree(paths->onion); + if (paths->recovery != NULL) + HDfree(paths->recovery); + } + return NULL; +} + +static void +onion_filepaths_destroy(struct onion_filepaths *s) +{ + HDfree(s->canon); + HDfree(s->onion); + HDfree(s->recovery); + HDfree(s); +} + + +#if 0 +static int +is_onion_data_page_aligned(const struct H5FD__onion_history_header *header) +{ + return header->flags & H5FD__ONION_HEADER_FLAG_PAGE_ALIGNMENT; +} + +static uint32_t +up_size_to_page_boundary(uint64_t size_in, uint32_t page_size) +{ + uint32_t _i = 0; /* number of pages occupied */ + for (_i = 1; (page_size * _i) < size_in; _i++); + return page_size * _i; +} +#endif + +/*----------------------------------------------------------------------------- + * + * Function: test_archival_index() + * + * Purpose: Unit-test mechanisms for the onion archival index. + * Specifies and verifies index-validation and -search routines. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_archival_index(void) +{ + /* We can ignore each entry's physical address and checksum values */ + struct H5FD__onion_index_entry e0 = { 1, 474}; + struct H5FD__onion_index_entry e1 = { 4, 558}; + struct H5FD__onion_index_entry e2 = { 5, 306}; + struct H5FD__onion_index_entry e3 = { 9, 515}; + struct H5FD__onion_index_entry e4 = {14, 386}; + struct H5FD__onion_index_entry e5 = {18, 90}; + struct H5FD__onion_index_entry e6 = {19, 94}; + struct H5FD__onion_index_entry e7 = {20, 509}; + struct H5FD__onion_index_entry empty[8]; + struct H5FD__onion_index_entry sorted[8] = { + e0, e1, e2, e3, e4, e5, e6, e7}; + struct H5FD__onion_index_entry sorted_duplicates[8] = { + e0, e1, e2, e2, e4, e5, e6, e7}; + struct H5FD__onion_index_entry sorted_incomplete[8] = { + e1, e3, e4, e5}; + /* partially-sorted list also aligned to 2 * page-size */ + struct H5FD__onion_index_entry sorted_partial[8] = { + e1, e4, e5, e7, e0, e6, e2, e3}; /* 0..3 sorted */ + struct H5FD__onion_index_entry unsorted[8] = { + e3, e1, e4, e5, e0, e6, e2, e7}; + struct H5FD__onion_archival_index aix = { + H5FD__ONION_ARCHIVAL_INDEX_MAGIC, + H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR, + 1, /* page_size_log2 */ + 8, /* list must be populated and sorted through 0 .. (count-1) */ + sorted, /* list */ + }; + const struct H5FD__onion_index_entry *entry_out_p = NULL; + + TESTING("archival index"); + + /* + * Failing validity checks + */ + + aix.magic++; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* invalid magic should fail */ + aix.magic--; + + aix.version++; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* invalid version should fail */ + aix.version = 0; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* invalid version should fail */ + aix.version = H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR; + + aix.list = NULL; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* list cannot be NULL */ + + aix.list = empty; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* list cannot be empty */ + + aix.list = sorted_incomplete; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* list must be full */ + + aix.list = unsorted; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* list must be sorted */ + + aix.list = sorted_duplicates; + if (H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* list cannot have duplicates */ + + /* + * Passing validity checks + */ + + aix.list = sorted; + if (!H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* everything in order should report valid */ + + aix.list = sorted_partial; + aix.n_entries = 4; + if (!H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* elements after n_entries are ignored */ + + /* + * Archival index search routine + */ + + aix.list = sorted; + aix.n_entries = 8; + + if (H5FD_onion_archival_index_find(&aix, 3, &entry_out_p) != 0) + TEST_ERROR; /* address not in array -> returns 0 */ + if (entry_out_p != NULL) + TEST_ERROR; /* pointer should remain unset */ + + if (H5FD_onion_archival_index_find(&aix, 4, &entry_out_p) != 1) + TEST_ERROR; /* address found -> should return 1 */ + if (NULL == entry_out_p) + TEST_ERROR; /* pointer should be set */ + if (558 != entry_out_p->phys_addr) + TEST_ERROR; /* incorrect address recorded */ + + + /* + * Test search edge cases + */ + + aix.list = sorted_incomplete; + aix.n_entries = 4; + + if (H5FD_onion_archival_index_find(&aix, 1, &entry_out_p) != 0) + TEST_ERROR; /* address not in array -> returns 0 */ + + if (H5FD_onion_archival_index_find(&aix, 101, &entry_out_p) != 0) + TEST_ERROR; /* address not in array -> returns 0 */ + + /* + * Empty archival index + */ + + entry_out_p = NULL; + aix.n_entries = 0; /* actually populated list is irrelevant */ + if (H5FD_onion_archival_index_find(&aix, 3, &entry_out_p) != 0) + TEST_ERROR; /* address not in array -> returns 0 */ + if (entry_out_p != NULL) + TEST_ERROR; /* pointer should remain unset */ + + PASSED(); + return 0; + +error: + return -1; +} /* end test_archival_index() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_revision_index() + * + * Purpose: TBD + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_revision_index(void) +{ + struct H5FD__onion_revision_index *rix_p = NULL; + struct H5FD__onion_index_entry entry = { + 42, /* logi_page */ + 111112, /* phys_addr */ + }; + const struct H5FD__onion_index_entry *entry_out_p = NULL; + + TESTING("revision index"); + + /* Test index creation + */ + + rix_p = H5FD_onion_revision_index_init(ONION_TEST_PAGE_SIZE_5); + if (NULL == rix_p) + TEST_ERROR; /* unable to initialize working index */ + if (H5FD__ONION_REVISION_INDEX_MAGIC != rix_p->magic) + TEST_ERROR; + if (H5FD__ONION_REVISION_INDEX_VERSION_CURR != rix_p->version) + TEST_ERROR; + if (0 != rix_p->n_entries) + TEST_ERROR; + + /* Test obviously failing inserts + */ + + if (H5FD_onion_revision_index_insert(rix_p, NULL) != FAIL) + TEST_ERROR; /* cannot be NULL */ + + /* Test missed search + */ + + if (H5FD_onion_revision_index_find(rix_p, entry.logi_page, &entry_out_p) \ + != 0) + TEST_ERROR; + + /* Test successful insertion and lookup + */ + + if (H5FD_onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; /* insertion failed */ + if (1 != rix_p->n_entries) + TEST_ERROR; + if (H5FD_onion_revision_index_find(rix_p, entry.logi_page, &entry_out_p) \ + < 0) + TEST_ERROR; /* lookup failed */ + if (NULL == entry_out_p) + TEST_ERROR; /* failure to set output parameter */ + if (entry.logi_page != entry_out_p->logi_page) + TEST_ERROR; + if (H5FD_onion_revision_index_find(rix_p, entry.logi_page + 1, \ + &entry_out_p) != 0) + TEST_ERROR; /* seeking other, absent page should miss */ + + /* Test / demonstrate stored entry independent of user object + */ + + entry.logi_page = 100; + entry.phys_addr = 101; + if (H5FD_onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + if (2 != rix_p->n_entries) + TEST_ERROR; + entry.logi_page = 500; + entry.phys_addr = 501; + if (H5FD_onion_revision_index_find(rix_p, 100, &entry_out_p) < 0) + TEST_ERROR; + if (100 != entry_out_p->logi_page || 101 != entry_out_p->phys_addr) + TEST_ERROR; + + /* Demonstrate updating an entry + */ + + /* Error cases */ + + entry.logi_page = 100; /* phys_addr still 501, checksum bbbbbbbb */ + if (H5FD_onion_revision_index_insert(rix_p, &entry) >= 0) + TEST_ERROR; /* all components but sum must match */ + entry.phys_addr = 101; + + /* Successful update */ + + entry.logi_page = 100; + entry.phys_addr = 101; + if (H5FD_onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + if (2 != rix_p->n_entries) + TEST_ERROR; /* Should still be two unique entries, not three */ + if (H5FD_onion_revision_index_find(rix_p, 100, &entry_out_p) < 0) + TEST_ERROR; + if (100 != entry_out_p->logi_page || 101 != entry_out_p->phys_addr) + TEST_ERROR; + + if (H5FD_onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + rix_p = NULL; + + PASSED(); + return 0; + +error: + if (rix_p != NULL) { + (void)H5FD_onion_revision_index_destroy(rix_p); + } + return -1; +} /* end test_revision_index() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_revision_index_collisions() + * + * Purpose: With knowledge of the revision index implementation, test + * hash key collisions. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_revision_index_collisions(void) +{ + H5FD__onion_revision_index_t *rix_p = NULL; + struct H5FD__onion_index_entry entry = { + 0, /* logi_page */ + 0, /* phys_addr */ + }; + const struct H5FD__onion_index_entry *entry_out_p = NULL; + uint64_t i = 0; + const uint64_t n_insert = 40; + const uint64_t offset_from_power = 5; + + TESTING("revision index collisions"); + + rix_p = H5FD_onion_revision_index_init(ONION_TEST_PAGE_SIZE_5); + if (NULL == rix_p) + TEST_ERROR; /* unable to initialize working index */ + + for (i = 0; i < n_insert; i++) { + entry.phys_addr = i; + entry.logi_page = U64_EXP2(i) + offset_from_power; + if (H5FD_onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + } + + if (n_insert != rix_p->n_entries) + TEST_ERROR; + + for (i = 0; i < n_insert; i++) { + uint64_t page_id = U64_EXP2(i) + offset_from_power; + + if (H5FD_onion_revision_index_find(rix_p, page_id, &entry_out_p) != 1) + TEST_ERROR; + if (entry_out_p->phys_addr != i) + TEST_ERROR; + } + + if (H5FD_onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + rix_p = NULL; + + PASSED(); + return 0; + +error: + if (rix_p != NULL) { + (void)H5FD_onion_revision_index_destroy(rix_p); + } + return -1; +} /* end test_revision_index_collisions() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_revision_index_resizing() + * + * Purpose: With knowledge of the revision index implementation, test + * one or more resizig of the index. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_revision_index_resizing(void) +{ + H5FD__onion_revision_index_t *rix_p = NULL; + struct H5FD__onion_index_entry entry = { + 0, /* logi_page */ + 0, /* phys_addr */ + }; + const struct H5FD__onion_index_entry *entry_out_p = NULL; + uint64_t i = 0; + const uint64_t n_insert = U64_EXP2( \ + (H5FD__ONION_REVISION_INDEX_STARTING_SIZE_LOG2 + 3)); + + TESTING("revision index resizing"); + + rix_p = H5FD_onion_revision_index_init(ONION_TEST_PAGE_SIZE_5); + if (NULL == rix_p) + TEST_ERROR; /* unable to initialize working index */ + + for (i = 0; i < n_insert; i++) { + entry.logi_page = i; + entry.phys_addr = ((uint64_t)(-1) - i); + if (H5FD_onion_revision_index_insert(rix_p, &entry) < 0) + TEST_ERROR; + } + + if (n_insert != rix_p->n_entries) + TEST_ERROR; + + for (i = 0; i < n_insert; i++) { + uint64_t page_id = i; + + if (H5FD_onion_revision_index_find(rix_p, page_id, &entry_out_p) != 1) + TEST_ERROR; + if (entry_out_p->phys_addr != ((uint64_t)(-1) - i)) + TEST_ERROR; + } + + if (H5FD_onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + rix_p = NULL; + + PASSED(); + return 0; + +error: + if (rix_p != NULL) { + (void)H5FD_onion_revision_index_destroy(rix_p); + } + return -1; +} /* end test_revision_index_resizing() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_revision_index_to_archival_index() + * + * Purpose: Verify to_archival_index(). + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_revision_index_to_archival_index(void) +{ + H5FD__onion_revision_index_t *rix_p = NULL; + struct H5FD__onion_index_entry rix_entry = { + 0, /* logi_page */ + 0, /* phys_addr */ + }; + struct H5FD__onion_archival_index aix = { + H5FD__ONION_ARCHIVAL_INDEX_MAGIC, + H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR, + 5, /* page_size_log2 */ + 0, /* n_entries to be set */ + NULL, + }; + struct H5FD__onion_index_entry *_list = NULL; + const uint64_t n_insert = 10; + uint64_t i = 0; + + TESTING("revision index to archival index"); + + /* + * SETUP + */ + + rix_p = H5FD_onion_revision_index_init(ONION_TEST_PAGE_SIZE_5); + if (NULL == rix_p) + TEST_ERROR; /* unable to initialize working index */ + + /* Add scattered entries in reverse order. */ + for (i = 0; i < n_insert; i++) { + uint64_t n = 2003 * (n_insert - i) + 47; + + rix_entry.logi_page = n; + rix_entry.phys_addr = n * 13; + if (H5FD_onion_revision_index_insert(rix_p, &rix_entry) < 0) + TEST_ERROR; + } + + if (n_insert != rix_p->n_entries) + TEST_ERROR; + + aix.list = (struct H5FD__onion_index_entry *)HDmalloc(0); + if (NULL == aix.list) + TEST_ERROR; + + aix.n_entries = 0; + + /* + * FAILING CASES + */ + + if (H5FD_onion_merge_revision_index_into_archival_index(NULL, NULL) \ + != FAIL) + TEST_ERROR; /* both cannot be null */ + + if (H5FD_onion_merge_revision_index_into_archival_index(NULL, &aix) \ + != FAIL) + TEST_ERROR; /* revision index cannot be null */ + + rix_p->magic++; + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) \ + != FAIL) + TEST_ERROR; /* revision index magic must be valid */ + rix_p->magic--; + + rix_p->version++; + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) \ + != FAIL) + TEST_ERROR; /* revision index version must be valid */ + rix_p->version--; + + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, NULL) \ + != FAIL) + TEST_ERROR; /* archival index cannot be null */ + + aix.magic++; + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) \ + != FAIL) + TEST_ERROR; /* archival index magic must be valid */ + aix.magic--; + + aix.version++; + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) \ + != FAIL) + TEST_ERROR; /* archival index version must be valid */ + aix.version--; + + _list = aix.list; + aix.list = NULL; + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) \ + != FAIL) + TEST_ERROR; /* list pointer must exist */ + aix.list = _list; + _list = NULL; + + aix.page_size_log2 += 1; + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) \ + != FAIL) + TEST_ERROR; /* page sizes must match */ + aix.page_size_log2 -= 1; + + /* Successful merge into empty archival index + */ + + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0) + TEST_ERROR; + + if (!H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* entries not sorted, or other obscure issue */ + + if (n_insert != aix.n_entries) + TEST_ERROR; /* failed to resize and/or update archival index info */ + + for (i = 0; i < n_insert; i++) { + const struct H5FD__onion_index_entry *aix_entry_p = NULL; + uint64_t n = 2003 * (i + 1) + 47; + + aix_entry_p = &aix.list[i]; + + if (aix_entry_p->logi_page != n) + TEST_ERROR; + if (aix_entry_p->phys_addr != (n * 13)) + TEST_ERROR; + } + + /* Successful merge into populated archival index + */ + + HDfree(aix.list); + aix.list = NULL; + aix.list = (struct H5FD__onion_index_entry *)HDmalloc( + sizeof(struct H5FD__onion_index_entry) * 2); + if (NULL == aix.list) + TEST_ERROR; + aix.list[0].logi_page = 47; + aix.list[0].phys_addr = 47 * 13; + aix.list[1].logi_page = (2003 * (n_insert + 1) + 47); + aix.list[1].phys_addr = (2003 * (n_insert + 1) + 47) * 13; + aix.n_entries = 2; + + if (!H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* entries not sorted, or other obscure issue */ + + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0) + TEST_ERROR; + + if (!H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* entries not sorted, or other obscure issue */ + + if (n_insert + 2 != aix.n_entries) + TEST_ERROR; + + for (i = 0; i < (n_insert + 2); i++) { + const struct H5FD__onion_index_entry *aix_entry_p = NULL; + uint64_t n = 2003 * i + 47; + + aix_entry_p = &aix.list[i]; + + if (aix_entry_p->logi_page != n) + TEST_ERROR; + if (aix_entry_p->phys_addr != (n * 13)) + TEST_ERROR; + } + + /* Merged enties from revision index replace existing entries + */ + + HDfree(aix.list); + aix.list = NULL; + aix.list = (struct H5FD__onion_index_entry *)HDmalloc( + sizeof(struct H5FD__onion_index_entry) * 2); + if (NULL == aix.list) + TEST_ERROR; + aix.list[0].logi_page = 2003 * (n_insert / 2) + 47; + aix.list[0].phys_addr = 103; + aix.list[1].logi_page = 2003 * (n_insert / 2 + 1) + 47; + aix.list[1].phys_addr = 101; + aix.n_entries = 2; + + if (!H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* entries not sorted, or other obscure issue */ + + if (H5FD_onion_merge_revision_index_into_archival_index(rix_p, &aix) < 0) + TEST_ERROR; + + if (!H5FD_onion_archival_index_is_valid(&aix)) + TEST_ERROR; /* entries not sorted, or other obscure issue */ + + if (n_insert != aix.n_entries) + TEST_ERROR; + + for (i = 0; i < n_insert; i++) { + const struct H5FD__onion_index_entry *aix_entry_p = NULL; + uint64_t n = 2003 * (i + 1) + 47; + + aix_entry_p = &aix.list[i]; + + if (aix_entry_p->logi_page != n) + TEST_ERROR; + if (aix_entry_p->phys_addr != (n * 13)) + TEST_ERROR; + } + + /* + * CLEANUP + */ + + if (H5FD_onion_revision_index_destroy(rix_p) < 0) + TEST_ERROR; + rix_p = NULL; + + HDfree(aix.list); + aix.list = NULL; + aix.magic++; + + PASSED(); + return 0; + +error: + if (rix_p != NULL) { + (void)H5FD_onion_revision_index_destroy(rix_p); + } + if (aix.list != NULL) { + HDfree(aix.list); + } + return -1; +} /* end test_revision_index_to_archival_index() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_fapl() + * + * Purpose: Verify H5Pget and set behavior, and data-consistency checks. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_fapl(void) +{ + H5FD_onion_fapl_info_t info_in = { + H5FD_ONION_FAPL_INFO_MAGIC, + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_1, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "indoor speaking voices", /* comment */ + }; + H5FD_onion_fapl_info_t info_out; + hid_t dxpl_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t fapl_id_sec2 = H5I_INVALID_HID; + herr_t ret = FAIL; + + TESTING("file access property list"); + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + if (H5I_INVALID_HID == dxpl_id) + TEST_ERROR; + + fapl_id_sec2 = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id_sec2) + TEST_ERROR; + if (H5Pset_fapl_sec2(fapl_id_sec2)) + TEST_ERROR; + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + + /* + * Set FAPL + */ + + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(H5I_INVALID_HID, &info_in); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* fapl_id must be valid */ + + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(fapl_id, NULL); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* info pointer cannot be NULL */ + + info_in.magic++; + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* info magic must be valid */ + info_in.magic--; + + info_in.version++; + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* info version must be valid */ + info_in.version--; + + info_in.page_size = 7; + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* page size must be valid power of 2 */ + + info_in.page_size = 0; + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* page size must be greater than zero */ + info_in.page_size = ONION_TEST_PAGE_SIZE_1; + + info_in.backing_fapl_id = H5I_INVALID_HID; + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* backing fapl ID cannot be invalid */ + + info_in.backing_fapl_id = dxpl_id; + H5E_BEGIN_TRY { + ret = H5Pset_fapl_onion(fapl_id, &info_in); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* backing fapl ID must be file access proptery list ID */ + info_in.backing_fapl_id = H5P_DEFAULT; + + + if (H5Pset_fapl_onion(fapl_id, &info_in) < 0) + TEST_ERROR; + + /* + * Get onion fapl info + */ + + H5E_BEGIN_TRY { + ret = H5Pget_fapl_onion(fapl_id, NULL); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* info_out pointer cannot be NULL */ + + H5E_BEGIN_TRY { + ret = H5Pget_fapl_onion(H5I_INVALID_HID, &info_out); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* fapl_id must be valid */ + + H5E_BEGIN_TRY { + ret = H5Pget_fapl_onion(fapl_id_sec2, &info_out); + } H5E_END_TRY; + if (SUCCEED == ret) + TEST_ERROR; /* wrong fapl_id in */ + + if (H5Pget_fapl_onion(fapl_id, &info_out) < 0) + TEST_ERROR; + if (H5FD_ONION_FAPL_INFO_MAGIC != info_out.magic) + TEST_ERROR; + if (H5FD_ONION_FAPL_INFO_VERSION_CURR != info_out.version) + TEST_ERROR; + if (H5P_DEFAULT != info_out.backing_fapl_id) + TEST_ERROR; + if (ONION_TEST_PAGE_SIZE_1 != info_out.page_size) + TEST_ERROR; + if (H5FD_ONION_STORE_TARGET_ONION != info_out.store_target) + TEST_ERROR; + if (H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST != info_out.revision_id) + TEST_ERROR; + if (0 != info_out.creation_flags) + TEST_ERROR; + if (0 != info_out.force_write_open) + TEST_ERROR; + if (HDstrcmp(info_in.comment, info_out.comment)) + TEST_ERROR; + + /* + * Cleanup + */ + + if (H5Pclose(dxpl_id) < 0) + TEST_ERROR; + dxpl_id = H5I_INVALID_HID; + + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + if (H5Pclose(fapl_id_sec2) < 0) + TEST_ERROR; + fapl_id_sec2 = H5I_INVALID_HID; + + PASSED(); + return 0; + +error: + if (H5I_INVALID_HID != dxpl_id) + (void)H5Pclose(dxpl_id); + if (H5I_INVALID_HID != fapl_id) + (void)H5Pclose(fapl_id); + if (H5I_INVALID_HID != fapl_id_sec2) + (void)H5Pclose(fapl_id_sec2); + + return -1; +} /* end test_fapl() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_header_encode_decode() + * + * Purpose: Verify onion header encoding and decoding behavior. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_header_encode_decode(void) +{ + unsigned char buf[64]; + unsigned char exp[64] = { /* bogus but unique values */ + 'O', 'H', 'D', 'H', /* NOTE: match signature define in onion_priv.h */ + 1, 12, 0, 0, /* NOTE: update version w/ "current" as needed */ + 0, 16, 0, 0, + 0x11, 0x00, 0, 0, 0x02, 0, 0, 0, /* origin_eof */ + 0x40, 0xe2, 0x01, 0, 0, 0, 0, 0, /* whole_history_addr */ + 88, 0, 0, 0, 0, 0, 0, 0, /* whole_history_size */ + 0, 0, 0, 0 /* sum populated below */ + }; + unsigned char *ptr = NULL; + uint32_t sum = 0; + uint32_t sum_out = 0; + size_t i = 0; + uint64_t size_ret = 0; + struct H5FD__onion_history_header hdr; + struct H5FD__onion_history_header hdr_out; + + TESTING("encode/decode history header"); + + sum = H5_checksum_fletcher32(exp, H5FD__ONION_ENCODED_SIZE_HEADER - 4); + ptr = exp + H5FD__ONION_ENCODED_SIZE_HEADER - 4; + UINT32ENCODE(ptr, sum); + + hdr.magic = H5FD__ONION_HEADER_MAGIC; + hdr.version = H5FD__ONION_HEADER_VERSION_CURR; + hdr.flags = 12; + hdr.origin_eof = 8589934609ull, + hdr.page_size = 4096; + hdr.whole_history_addr = 123456; + hdr.whole_history_size = 88; + + if (H5FD_onion_history_header_encode(&hdr, buf, &sum_out) \ + != H5FD__ONION_ENCODED_SIZE_HEADER) + TEST_ERROR; + + if (sum != sum_out) + TEST_ERROR; + + for (i = 0; i < H5FD__ONION_ENCODED_SIZE_HEADER; i++) { + if (exp[i] != buf[i]) { + HDprintf("first mismatched byte at %llu: %02x %02x\n", i, exp[i], \ + buf[i]); + TEST_ERROR; + } + } + + hdr_out.magic = H5FD__ONION_HEADER_MAGIC; + hdr_out.version = H5FD__ONION_HEADER_VERSION_CURR; + hdr_out.flags = 0; + hdr_out.page_size = 0; + hdr_out.whole_history_addr = 0; + hdr_out.whole_history_size = 0; + + /* Invalid header signature prevents decoding. + */ + + exp[3] = 'X'; /* invalidate encoded signature */ + H5E_BEGIN_TRY { + size_ret = H5FD_onion_history_header_decode(exp, &hdr_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[3] = 'H'; /* reset */ + + /* Invalid header version prevents decoding. + */ + + exp[4] = 0; /* encoded version 0?!? */ + H5E_BEGIN_TRY { + size_ret = H5FD_onion_history_header_decode(exp, &hdr_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[4] = H5FD__ONION_HEADER_VERSION_CURR + 1; /* encoded super-version?! */ + H5E_BEGIN_TRY { + size_ret = H5FD_onion_history_header_decode(exp, &hdr_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[4] = H5FD__ONION_HEADER_VERSION_CURR; /* reset */ + + /* Valid header can be decoded. + */ + + if (H5FD_onion_history_header_decode(buf, &hdr_out) \ + != H5FD__ONION_ENCODED_SIZE_HEADER) + TEST_ERROR; + if (H5FD__ONION_HEADER_MAGIC != hdr_out.magic) + TEST_ERROR; + if (H5FD__ONION_HEADER_VERSION_CURR != hdr_out.version) + TEST_ERROR; + if (hdr.flags != hdr_out.flags) + TEST_ERROR; + if (hdr.page_size != hdr_out.page_size) + TEST_ERROR; + if (hdr.whole_history_addr != hdr_out.whole_history_addr) + TEST_ERROR; + if (hdr.whole_history_size != hdr_out.whole_history_size) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return -1; +} /* end test_header_encode_decode() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_whole_history_encode_decode_empty() + * + * Purpose: Verify onion whole-history encoding and decoding behavior. + * Tests the case of the "empty" whole-history. + * Verifies behavior in standard error cases. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_whole_history_encode_decode_empty(void) +{ + unsigned char buf[32]; + unsigned char exp[32] = { + 'O', 'W', 'H', 'S', /* NOTE: match signature define in onion_priv.h */ + 1, 0, 0, 0, /* NOTE: update version w/ "current" as needed */ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 /* sum populated below */ + }; + unsigned char *ptr = NULL; + uint32_t sum = 0; + uint32_t sum_out = 0; + size_t i = 0; + uint64_t size_ret = 0; + struct H5FD__onion_whole_history whs = { + H5FD__ONION_WHOLE_HISTORY_MAGIC, + H5FD__ONION_WHOLE_HISTORY_VERSION_CURR, + 0, /* n_revisions */ + NULL, /* list */ + 0, /* checksum */ + }; + struct H5FD__onion_whole_history whs_out = { + H5FD__ONION_WHOLE_HISTORY_MAGIC, + H5FD__ONION_WHOLE_HISTORY_VERSION_CURR, + 0, /* n_revisions */ + NULL, /* list */ + 0, /* checksum */ + }; + + TESTING("encode/decode whole-history (empty and failures)"); + + /* Generage checksum but don't store it yet */ + sum = H5_checksum_fletcher32(exp, \ + H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY - 4); + ptr = exp + H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY - 4; + UINT32ENCODE(ptr, sum); + + if (H5FD_onion_whole_history_encode(&whs, buf, &sum_out) \ + != H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY) + TEST_ERROR; + for (i = 0; i < 20; i++) { + if (exp[i] != buf[i]) { + HDprintf("first mismatched byte at %llu: %02x %02x\n", i, exp[i], \ + buf[i]); + TEST_ERROR; + } + } + if (sum != sum_out) + TEST_ERROR; + whs.checksum = sum; /* set to compare later */ + + /* Invalid signature prevents decoding. + */ + + exp[3] = 'X'; /* invalidate encoded signature */ + H5E_BEGIN_TRY { + size_ret = H5FD_onion_whole_history_decode(exp, &whs_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[3] = 'H'; /* reset */ + + /* Invalid version prevents decoding. + */ + + exp[4] = 0; /* encoded version 0?!? */ + H5E_BEGIN_TRY { + size_ret = H5FD_onion_whole_history_decode(exp, &whs_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[4] = H5FD__ONION_WHOLE_HISTORY_VERSION_CURR + 1; + H5E_BEGIN_TRY { + size_ret = H5FD_onion_whole_history_decode(exp, &whs_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + exp[4] = H5FD__ONION_WHOLE_HISTORY_VERSION_CURR; /* reset */ + + /* Valid summary can be decoded. + */ + + if (H5FD_onion_whole_history_decode(buf, &whs_out) \ + != H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY) + TEST_ERROR; + if (H5FD__ONION_WHOLE_HISTORY_MAGIC != whs_out.magic) + TEST_ERROR; + if (H5FD__ONION_WHOLE_HISTORY_VERSION_CURR != whs_out.version) + TEST_ERROR; + if (whs.n_revisions != whs_out.n_revisions) + TEST_ERROR; + if (whs.checksum != whs_out.checksum) + TEST_ERROR; + if (NULL != whs_out.record_pointer_list) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return -1; +} /* end test_whole_history_encode_decode_empty() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_whole_history_encode_decode() + * + * Purpose: Verify onion whole-history encoding and decoding behavior. + * Encode/decode with some set of revision record pointers. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_whole_history_encode_decode(void) +{ + unsigned char *buf = NULL; + unsigned char exp[80] = { + 'O', 'W', 'H', 'S', /* NOTE: match signature define in onion_priv.h */ + 1, 0, 0, 0, /* NOTE: update version w/ "current" as needed */ + 3, 0, 0, 0, 0, 0, 0, 0, + /* rev0 pointer */ + 56, 2, 0, 0, 0, 0, 0, 0, + 238, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* sum populated below */ + /* rev1 pointer */ + 121, 173, 3, 0, 0, 0, 0, 0, + 203, 17, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* sum populated below */ + /* rev2 pointer */ + 96, 158, 52, 198, 213, 0, 0, 0, + 240, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* sum populated below */ + /* final checksum */ + 0, 0, 0, 0 /* sum populated below */ + }; + unsigned char *buf_p = NULL; + uint32_t sum_out = 0; + size_t i = 0; + struct H5FD__onion_whole_history whs = { + H5FD__ONION_WHOLE_HISTORY_MAGIC, + H5FD__ONION_WHOLE_HISTORY_VERSION_CURR, + 3, /* n_revisions */ + NULL, /* list set below */ + 0, /* checksum not set by us */ + }; + struct H5FD__onion_whole_history whs_out = { + H5FD__ONION_WHOLE_HISTORY_MAGIC, + H5FD__ONION_WHOLE_HISTORY_VERSION_CURR, + 0, /* n_revisions must start as zero */ + NULL, /* list */ + 0, /* checksum */ + }; + uint64_t exp_size = H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY + \ + H5FD__ONION_ENCODED_SIZE_RECORD_POINTER * whs.n_revisions; + + TESTING("encode/decode whole-history"); + + if (80 != exp_size) + TEST_ERROR; + + whs.record_pointer_list = (struct H5FD__onion_record_pointer *)HDcalloc( \ + whs.n_revisions, sizeof(struct H5FD__onion_record_pointer)); + if (NULL == whs.record_pointer_list) + TEST_ERROR; + + /* must match values in exp */ + whs.record_pointer_list[0].phys_addr = 568ull; + whs.record_pointer_list[0].record_size = 238ull; + whs.record_pointer_list[1].phys_addr = 241017ull; + whs.record_pointer_list[1].record_size = 4555ull; + whs.record_pointer_list[2].phys_addr = 918153371232ull; + whs.record_pointer_list[2].record_size = 240ull; + + /* populate revision pointer sums in exp */ + for (i = 0; i < whs.n_revisions; i++) { + uint64_t whs_pre = H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY - 4; + uint64_t ptr_pre = H5FD__ONION_ENCODED_SIZE_RECORD_POINTER - 4; + uint64_t ptr_size = H5FD__ONION_ENCODED_SIZE_RECORD_POINTER; + + buf_p = exp + whs_pre + ptr_size * i; + whs.record_pointer_list[i].checksum = H5_checksum_fletcher32(buf_p, \ + ptr_pre); + buf_p += ptr_pre; + UINT32ENCODE(buf_p, whs.record_pointer_list[i].checksum); + } + + /* compute, populate, and store exp final sum */ + whs.checksum = H5_checksum_fletcher32(exp, exp_size - 4); + buf_p = exp + exp_size - 4; + UINT32ENCODE(buf_p, whs.checksum); + + buf = (unsigned char *)HDmalloc(exp_size); + if (NULL == buf) + TEST_ERROR; + + if (H5FD_onion_whole_history_encode(&whs, buf, &sum_out) != exp_size) + TEST_ERROR; + for (i = 0; i < exp_size; i++) { + if (exp[i] != buf[i]) + TEST_ERROR; + } + if (whs.checksum != sum_out) + TEST_ERROR; + + /* Initial decode, gets always-present components. + */ + + whs_out.n_revisions = 0; /* must be initialized to 0 */ + if (H5FD_onion_whole_history_decode(exp, &whs_out) != exp_size) + TEST_ERROR; + if (H5FD__ONION_WHOLE_HISTORY_MAGIC != whs_out.magic) + TEST_ERROR; + if (H5FD__ONION_WHOLE_HISTORY_VERSION_CURR != whs_out.version) + TEST_ERROR; + if (whs.n_revisions != whs_out.n_revisions) + TEST_ERROR; + if (NULL != whs_out.record_pointer_list) + TEST_ERROR; /* Must be created by us */ + + /* True decode requires allocating space for record pointers + */ + + whs_out.record_pointer_list = (struct H5FD__onion_record_pointer *) \ + HDcalloc(whs_out.n_revisions, \ + sizeof(struct H5FD__onion_record_pointer)); + if (NULL == whs_out.record_pointer_list) + TEST_ERROR; + + if (H5FD_onion_whole_history_decode(exp, &whs_out) != exp_size) + TEST_ERROR; + if (H5FD__ONION_WHOLE_HISTORY_MAGIC != whs_out.magic) + TEST_ERROR; + if (H5FD__ONION_WHOLE_HISTORY_VERSION_CURR != whs_out.version) + TEST_ERROR; + if (whs.n_revisions != whs_out.n_revisions) + TEST_ERROR; + if (whs.checksum != whs_out.checksum) + TEST_ERROR; + if (NULL == whs_out.record_pointer_list) + TEST_ERROR; + for (i = 0; i < whs.n_revisions; i++) { + struct H5FD__onion_record_pointer exp_rp = whs.record_pointer_list[i]; + struct H5FD__onion_record_pointer act_rp = \ + whs_out.record_pointer_list[i]; + + if (exp_rp.phys_addr != act_rp.phys_addr) + TEST_ERROR; + if (exp_rp.record_size != act_rp.record_size) + TEST_ERROR; + if (exp_rp.checksum != act_rp.checksum) + TEST_ERROR; + } + + HDfree(whs_out.record_pointer_list); + whs_out.record_pointer_list = NULL; + + HDfree(buf); + buf = NULL; + + HDfree(whs.record_pointer_list); + whs.record_pointer_list = NULL; + + PASSED(); + return 0; + +error: + if (whs_out.record_pointer_list != NULL) + HDfree(whs_out.record_pointer_list); + if (buf != NULL) + HDfree(buf); + if (whs.record_pointer_list != NULL) + HDfree(whs.record_pointer_list); + return -1; +} /* end test_whole_history_encode_decode() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_revision_record_encode_decode() + * + * Purpose: Verify onion revision-record encoding and decoding behavior. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_revision_record_encode_decode(void) +{ + unsigned char *buf = NULL; + unsigned char exp[189] = { + 'O', 'R', 'R', 'S', /* signature */ + 1, 0, 0, 0, /* NOTE: update version w/ "current" as needed */ + 5, 0, 0, 0, 0, 0, 0, 0, /* revision ID */ + 2, 0, 0, 0, 0, 0, 0, 0, /* parent revision ID */ + '1', '9', '4', '1', '1', '2', '0', '7', /* Time of Creation */ + 'T', '1', '9', '0', '6', '4', '3', 'Z', /* ... */ + 0x11, 0x00, 0, 0, 0x02, 0, 0, 0, /* logical file size */ + 0, 16, 0, 0, /* page size */ + 143, 25, 0, 0, /* user ID */ + 4, 0, 0, 0, 0, 0, 0, 0, /* n_entries */ + 8, 0, 0, 0, /* username size */ + 25, 0, 0, 0, /* comment size */ + /* entry0 pointer */ + 0, 0xB0, 0x1E, 0, 0, 0, 0, 0, /* logical offset */ + 0x4B, 0x02, 0, 0, 0, 0, 0, 0, /* physical address */ + 0, 0, 0, 0, /* sum populated below */ /* checksum */ + /* entry1 pointer */ + 0, 0xF0, 0x2E, 0, 0, 0, 0, 0, + 0xA7, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* sum populated below */ + /* entry2 pointer */ + 0, 0x50, 0x15, 0, 0, 0x20, 0, 0, + 0x11, 0, 0, 0, 0x02, 0, 0, 0, + 0, 0, 0, 0, /* sum populated below */ + /* entry3 pointer */ + 0, 0xE0, 0x24, 0, 0, 0, 0, 0, + 0xB1, 0x01, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* sum populated below */ + 'J', 'o', 'h', 'n', 'D', 'o', 'e', '\0',/* username */ + 'E', 'x', 'a', 'm', 'p', 'l', 'e', ' ', /* comment */ + 'c', 'o', 'm', 'm', 'e', 'n', 't', ' ', /* ... */ + 'm', 'e', 's', 's', 'a', 'g', 'e', '.', /* ... */ + '\0', /* ... */ + /* final checksum */ + 0, 0, 0, 0 /* sum populated below */ /* checksum */ + }; + unsigned char *buf_p = NULL; + size_t i = 0; + uint64_t size_ret; + struct H5FD__onion_revision_record r_out; + uint32_t sum_out = 0; + struct H5FD__onion_revision_record record = { + H5FD__ONION_REVISION_RECORD_MAGIC, + H5FD__ONION_REVISION_RECORD_VERSION_CURR, + 5, /* revision ID */ + 2, /* parent revision ID */ + {'\0'}, /* time of creation - populated below */ + 8589934609ull, /* logical file size */ + 6543, /* user ID */ + 8, /* username size */ + 25, /* comment size */ + { H5FD__ONION_ARCHIVAL_INDEX_MAGIC, + H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR, + 12, /* page_size_log2 */ + 4, /* n_entries */ + NULL, /* list - populated below */ + }, /* archival index struct */ + (char *)"JohnDoe", /* username */ /* cast OK --JOS */ + (char *)"Example comment message.", /* comment */ /* cast OK --JOS */ + 0, /* checksum - computed for us */ + }; + uint64_t exp_size = H5FD__ONION_ENCODED_SIZE_REVISION_RECORD \ + + (H5FD__ONION_ENCODED_SIZE_INDEX_ENTRY \ + * record.archival_index.n_entries) \ + + strlen("JohnDoe") + 1 \ + + strlen("Example comment message.") + 1; + + r_out.archival_index.list = NULL; + r_out.comment = NULL; + r_out.username = NULL; + + + TESTING("encode/decode revision record"); + + if (189 != exp_size) + TEST_ERROR; + + HDmemcpy(record.time_of_creation, "19411207T190643Z", 16); + record.archival_index.list = (struct H5FD__onion_index_entry *)HDcalloc( \ + record.archival_index.n_entries, \ + sizeof(struct H5FD__onion_index_entry)); + if (NULL == record.archival_index.list) + TEST_ERROR; + /* convert logi_page and should match address in expected buffer */ + record.archival_index.list[0].logi_page = 491ull; + record.archival_index.list[0].phys_addr = 587ull; + record.archival_index.list[1].logi_page = 751ull; + record.archival_index.list[1].phys_addr = 167ull; + record.archival_index.list[2].logi_page = 8589934933ull; + record.archival_index.list[2].phys_addr = 8589934609ull; + record.archival_index.list[3].logi_page = 590ull; + record.archival_index.list[3].phys_addr = 433ull; + + /* Set expected checksum for each archival index entry in buffer */ + for (i = 0; i < record.archival_index.n_entries; i ++) { + uint64_t rec_pre = H5FD__ONION_ENCODED_SIZE_REVISION_RECORD - 4; + uint64_t idx_pre = H5FD__ONION_ENCODED_SIZE_INDEX_ENTRY - 4; + uint64_t idx_size = H5FD__ONION_ENCODED_SIZE_INDEX_ENTRY; + + buf_p = exp + rec_pre + idx_size * i; + sum_out = H5_checksum_fletcher32(buf_p, idx_pre); + buf_p += idx_pre; + UINT32ENCODE(buf_p, sum_out); + } + + sum_out = 0; + + record.checksum = H5_checksum_fletcher32(exp, exp_size - 4); + buf_p = exp + exp_size - 4; + UINT32ENCODE(buf_p, record.checksum); + + /* required initialization for record-out structure */ + r_out.magic = H5FD__ONION_REVISION_RECORD_MAGIC; + r_out.version = H5FD__ONION_REVISION_RECORD_VERSION_CURR; + r_out.username_size = 0; + r_out.comment_size = 0; + r_out.username = NULL; + r_out.comment = NULL; + r_out.archival_index.magic = H5FD__ONION_ARCHIVAL_INDEX_MAGIC; + r_out.archival_index.version = H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR; + r_out.archival_index.n_entries = 0; + r_out.archival_index.list = NULL; + + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * exp_size); + if (NULL == buf) + TEST_ERROR; + + /* Test encode + */ + + if (H5FD_onion_revision_record_encode(&record, buf, &sum_out) != exp_size) + TEST_ERROR; + for (i = 0; i < exp_size; i++) { + if (exp[i] != buf[i]) + TEST_ERROR; + } + if (record.checksum != sum_out) + TEST_ERROR; + + HDfree(buf); + buf = NULL; + + /* Test decode (malformed encoding) + */ + + /* invalid signature */ + exp[2] = 'Y'; + H5E_BEGIN_TRY { + size_ret = H5FD_onion_revision_record_decode(exp, &r_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + exp[2] = 'R'; /* reset */ + + /* zero version */ + exp[4] = 0; + H5E_BEGIN_TRY { + size_ret = H5FD_onion_revision_record_decode(exp, &r_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + + /* advance version */ + exp[4] = H5FD__ONION_REVISION_RECORD_VERSION_CURR + 1; + H5E_BEGIN_TRY { + size_ret = H5FD_onion_revision_record_decode(exp, &r_out); + } H5E_END_TRY; + if (0 != size_ret) + TEST_ERROR; + exp[4] = H5FD__ONION_REVISION_RECORD_VERSION_CURR; /* reset */ + + /* Test successful decode + */ + + /* Initial decode; get variable-length component sizes */ + if (H5FD_onion_revision_record_decode(exp, &r_out) != exp_size) + TEST_ERROR; + if (record.username_size != r_out.username_size) + TEST_ERROR; + if (record.comment_size != r_out.comment_size) + TEST_ERROR; + if (record.archival_index.n_entries != r_out.archival_index.n_entries) + TEST_ERROR; + + /* Allocate variable-length components */ + r_out.username = (char *)HDcalloc(r_out.username_size, sizeof(char)); + if (NULL == r_out.username) + TEST_ERROR; + r_out.comment = (char *)HDcalloc(r_out.comment_size, sizeof(char)); + if (NULL == r_out.comment) + TEST_ERROR; + r_out.archival_index.list = (struct H5FD__onion_index_entry *)HDcalloc( \ + r_out.archival_index.n_entries, \ + sizeof(struct H5FD__onion_index_entry)); + if (NULL == r_out.archival_index.list) + TEST_ERROR; + + /* Decode into all components */ + if (H5FD_onion_revision_record_decode(exp, &r_out) != exp_size) + TEST_ERROR; + if (H5FD__ONION_REVISION_RECORD_MAGIC != r_out.magic) + TEST_ERROR; + if (H5FD__ONION_REVISION_RECORD_VERSION_CURR != r_out.version) + TEST_ERROR; + if (record.user_id != r_out.user_id) + TEST_ERROR; + if (record.revision_id != r_out.revision_id) + TEST_ERROR; + if (record.parent_revision_id != r_out.parent_revision_id) + TEST_ERROR; + if (record.parent_revision_id != r_out.parent_revision_id) + TEST_ERROR; + if (record.checksum != r_out.checksum) + TEST_ERROR; + if (HDstrncmp(record.time_of_creation, r_out.time_of_creation, 16) != 0) + TEST_ERROR; + if (record.username_size != r_out.username_size) + TEST_ERROR; + if (record.username_size != HDstrlen(r_out.username) + 1) + TEST_ERROR; + if (HDstrlen(record.username) != HDstrlen(r_out.username)) + TEST_ERROR; + if (HDstrcmp(record.username, r_out.username) != 0) + TEST_ERROR; + if (record.comment_size != r_out.comment_size) + TEST_ERROR; + if (record.comment_size != HDstrlen(r_out.comment) + 1) + TEST_ERROR; + if (HDstrlen(record.comment) != HDstrlen(r_out.comment)) + TEST_ERROR; + if (HDstrcmp(record.comment, r_out.comment) != 0) + TEST_ERROR; + + if (H5FD__ONION_ARCHIVAL_INDEX_MAGIC != r_out.archival_index.magic) + TEST_ERROR; + if (H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR != \ + r_out.archival_index.version) + TEST_ERROR; + if (record.archival_index.page_size_log2 != \ + r_out.archival_index.page_size_log2) + TEST_ERROR; + if (record.archival_index.n_entries != r_out.archival_index.n_entries) + TEST_ERROR; + for (i = 0; i < record.archival_index.n_entries; i++) { + struct H5FD__onion_index_entry *ep = &record.archival_index.list[i]; + struct H5FD__onion_index_entry *ap = &r_out.archival_index.list[i]; + + if (ep->phys_addr != ap->phys_addr) + TEST_ERROR; + if (ep->logi_page != ap->logi_page) + TEST_ERROR; + } + + /* Cleanup + */ + + HDfree(r_out.archival_index.list); + r_out.archival_index.list = NULL; + + HDfree(r_out.comment); + r_out.comment = NULL; + + HDfree(r_out.username); + r_out.username = NULL; + + HDfree(record.archival_index.list); + record.archival_index.list = NULL; + + PASSED(); + return 0; + +error: + if (r_out.archival_index.list != NULL) + HDfree(r_out.archival_index.list); + if (r_out.comment != NULL) + HDfree(r_out.comment); + if (r_out.username != NULL) + HDfree(r_out.username); + if (buf != NULL) + HDfree(buf); + if (record.archival_index.list != NULL) + HDfree(record.archival_index.list); + return -1; +} /* end test_revision_record_encode_decode() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * Use VFL to open target file and check that its bytes exactly match those + * of given buffer 'exp'[ected]. + * + * Returns 0 if successful, -1 if error or mismatch. + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static int +compare_file_bytes_exactly(const char *filepath, hid_t fapl_id, size_t nbytes, + const unsigned char *exp) +{ + H5FD_t *raw_vfile = NULL; /* virtual file to look at raw file contents */ + unsigned char *act_buf = NULL; /* allocated area for actual file bytes */ + size_t i = 0; + uint64_t filesize = 0; + + raw_vfile = H5FDopen(filepath, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == raw_vfile) + TEST_ERROR; + + filesize = (uint64_t)H5FDget_eof(raw_vfile, H5FD_MEM_DRAW); + if ((uint64_t)nbytes != filesize) + TEST_ERROR; + + act_buf = (unsigned char *)HDmalloc(nbytes); + if (NULL == act_buf) + TEST_ERROR; + for (i = 0; i < nbytes; i++) + act_buf[i] = (unsigned char)(-1); /* fill with bogus all-1s */ + if (H5FDset_eoa(raw_vfile, H5FD_MEM_DRAW, nbytes) < 0) + TEST_ERROR; + if (H5FDread(raw_vfile, H5FD_MEM_DRAW, H5P_DEFAULT, 0, nbytes, act_buf) + < 0) + TEST_ERROR; + + /* compare raw bytes data */ + for (i = 0; i < nbytes; i++) { + if (exp[i] != act_buf[i]) { + HDprintf("first mismatched byte %llu: expected 0x%02X was 0x%02X\n", + i, exp[i], act_buf[i]); + TEST_ERROR; + } + } + + if (H5FDclose(raw_vfile) < 0) + TEST_ERROR; + raw_vfile = NULL; + + HDfree(act_buf); + act_buf = NULL; + + return 0; + +error: + if (act_buf != NULL) + HDfree(act_buf); + if (raw_vfile != NULL) + H5FDclose(raw_vfile); + return -1; +} /* end compare_file_bytes_exactly() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * Do a manual read of the onion history (separate, single "Onion" file). + * Verify that the history data is well-formed and matches the expected state. + * + * Inspect file contents on backing store. + * Return -1 on problem, 0 if okay. + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static int +verify_history_as_expected_onion(H5FD_t *raw_file, \ + struct expected_history *filter) +{ + unsigned char *buf = NULL; /* allocated area for actual file bytes */ + struct H5FD__onion_history_header hdr_out; + struct H5FD__onion_whole_history whs_out; + struct H5FD__onion_revision_record rev_out; + uint64_t filesize = 0; + uint64_t readsize = 0; + size_t i = 0; + + hdr_out.magic = H5FD__ONION_HEADER_MAGIC; + hdr_out.version = H5FD__ONION_HEADER_VERSION_CURR; + + whs_out.magic = H5FD__ONION_WHOLE_HISTORY_MAGIC; + whs_out.version = H5FD__ONION_WHOLE_HISTORY_VERSION_CURR; + whs_out.n_revisions = 0; + whs_out.record_pointer_list = NULL; + + rev_out.magic = H5FD__ONION_REVISION_RECORD_MAGIC; + rev_out.version = H5FD__ONION_REVISION_RECORD_VERSION_CURR; + rev_out.archival_index.magic = H5FD__ONION_ARCHIVAL_INDEX_MAGIC; + rev_out.archival_index.version = H5FD__ONION_ARCHIVAL_INDEX_VERSION_CURR; + + + filesize = (uint64_t)H5FDget_eof(raw_file, H5FD_MEM_DRAW); + if (H5FDset_eoa(raw_file, H5FD_MEM_DRAW, filesize) < 0) + TEST_ERROR; + + /* Injest onion header. + */ + + readsize = MIN(filesize, H5FD__ONION_ENCODED_SIZE_HEADER); + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * readsize); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, readsize, buf) < 0) + TEST_ERROR; + + readsize = H5FD_onion_history_header_decode(buf, &hdr_out); + if (0 == readsize) + TEST_ERROR; + if (H5FD__ONION_HEADER_VERSION_CURR != hdr_out.version) + TEST_ERROR; + if (HDmemcmp(&hdr_out.checksum, &buf[readsize - 4], 4) != 0) + TEST_ERROR; + if (hdr_out.checksum != H5_checksum_fletcher32(buf, readsize - 4)) + TEST_ERROR; + if (filter->page_size != hdr_out.page_size) + TEST_ERROR; + if (hdr_out.whole_history_addr + hdr_out.whole_history_size != filesize) + TEST_ERROR; + if (filter->origin_eof != hdr_out.origin_eof) + TEST_ERROR; + + HDfree(buf); + buf = NULL; + + /* Injest whole-history. + */ + + readsize = hdr_out.whole_history_size; + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * readsize); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, \ + hdr_out.whole_history_addr, readsize, buf) < 0) + TEST_ERROR; + + /* Initial read, get count of revisions */ + readsize = H5FD_onion_whole_history_decode(buf, &whs_out); + if (0 == readsize) + TEST_ERROR; + if (H5FD__ONION_WHOLE_HISTORY_VERSION_CURR != whs_out.version) + TEST_ERROR; + if (HDmemcmp(&whs_out.checksum, &buf[readsize - 4], 4) != 0) + TEST_ERROR; + if (whs_out.checksum != H5_checksum_fletcher32(buf, readsize - 4)) + TEST_ERROR; + if (filter->n_revisions != whs_out.n_revisions) + TEST_ERROR; + + /* Final read, populate pointers to revision records */ + whs_out.record_pointer_list = (struct H5FD__onion_record_pointer *) \ + HDcalloc(whs_out.n_revisions, \ + sizeof(struct H5FD__onion_record_pointer)); + if (NULL == whs_out.record_pointer_list) + TEST_ERROR; + if (H5FD_onion_whole_history_decode(buf, &whs_out) != readsize) + TEST_ERROR; + + /* Re-use buffer space to sanity-check checksum for record pointer(s). */ + HDassert(readsize >= sizeof(struct H5FD__onion_record_pointer)); + for (i = 0; i < whs_out.n_revisions; i++) { +#if 0 + uint32_t sum = 0; +#endif + + HDmemcpy(buf, &whs_out.record_pointer_list[i].phys_addr, 8); + HDmemcpy(buf + 8, &whs_out.record_pointer_list[i].record_size, 8); +#if 0 + sum = H5_checksum_fletcher32(buf, 16); +#endif + if (whs_out.record_pointer_list[i].checksum != \ + H5_checksum_fletcher32(buf, 16)) + TEST_ERROR; + } + + HDfree(buf); + buf = NULL; + + /* Injest revision(s). + */ + + for (i = 0; i < whs_out.n_revisions; i++) { + struct H5FD__onion_record_pointer *rpp = \ + &whs_out.record_pointer_list[i]; + struct expected_revision *erp = &filter->revisions[i]; + + rev_out.archival_index.list = NULL; + rev_out.archival_index.n_entries = 0; + rev_out.archival_index.page_size_log2 = 0; + rev_out.comment_size = 0; + rev_out.comment = NULL; + rev_out.username_size = 0; + rev_out.username = NULL; + + readsize = rpp->record_size; + buf = (unsigned char *)HDmalloc((size_t)rpp->record_size); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(raw_file, H5FD_MEM_DRAW, H5P_DEFAULT, rpp->phys_addr, \ + rpp->record_size, buf) < 0) + TEST_ERROR; + + /* Initial revision read -- get fixed components */ + readsize = H5FD_onion_revision_record_decode(buf, &rev_out); + if (0 == readsize) + TEST_ERROR; + if (rpp->record_size != readsize) + TEST_ERROR; + if (H5FD__ONION_REVISION_RECORD_MAGIC != rev_out.magic) + TEST_ERROR; + if (H5FD__ONION_REVISION_RECORD_VERSION_CURR != rev_out.version) + TEST_ERROR; + if (HDmemcmp(&rev_out.checksum, &buf[readsize - 4], 4) != 0) + TEST_ERROR; + if (rev_out.checksum != H5_checksum_fletcher32(buf, readsize - 4)) + TEST_ERROR; + if (erp->revision_id != rev_out.revision_id) + TEST_ERROR; + if (erp->parent_revision_id != rev_out.parent_revision_id) + TEST_ERROR; + if (erp->logi_eof != rev_out.logi_eof) + TEST_ERROR; + + /* Final read, get variable-length data */ + rev_out.comment = (char *)HDmalloc((size_t)rev_out.comment_size); + if (NULL == rev_out.comment) + TEST_ERROR; + rev_out.archival_index.list = \ + (struct H5FD__onion_index_entry *)HDcalloc( \ + rev_out.archival_index.n_entries, \ + sizeof(struct H5FD__onion_index_entry)); + if (NULL == rev_out.archival_index.list) + TEST_ERROR; + rev_out.username = (char *)HDmalloc((size_t)rev_out.username_size); + if (NULL == rev_out.username) + TEST_ERROR; + + readsize = H5FD_onion_revision_record_decode(buf, &rev_out); + if (rpp->record_size != readsize) + TEST_ERROR; + + /* Compare revision info with expected filter */ + if (erp->comment == NULL) { + if (rev_out.comment_size != 0) + TEST_ERROR; + } + else { + if (HDstrlen(rev_out.comment) != HDstrlen(erp->comment)) + TEST_ERROR; + if (HDstrcmp(rev_out.comment, erp->comment) != 0) + TEST_ERROR; + } + if (erp->n_index_entries != (uint64_t)(-1) \ + && erp->n_index_entries != rev_out.archival_index.n_entries) + TEST_ERROR; + + + HDfree(buf); + buf = NULL; + + HDfree(rev_out.comment); + rev_out.comment = NULL; + + HDfree(rev_out.archival_index.list); + rev_out.archival_index.list = NULL; + + HDfree(rev_out.username); + rev_out.username = NULL; + } + + HDfree(whs_out.record_pointer_list); + whs_out.record_pointer_list = NULL; + + return 0; + +error: + if (buf != NULL) + HDfree(buf); + if (rev_out.comment != NULL) + HDfree(rev_out.comment); + if (rev_out.archival_index.list != NULL) + HDfree(rev_out.archival_index.list); + if (rev_out.username != NULL) + HDfree(rev_out.username); + if (whs_out.record_pointer_list != NULL) + HDfree(whs_out.record_pointer_list); + + return -1; + +} /* end verify_history_as_expected_onion() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * Verify file bytes on the backing store + * + onion storage target + * + create from nothing + * + stage 0 (initializing) + * + open (not yet written) + * + "Empty" .h5 file created + * + .onion file created w/ only header (0 whole-hist addr) + * + .onion.recovery created w/ "empty" whole-history + * + Cannot open onionized canonical file (incomplete history, no rev) + * + * Inspect file contents on backing store. + * Return -1 on problem, 0 if okay. + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static int +verify_stored_onion_create_0_open(struct onion_filepaths *paths, + H5FD_onion_fapl_info_t *onion_info) +{ + H5FD_t *file = NULL; /* virtual file to look at raw file contents */ + unsigned char *act_buf = NULL; /* allocated area for actual file bytes */ + hid_t fapl_id = onion_info->backing_fapl_id; +#if 0 + herr_t err_ret = FAIL; +#endif + unsigned char hdr_exp_bytes[] = { + 'O', 'H', 'D', 'H', + 1, 1, 0, 0, + 0, 0, 0, 0, /* page-size encoded below */ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 20, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, /* checksum encoded below */ + }; + size_t whs_exp_bytes_size = 20; + unsigned char whs_exp_bytes[] = { + 'O', 'W', 'H', 'S', + 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 /* checksum encoded below */ + }; + unsigned char *ptr = NULL; + uint32_t sum = 0; + hid_t onion_fapl_id = H5I_INVALID_HID; + + onion_fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == onion_fapl_id) + TEST_ERROR + if (H5Pset_fapl_onion(onion_fapl_id, onion_info) < 0) + TEST_ERROR + + /* Finish populating expected header bytes + */ + ptr = hdr_exp_bytes + 8; /* WARNING: must match format */ + UINT32ENCODE(ptr, onion_info->page_size); + sum = H5_checksum_fletcher32(hdr_exp_bytes, \ + H5FD__ONION_ENCODED_SIZE_HEADER - 4); + ptr = hdr_exp_bytes + H5FD__ONION_ENCODED_SIZE_HEADER - 4; + UINT32ENCODE(ptr, sum); + ptr = NULL; + + /* Finish populating expected whole-history bytes + */ + sum = H5_checksum_fletcher32(whs_exp_bytes, \ + H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY - 4); + ptr = whs_exp_bytes + H5FD__ONION_ENCODED_SIZE_WHOLE_HISTORY - 4; + UINT32ENCODE(ptr, sum); + ptr = NULL; + +#if 0 /* TODO: fails because EOA not set, not because it's empty */ + /* Look at h5 file: should have zero bytes. + */ + + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + + act_buf = (unsigned char *)HDcalloc(1, 8); /* any size would do */ + if (NULL == act_buf) + TEST_ERROR; + + H5E_BEGIN_TRY { + err_ret = H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 1, act_buf); + } H5E_END_TRY; + if (err_ret != FAIL) + TEST_ERROR; /* cannot read from empty file */ + + HDfree(act_buf); + act_buf = NULL; + + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; +#endif /* TODO */ + + /* Look at onion file: should have header. + */ + + if (compare_file_bytes_exactly(paths->onion, fapl_id, \ + H5FD__ONION_ENCODED_SIZE_HEADER, hdr_exp_bytes) < 0) + TEST_ERROR; + + /* Look at history backing file: should have nascent whole-history. + */ + + if (compare_file_bytes_exactly(paths->recovery, fapl_id, \ + whs_exp_bytes_size, whs_exp_bytes) < 0) + TEST_ERROR; + + /* Inspect .h5 file contents + */ + + if (compare_file_bytes_exactly(paths->canon, fapl_id, 8, \ + (const unsigned char *)"ONIONEOF") < 0) + { + TEST_ERROR; + } + + if (H5Pclose(onion_fapl_id) < 0) + TEST_ERROR; + onion_fapl_id = H5I_INVALID_HID; + + return 0; + +error: + if (file != NULL) + (void)H5FDclose(file); + if (act_buf != NULL) + HDfree(act_buf); + if (onion_fapl_id != H5I_INVALID_HID) + (void)H5Pclose(onion_fapl_id); + + return -1; +} /* end verify_stored_onion_create_0_open() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_create_oniontarget() + * + * Purpose: Test the ability of the Onion VFD to create a valid + * 'onionized' file. + * + * When `truncate_canonical` is FALSE, the canonical file is + * nonexistent on the backing store on onion-creation. + * When `truncate_canonical` is TRUE, a canonical file is created + * on the backing store with known contents, which are to be + * truncated on onion-creation. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_create_oniontarget(hbool_t truncate_canonical, hbool_t with_initial_data) +{ + const char *basename = "somesuch"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_MAGIC, + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "initial commit" /* comment */ + }; + H5FD_t *vfile_raw = NULL; /* virtual file to look at raw file contents */ + H5FD_t *vfile_rw = NULL; /* Onion virtual file for read/write */ + H5FD_t *vfile_ro = NULL; /* Onion virtual file for read-only */ + struct expected_history filter; + char *buf = NULL; + + if (TRUE == truncate_canonical && TRUE == with_initial_data) { + TESTING("onion creation; truncate extant canonical; w/ initial data"); + } + else + if (TRUE == truncate_canonical) { + TESTING("onion creation; truncate extant canonical; no initial data"); + } + else + if (TRUE == with_initial_data) { + TESTING("onion creation; no extant canonical; w/ initial data"); + } + else { + TESTING("onion creation; no extant canonical; no initial data"); + } + + /********* + * SETUP * + *********/ + + onion_info.backing_fapl_id = h5_fileaccess(); + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + paths = onion_filepaths_init(basename, &onion_info); + if (NULL == paths) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /* create canonical file to be truncated */ + if (TRUE == truncate_canonical) { + /* Create canonical file. */ + vfile_raw = H5FDopen(paths->canon, flags_create_s, \ + onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == vfile_raw) + TEST_ERROR; + if (H5FDset_eoa(vfile_raw, H5FD_MEM_DRAW, b_list_size_s) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_raw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, \ + b_list_size_s, b_list_s) < 0) + { + TEST_ERROR; + } + if (H5FDclose(vfile_raw) < 0) + TEST_ERROR; + vfile_raw = NULL; + H5E_BEGIN_TRY { + vfile_raw = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, \ + HADDR_UNDEF); + } H5E_END_TRY; + if (NULL != vfile_raw) + TEST_ERROR; /* no onion history to onion-open created file */ + + /* Create "existing onion file". */ + vfile_raw = H5FDopen(paths->onion, flags_create_s, \ + onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == vfile_raw) + TEST_ERROR; + if (H5FDset_eoa(vfile_raw, H5FD_MEM_DRAW, b_list_size_s) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_raw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, \ + 23, "prior history stand-in") < 0) + { + TEST_ERROR; + } + if (H5FDclose(vfile_raw) < 0) + TEST_ERROR; + vfile_raw = NULL; + } /* end if to create canonical file for truncation */ + + /* + * OPENED + */ + + /* Begin creation of onionized file from nothing. + */ + + vfile_rw = H5FDopen(paths->canon, flags_create_s, fapl_id, HADDR_UNDEF); + if (NULL == vfile_rw) + TEST_ERROR; + + if (verify_stored_onion_create_0_open(paths, &onion_info) < 0) + TEST_ERROR + + H5E_BEGIN_TRY { + vfile_ro = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + } H5E_END_TRY; + if (vfile_ro != NULL) + TEST_ERROR; /* onionization (creation) not complete; nothing to open */ + + /* + * WRITING + */ + + if (TRUE == with_initial_data) { + haddr_t half_size = 0; + haddr_t buf_size = 0; + + /* Write the sub-page entry at addr 0 */ + if (4 >= onion_info.page_size) + TEST_ERROR; + if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, 4) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 4, a_list_s) \ + < 0) + { + TEST_ERROR; + } + + /* Verify logical file contents. */ + buf = (char *)HDmalloc(sizeof(char) * 4); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, 4, buf) < 0) { + TEST_ERROR; + } + if (HDmemcmp(a_list_s, buf, 4) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + + /* Write the latter half of buffer at addr 0 (more than one page) */ + half_size = a_list_size_s / 2; + buf_size = a_list_size_s - half_size; + if (buf_size <= onion_info.page_size) + TEST_ERROR; + if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, buf_size) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, buf_size, \ + a_list_s + half_size) < 0) + { + TEST_ERROR; + } + + /* Verify logical file contents. */ + buf = (char *)HDmalloc(sizeof(char) * buf_size); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, buf_size, buf) \ + < 0) + { + TEST_ERROR; + } + if (HDmemcmp(a_list_s + half_size, buf, buf_size) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + + /* Overwrite existing data with entire buffer at addr 0 */ + if (H5FDset_eoa(vfile_rw, H5FD_MEM_DRAW, a_list_size_s) < 0) + TEST_ERROR; + if (H5FDwrite(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, \ + a_list_s) < 0) + { + TEST_ERROR; + } + + /* Verify logical file contents. */ + buf = (char *)HDmalloc(sizeof(char) * a_list_size_s); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(vfile_rw, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, \ + buf) < 0) + { + TEST_ERROR; + } + if (HDmemcmp(a_list_s, buf, a_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + } /* end if writing data to logical file */ + + /* + * CLOSED + */ + + if (H5FDclose(vfile_rw) < 0) + TEST_ERROR; + vfile_rw = NULL; + + /* Look at h5 file: should be known-empty + */ + + if (compare_file_bytes_exactly(paths->canon, onion_info.backing_fapl_id, \ + 8, (const unsigned char *)"ONIONEOF") < 0) + { + TEST_ERROR; + } + + /* Look at recovery file: should be gone. + */ + + H5E_BEGIN_TRY { + vfile_raw = H5FDopen(paths->recovery, H5F_ACC_RDONLY, \ + onion_info.backing_fapl_id, HADDR_UNDEF); + } H5E_END_TRY; + if (NULL != vfile_raw) + TEST_ERROR; + + /* Inspect onion file. + */ + + vfile_raw = H5FDopen(paths->onion, H5F_ACC_RDONLY, \ + onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == vfile_raw) + TEST_ERROR; + + filter.page_size = onion_info.page_size; + filter.n_revisions = 1; + filter.origin_eof = 0; + filter.revisions[0].comment = onion_info.comment; + filter.revisions[0].n_index_entries = (uint64_t)(-1); /* don't care */ + filter.revisions[0].revision_id = 0; + filter.revisions[0].parent_revision_id = 0; + filter.revisions[0].logi_eof = (TRUE == with_initial_data) \ + ? a_list_size_s : 0; + + if (verify_history_as_expected_onion(vfile_raw, &filter) < 0) + TEST_ERROR; + + if (H5FDclose(vfile_raw) < 0) + TEST_ERROR; + vfile_raw = NULL; + + /* R/O open the file with Onion VFD; inspect logial file. + */ + + vfile_ro = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == vfile_ro) + TEST_ERROR; + + if (TRUE == with_initial_data) { + if (H5FDget_eof(vfile_ro, H5FD_MEM_DRAW) != a_list_size_s) + TEST_ERROR; + if (H5FDget_eoa(vfile_ro, H5FD_MEM_DRAW) != 0) + TEST_ERROR; + if (H5FDset_eoa(vfile_ro, H5FD_MEM_DRAW, a_list_size_s) < 0) + TEST_ERROR; + buf = (char *)HDmalloc(sizeof(char) * a_list_size_s * 64); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(vfile_ro, H5FD_MEM_DRAW, H5P_DEFAULT, 0, a_list_size_s, \ + buf) < 0) + { + TEST_ERROR; + } + if (HDmemcmp(a_list_s, buf, a_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + } /* end if data was written to initial revision */ + else { + if (H5FDget_eoa(vfile_ro, H5FD_MEM_DRAW) != 0) + TEST_ERROR; + if (H5FDget_eof(vfile_ro, H5FD_MEM_DRAW) != 0) + TEST_ERROR; + } /* end if initial revision has no data */ + + if (H5FDclose(vfile_ro) < 0) + TEST_ERROR; + vfile_ro = NULL; + + /* + * CLEANUP + */ + + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + if (buf != NULL) + HDfree(buf); + + if (vfile_raw != NULL) + (void)H5FDclose(vfile_raw); + if (vfile_rw != NULL) + (void)H5FDclose(vfile_rw); + if (vfile_ro != NULL) + (void)H5FDclose(vfile_ro); + + if (fapl_id != H5I_INVALID_HID) + (void)H5Pclose(fapl_id); + + return -1; +} /* end test_create_oniontarget() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_several_revisions_with_logical_gaps() + * + * Purpose: Test the ability of the Onion VFD to create a valid + * 'onionized' file. + * + * When `truncate_canonical` is FALSE, the canonical file is + * nonexistent on the backing store on onion-creation. + * When `truncate_canonical` is TRUE, a canonical file is created + * on the backing store with known contents, which are to be + * truncated on onion-creation. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_several_revisions_with_logical_gaps(void) +{ + const char *basename = "somesuch"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_MAGIC, + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* flags */ + "first" /* comment */ + }; + H5FD_t *file = NULL; /* Onion virtual file for read/write */ + struct expected_history filter; + unsigned char *buf = NULL; + struct revise_revision about[4]; +#if 0 + struct H5FD__onion_history_header hdr_out; + struct H5FD__onion_revision_record rev_out; +#endif + struct H5FD__onion_whole_history whs_out; + size_t i = 0; + haddr_t size = 0; + uint64_t a_off = ONION_TEST_PAGE_SIZE_5 + 7; /* 39 */ + uint64_t b_off = (((a_off + a_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) \ + >> 5) << 5) + ONION_TEST_PAGE_SIZE_5 + 7; /* full page between */ + + TESTING("multiple revisions with gaps and overlap"); + + /********* + * SETUP * + *********/ + +#if 0 + hdr_out.magic = H5FD__ONION_HEADER_MAGIC; + hdr_out.version = H5FD__ONION_HEADER_VERSION_CURR; + rev_out.magic = H5FD__ONION_REVISION_RECORD_MAGIC; + rev_out.version = H5FD__ONION_REVISION_RECORD_VERSION_CURR; +#endif + whs_out.magic = H5FD__ONION_WHOLE_HISTORY_MAGIC; + whs_out.version = H5FD__ONION_WHOLE_HISTORY_VERSION_CURR; + whs_out.n_revisions = 0; + whs_out.record_pointer_list = NULL; + + onion_info.backing_fapl_id = h5_fileaccess(); + + paths = onion_filepaths_init(basename, &onion_info); + if (NULL == paths) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /* Empty first revision */ + about[0].magic = ONION_TEST_REV_REV_MAGIC; + about[0].truncate = TRUE; + about[0].revision_id = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[0].comment = "first"; + about[0].n_writes = 0; + + about[1].magic = ONION_TEST_REV_REV_MAGIC; + about[1].truncate = FALSE; + about[1].revision_id = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[1].comment = "second"; + about[1].n_writes = 1; + about[1].writes[0].offset = a_off; + about[1].writes[0].size = a_list_size_s; + about[1].writes[0].buf = a_list_s; + + about[2].magic = ONION_TEST_REV_REV_MAGIC; + about[2].truncate = FALSE; + about[2].revision_id = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[2].comment = "third"; + about[2].n_writes = 1; /* TODO: several writes */ + about[2].writes[0].offset = b_off; + about[2].writes[0].size = b_list_size_s; + about[2].writes[0].buf = b_list_s; + + about[3].magic = ONION_TEST_REV_REV_MAGIC; + about[3].truncate = FALSE; + about[3].revision_id = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[3].comment = "fourth"; + about[3].n_writes = 1; + about[3].writes[0].offset = 0; + about[3].writes[0].size = a_list_size_s; + about[3].writes[0].buf = a_list_s; + + if (do_onion_open_and_writes(paths->canon, &onion_info, 4, about) < 0) + TEST_ERROR; + + /* Inspect logical file */ + + /* Empty first revision */ + onion_info.revision_id = 0; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + if (0 != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* One offset block in second revision */ + onion_info.revision_id = 1; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + size = a_off + a_list_size_s; + if (size != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * size); + if (NULL == buf) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0) + TEST_ERROR; + for (i = 0; i < a_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + a_off, a_list_s, a_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + /* Repeat read at page offset; test possible read offset error */ + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) \ + * ONION_TEST_PAGE_SIZE_5); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, ONION_TEST_PAGE_SIZE_5, \ + ONION_TEST_PAGE_SIZE_5, buf) < 0) + TEST_ERROR; + size = a_off - ONION_TEST_PAGE_SIZE_5; + for (i = 0; i < size; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + size, a_list_s, ONION_TEST_PAGE_SIZE_5 - size) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* Two offset blocks in third revision */ + onion_info.revision_id = 2; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + size = b_off + b_list_size_s; + if (size != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * size); + if (NULL == buf) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0) + TEST_ERROR; + for (i = 0; i < a_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + a_off, a_list_s, a_list_size_s) != 0) + TEST_ERROR; + for (i = a_off + a_list_size_s; i < b_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + b_off, b_list_s, b_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* From start and partial overwrite in fourth revision */ + onion_info.revision_id = 3; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + size = b_off + b_list_size_s; + if (size != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * size); + if (NULL == buf) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, size) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, size, buf) < 0) + TEST_ERROR; + if (HDmemcmp(buf, a_list_s, a_list_size_s) != 0) + TEST_ERROR; + if (HDmemcmp(buf + a_list_size_s, \ + a_list_s + a_list_size_s - a_off, \ + a_off) != 0) + TEST_ERROR; + for (i = a_off + a_list_size_s; i < b_off; i++) { + if (0 != buf[i]) + TEST_ERROR; + } + if (HDmemcmp(buf + b_off, b_list_s, b_list_size_s) != 0) + TEST_ERROR; + HDfree(buf); + buf = NULL; + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* No fifth revision */ + onion_info.revision_id = 4; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + H5E_BEGIN_TRY { + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + } H5E_END_TRY; + if (NULL != file) + TEST_ERROR; + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + /* Inspect history construction */ + + file = H5FDopen(paths->onion, H5F_ACC_RDONLY, \ + onion_info.backing_fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + + filter.page_size = onion_info.page_size; + filter.n_revisions = 4; + filter.origin_eof = 0; + + filter.revisions[0].comment = "first"; + filter.revisions[0].n_index_entries = 0; + filter.revisions[0].revision_id = 0; + filter.revisions[0].parent_revision_id = 0; + filter.revisions[0].logi_eof = 0; + + filter.revisions[1].comment = "second"; + filter.revisions[1].n_index_entries = \ + (a_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5; + filter.revisions[1].revision_id = 1; + filter.revisions[1].parent_revision_id = 0; + filter.revisions[1].logi_eof = a_off + a_list_size_s; + + filter.revisions[2].comment = "third"; + filter.revisions[2].n_index_entries = filter.revisions[1].n_index_entries \ + + ((b_list_size_s + ONION_TEST_PAGE_SIZE_5 - 1) >> 5); + filter.revisions[2].revision_id = 2; + filter.revisions[2].parent_revision_id = 1; + filter.revisions[2].logi_eof = b_off + b_list_size_s; + + filter.revisions[3].comment = "fourth"; + filter.revisions[3].n_index_entries = filter.revisions[2].n_index_entries \ + + 1; + filter.revisions[3].revision_id = 3; + filter.revisions[3].parent_revision_id = 2; + filter.revisions[3].logi_eof = b_off + b_list_size_s; + + if (verify_history_as_expected_onion(file, &filter) < 0) + TEST_ERROR; + + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + + /* + * CLEANUP + */ + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + if (whs_out.record_pointer_list != NULL) + HDfree(whs_out.record_pointer_list); + if (buf != NULL) + HDfree(buf); + if (file != NULL) + (void)H5FDclose(file); + if (fapl_id != H5I_INVALID_HID) + (void)H5Pclose(fapl_id); + + return -1; +} /* end test_several_revisions_with_logical_gaps() */ + +/*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * + * Function: do_onion_open_and_writes + * + * Purpose: Automate the process of creating/opening a file and performing + * a series of writes. + * + * Return: Success : 0 + * Failure : -1 + * + *- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static int +do_onion_open_and_writes(const char *filename, \ + H5FD_onion_fapl_info_t *onion_info_p, size_t n_ops, \ + struct revise_revision *about) +{ + hid_t fapl_id = H5I_INVALID_HID; + H5FD_t *file = NULL; /* Onion virtual file for read/write */ + unsigned char *buf_vfy = NULL; + size_t i = 0; + + for (i = 0; i < n_ops; i++) { + size_t j = 0; + unsigned int flags = H5F_ACC_RDWR; + + if (about[i].magic != ONION_TEST_REV_REV_MAGIC) + goto error; + if (i != 0 && about[i].truncate == TRUE) + goto error; + + if (TRUE == about[i].truncate) + flags |= H5F_ACC_CREAT | H5F_ACC_TRUNC; + + onion_info_p->revision_id = about[i].revision_id; + if (about[i].comment != NULL) { + j = MIN(strlen(about[i].comment), H5FD_ONION_FAPL_INFO_COMMENT_MAX_LEN); + HDmemcpy(onion_info_p->comment, about[i].comment, j); + } + onion_info_p->comment[j] = '\0'; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + goto error; + if (H5Pset_fapl_onion(fapl_id, onion_info_p) < 0) + goto error; + file = H5FDopen(filename, flags, fapl_id, HADDR_UNDEF); + if (NULL == file) + goto error; + + for (j = 0; j < about[i].n_writes; j++) { + struct write_info *wi = &about[i].writes[j]; + + /* Write to file */ + if (H5FDget_eoa(file, H5FD_MEM_DRAW) < wi->offset + wi->size \ + && H5FDset_eoa(file, H5FD_MEM_DRAW, wi->offset + wi->size) < 0) + TEST_ERROR; + if (H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, wi->offset, \ + wi->size, wi->buf) < 0) + TEST_ERROR; + /* Verify write as expected */ + buf_vfy = (unsigned char *)HDmalloc( + sizeof(unsigned char) * wi->size); + if (NULL == buf_vfy) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, wi->offset, \ + wi->size, buf_vfy) < 0) + TEST_ERROR; + if (HDmemcmp(buf_vfy, wi->buf, wi->size) != 0) { + const unsigned char *_buf = wi->buf; + size_t z = 0; + HDputs("i exp act"); + for (z = 0; z < wi->size; z++) + HDprintf("%02x %c %c\n", z, _buf[z], buf_vfy[z]); + fflush(stdout); + TEST_ERROR; + } + HDfree(buf_vfy); + buf_vfy = NULL; + } /* end for each write */ + + if (H5FDclose(file) < 0) + goto error; + file = NULL; + if (H5Pclose(fapl_id) < 0) + goto error; + fapl_id = H5I_INVALID_HID; + } /* end for each open-close cycle */ + + return 0; + +error: + if (file != NULL) + (void)H5FDclose(file); + if (fapl_id != H5I_INVALID_HID) + (void)H5Pclose(fapl_id); + if (buf_vfy != NULL) + HDfree(buf_vfy); + + return -1; +} /* end do_onion_open_and_writes() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_page_aligned_history_create() + * + * Purpose: Verify that, when specified in FAPL on onionization/creation, + * All history writes are aligned to page-size boundaries. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_page_aligned_history_create(void) +{ + const char *basename = "somesuch"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_MAGIC, + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT, + "initial commit" /* comment */ + }; + H5FD_t *file = NULL; /* Onion virtual file for read/write */ + unsigned char *buf = NULL; + struct revise_revision about[2]; + struct H5FD__onion_history_header hdr_out; +#if 0 + struct H5FD__onion_revision_record rev_out; +#endif + struct H5FD__onion_whole_history whs_out; + size_t i = 0; + uint64_t a_off = b_list_size_s - a_list_size_s; + + TESTING("page-aligned history on onion-created file"); + + /********* + * SETUP * + *********/ + + hdr_out.magic = H5FD__ONION_HEADER_MAGIC; + hdr_out.version = H5FD__ONION_HEADER_VERSION_CURR; +#if 0 + rev_out.magic = H5FD__ONION_REVISION_RECORD_MAGIC; + rev_out.version = H5FD__ONION_REVISION_RECORD_VERSION_CURR; +#endif + whs_out.magic = H5FD__ONION_WHOLE_HISTORY_MAGIC; + whs_out.version = H5FD__ONION_WHOLE_HISTORY_VERSION_CURR; + whs_out.n_revisions = 0; + whs_out.record_pointer_list = NULL; + + onion_info.backing_fapl_id = h5_fileaccess(); + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + paths = onion_filepaths_init(basename, &onion_info); + if (NULL == paths) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + about[0].magic = ONION_TEST_REV_REV_MAGIC; + about[0].truncate = TRUE; + about[0].revision_id = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[0].comment = "initial_commit"; + about[0].n_writes = 1; + about[0].writes[0].offset = 0; + about[0].writes[0].size = b_list_size_s; + about[0].writes[0].buf = b_list_s; + + about[1].magic = ONION_TEST_REV_REV_MAGIC; + about[1].truncate = FALSE; + about[1].revision_id = H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST; + about[1].comment = "second"; + about[1].n_writes = 1; + about[1].writes[0].offset = a_off; + about[1].writes[0].size = a_list_size_s; + about[1].writes[0].buf = a_list_s; + + if (do_onion_open_and_writes(paths->canon, &onion_info, 2, about) < 0) + TEST_ERROR; + + /* Inspect logical file */ + buf = (unsigned char *)HDmalloc(sizeof(unsigned char) * b_list_size_s); + if (NULL == buf) + TEST_ERROR; + file = H5FDopen(paths->canon, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + if (b_list_size_s != H5FDget_eof(file, H5FD_MEM_DRAW)) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, b_list_size_s) < 0) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, b_list_size_s, buf) < 0) + TEST_ERROR; + if (HDmemcmp(a_list_s, buf + a_off, a_list_size_s) != 0) { + size_t k; + HDprintf("aoff: %llu\n", a_off); + HDputs("i exp act"); + for (k = 0; k < b_list_size_s; k++) { + HDprintf("%3llu:: %c : %c\n", + k, (k < a_off) ? ' ' : a_list_s[k - a_off], buf[k]); + } + fflush(stdout); + TEST_ERROR; + } + if (HDmemcmp(b_list_s, buf, a_off) != 0){ + size_t k; + HDprintf("aoff: %llu\n", a_off); + HDputs("i exp act"); + for (k = 0; k < b_list_size_s; k++) { + HDprintf("%3llu:: %c : %c\n", + k, (k < a_off) ? b_list_s[k] : ' ', buf[k]); + } + fflush(stdout); + TEST_ERROR; + } + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + HDfree(buf); + buf = NULL; + + /* Inspect history construction */ + + file = H5FDopen(paths->onion, H5F_ACC_RDONLY, onion_info.backing_fapl_id, \ + HADDR_UNDEF); + if (NULL == file) + TEST_ERROR; + if (H5FDset_eoa(file, H5FD_MEM_DRAW, H5FDget_eof(file, H5FD_MEM_DRAW)) < 0) + TEST_ERROR; + + buf = (unsigned char *)HDmalloc(H5FD__ONION_ENCODED_SIZE_HEADER); + if (NULL == buf) + TEST_ERROR; + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, 0, \ + H5FD__ONION_ENCODED_SIZE_HEADER, buf) < 0) + TEST_ERROR; + if (H5FD_onion_history_header_decode(buf, &hdr_out) \ + != H5FD__ONION_ENCODED_SIZE_HEADER) + TEST_ERROR; + if (hdr_out.whole_history_addr & ((1 << 5) - 1)) /* 5::PAGE_SIZE_5 */ + TEST_ERROR; + HDfree(buf); + buf = NULL; + + buf = (unsigned char *)HDmalloc(hdr_out.whole_history_size); + if (H5FDread(file, H5FD_MEM_DRAW, H5P_DEFAULT, hdr_out.whole_history_addr,\ + hdr_out.whole_history_size, buf) < 0) + TEST_ERROR; + if (H5FD_onion_whole_history_decode(buf, &whs_out) \ + != hdr_out.whole_history_size) + TEST_ERROR; + if (whs_out.n_revisions != 2) + TEST_ERROR; + whs_out.record_pointer_list = (struct H5FD__onion_record_pointer *) \ + HDcalloc(whs_out.n_revisions, \ + sizeof(struct H5FD__onion_record_pointer)); + if (NULL == whs_out.record_pointer_list) + TEST_ERROR; + if (H5FD_onion_whole_history_decode(buf, &whs_out) \ + != hdr_out.whole_history_size) + TEST_ERROR; + HDfree(buf); + buf = NULL; + + for (i = 0; i < whs_out.n_revisions; i++) { + struct H5FD__onion_record_pointer *rr_p = \ + &whs_out.record_pointer_list[i]; + if (rr_p->phys_addr & ((1 << 5) - 1)) /* 5::PAGE_SIZE_5 */ + TEST_ERROR; + /* TODO: check phys_addr of each page entry? */ + } + + + HDfree(whs_out.record_pointer_list); + whs_out.record_pointer_list = NULL; + + if (H5FDclose(file) < 0) + TEST_ERROR; + file = NULL; + HDfree(buf); + buf = NULL; + + /* + * CLEANUP + */ + + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + if (whs_out.record_pointer_list != NULL) + HDfree(whs_out.record_pointer_list); + if (buf != NULL) + HDfree(buf); + if (file != NULL) + (void)H5FDclose(file); + if (fapl_id != H5I_INVALID_HID) + (void)H5Pclose(fapl_id); + + return -1; +} /* end test_page_aligned_history_create() */ + +/*----------------------------------------------------------------------------- + * + * Function: test_integration_create() + * + * Purpose: Create and make multiple revisions in an HDF5 file. + * + * Return: PASSED : 0 + * FAILED : -1 + * + *----------------------------------------------------------------------------- + */ +static int +test_integration_create(void) +{ + const char *basename = "example.h5"; + //const char *basename = "somesuch.h5"; + hid_t fapl_id = H5I_INVALID_HID; + struct onion_filepaths *paths = NULL; + H5FD_onion_fapl_info_t onion_info = { + H5FD_ONION_FAPL_INFO_MAGIC, + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5I_INVALID_HID, /* backing_fapl_id */ + ONION_TEST_PAGE_SIZE_5, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + H5FD_ONION_FAPL_INFO_CREATE_FLAG_ENABLE_PAGE_ALIGNMENT, + "initial commit" /* comment */ + }; + hid_t file_id = H5I_INVALID_HID; + + TESTING("onion-created HDF5 file with revisions"); + + /********* + * SETUP * + *********/ + + onion_info.backing_fapl_id = h5_fileaccess(); + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) + TEST_ERROR; + if (H5Pset_fapl_onion(fapl_id, &onion_info) < 0) + TEST_ERROR; + + paths = onion_filepaths_init(basename, &onion_info); + if (NULL == paths) + TEST_ERROR; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + + /* Create skeleton file */ +/* +HDputs("."); fflush(stdout); + file_id = H5Fcreate(paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (H5I_INVALID_HID == file_id) + TEST_ERROR; +HDputs("."); fflush(stdout); + + +/// + hsize_t dims[2] = {6, 8}; + + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hid_t space = H5Screate_simple(2, dims, maxdims); + + //hid_t space = H5Screate_simple(2, dims, NULL); + + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + hsize_t chunk[2] = {4, 4}; + herr_t status = H5Pset_chunk(dcpl, 2, chunk); + + int fillval = 99; + status = H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillval); + status = H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY); + hid_t dset = H5Dcreate(file_id, "/dset", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); + if (dset < 0) + TEST_ERROR + + if (H5Dclose(dset) < 0) + TEST_ERROR +/// + + + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; +*/ + + + // CREATE FILE WITHOUT ONION + + + hid_t file, space, dset, dcpl; /* Handles */ + herr_t status; + hsize_t dims[2] = {4, 7}, + maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}, + chunk[2] = {4, 4}; + int wdata[4][7], /* Write buffer */ + fillval, + i, j; + + /* + * Initialize data. + */ + for (i=0; i<4; i++) + for (j=0; j<7; j++) + wdata[i][j] = i * j - j; + + /* + * Create a new file using the default properties. + */ + //file = H5Fcreate ("example.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + file = H5Fcreate (paths->canon, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + /* + * Create dataspace with unlimited dimensions. + */ + space = H5Screate_simple (2, dims, maxdims); + + /* + * Create the dataset creation property list, and set the chunk + * size. + */ + dcpl = H5Pcreate (H5P_DATASET_CREATE); + status = H5Pset_chunk (dcpl, 2, chunk); + + /* + * Set the fill value for the dataset. + */ + fillval = 99; + status = H5Pset_fill_value (dcpl, H5T_NATIVE_INT, &fillval); + + /* + * Set the allocation time to "early". This way we can be sure + * that reading from the dataset immediately after creation will + * return the fill value. + */ + status = H5Pset_alloc_time (dcpl, H5D_ALLOC_TIME_EARLY); + + /* + * Create the dataset using the dataset creation property list. + */ + dset = H5Dcreate (file, "DS1", H5T_STD_I32LE, space, H5P_DEFAULT, dcpl, + H5P_DEFAULT); + + /* + * Write the data to the dataset. + */ + status = H5Dwrite (dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, + wdata[0]); + + /* + * Close and release resources. + */ + status = H5Pclose (dcpl); + status = H5Dclose (dset); + status = H5Sclose (space); + status = H5Fclose (file); + + +//////////////////////////// + + + +HDputs("."); fflush(stdout); +HDputs("."); fflush(stdout); + //file_id = H5Fopen("example.h5", H5F_ACC_RDWR, fapl_id); + file_id = H5Fopen(paths->canon, H5F_ACC_RDWR, fapl_id); +HDputs("."); fflush(stdout); + //file_id = H5Fopen(paths->canon, H5F_ACC_RDONLY, fapl_id); + //file_id = H5FDopen("example.h5", H5F_ACC_RDWR, fapl_id, HADDR_UNDEF); + if (H5I_INVALID_HID == file_id) { + printf("\n\n\n\nERROR OPENING\n\n\n\n"); + TEST_ERROR; + } +HDputs("."); fflush(stdout); + + +/// + dset = H5Dopen(file_id, "DS1", H5P_DEFAULT); + if (dset < 0) { + printf("\n\n\n\nERROR OPENING DSET\n\n\n\n"); + TEST_ERROR + } +HDputs("."); fflush(stdout); + int dset_data[2][2]; + for (i = 0; i < 2; i++) + for (j = 0; j < 2; j++) + dset_data[i][j] = i * 6 + j + 1; +HDputs("."); fflush(stdout); + status = H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset_data); +HDputs("."); fflush(stdout); +/// + + if (H5Fclose(file_id) < 0) + TEST_ERROR; + file_id = H5I_INVALID_HID; +HDputs("."); fflush(stdout); + + /* TODO */ + + /* + * CLEANUP + */ + + if (H5Pclose(fapl_id) < 0) + TEST_ERROR; + fapl_id = H5I_INVALID_HID; + + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + + PASSED(); + return 0; + +error: + + if (paths != NULL) { + HDremove(paths->canon); + HDremove(paths->onion); + HDremove(paths->recovery); + onion_filepaths_destroy(paths); + } + + if (file_id != H5I_INVALID_HID) + (void)H5Fclose(file_id); + if (fapl_id != H5I_INVALID_HID) + (void)H5Pclose(fapl_id); + + return -1; +} /* end test_integration_create() */ + +/*----------------------------------------------------------------------------- + * + * Function: main() + * + * Purpose: Perform unit tests on for the Onion VFD. + * + *----------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + + HDprintf("Testing Onion VFD functionality.\n"); + + h5_reset(); + + /* initialize */ + flags_create_s = H5F_ACC_RDWR | H5F_ACC_CREAT | H5F_ACC_TRUNC; + + /* Run tests. Return values on error are negative. */ + nerrors -= test_archival_index(); + nerrors -= test_revision_index(); + nerrors -= test_revision_index_collisions(); + nerrors -= test_revision_index_resizing(); + nerrors -= test_revision_index_to_archival_index(); + nerrors -= test_fapl(); + nerrors -= test_header_encode_decode(); + nerrors -= test_whole_history_encode_decode_empty(); + nerrors -= test_whole_history_encode_decode(); + nerrors -= test_revision_record_encode_decode(); + nerrors -= test_create_oniontarget(FALSE, FALSE); + nerrors -= test_create_oniontarget(TRUE, FALSE); + nerrors -= test_create_oniontarget(FALSE, TRUE); + nerrors -= test_create_oniontarget(TRUE, TRUE); + nerrors -= test_several_revisions_with_logical_gaps(); + nerrors -= test_page_aligned_history_create(); +// nerrors -= test_integration_create(); + +#if H5FD_ONION_ENABLE_INDEX_STATS + nerrors -= test_working_index_stats(); /* TODO */ +#endif /* H5FD_ONION_ENABLE_INDEX_STATS */ + + if (nerrors > 0) { + HDprintf("***** %d Onion TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + nerrors = 1; + } + else { + HDprintf("All Onion tests passed.\n"); + } + return nerrors; /* 0 if no errors, 1 if any errors */ + +} /* end main() */ + diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index eee9c53..8c6c890 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -81,7 +81,7 @@ const char *volnames[] = { * */ const char *drivernames[] = { - "sec2", "direct", "log", "windows", "stdio", "core", "family", "split", "multi", "mpio", "ros3", "hdfs", + "sec2", "direct", "log", "windows", "stdio", "core", "family", "split", "multi", "mpio", "ros3", "hdfs", "onion", }; #define NUM_VOLS (sizeof(volnames) / sizeof(volnames[0])) diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index 9d065f3..f8b6389 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -598,6 +598,7 @@ typedef enum { MPIO_VFD_IDX, ROS3_VFD_IDX, HDFS_VFD_IDX, + ONION_VFD_IDX, } driver_idx; /* The following include, h5tools_str.h, must be after the diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index 36114ba..448e967 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -50,6 +50,19 @@ static H5FD_hdfs_fapl_t hdfs_fa_g = { }; #endif /* H5_HAVE_LIBHDFS */ +static H5FD_onion_fapl_info_t onion_fa_g = { + H5FD_ONION_FAPL_INFO_MAGIC, + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + 32, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "indoor speaking voices", /* comment */ +}; +static int64_t onion_revision_g = -1; + /* module-scoped variables for XML option */ #define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd" #define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd" @@ -1263,7 +1276,9 @@ end_collect: case '3': vol_info_g.info_string = H5_optarg; break; - + case 'F': + onion_revision_g = atol(H5_optarg); + break; case '?': default: usage(h5tools_getprogname()); @@ -1402,6 +1417,10 @@ main(int argc, const char *argv[]) goto done; #endif } + else if (!HDstrcmp(driver_name_g, drivernames[ONION_VFD_IDX])) { + onion_fa_g.revision_id = onion_revision_g; + vfd_info.info = (void *)&onion_fa_g; + } if ((fapl_id = h5tools_get_fapl(H5P_DEFAULT, NULL, &vfd_info)) < 0) { error_msg("unable to create FAPL for file access\n"); |